diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index b7f7b2f..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/anthropics/claude-code/main/schemas/settings.schema.json", - "permissions": { - "allow": [ - "Read(//C/Users/carlos/Documents/Paper_CRPTO/**)", - "Bash(uv:*)", - "Bash(uv run:*)", - "Bash(uv sync:*)", - "Bash(uv lock:*)", - "Bash(uv pip:*)", - "Bash(just:*)", - "Bash(dvc status:*)", - "Bash(dvc dag:*)", - "Bash(dvc metrics:*)", - "Bash(dvc params:*)", - "Bash(dvc plots:*)", - "Bash(dvc exp:*)", - "Bash(dbt parse:*)", - "Bash(dbt compile:*)", - "Bash(dbt test:*)", - "Bash(dbt docs:*)", - "Bash(quarto --version)", - "Bash(quarto check:*)", - "Bash(quarto render book*)", - "Bash(quarto preview book*)", - "Bash(quarto add:*)", - "Bash(quarto list:*)", - "Bash(pytest:*)", - "Bash(ruff check:*)", - "Bash(ruff format:*)", - "Bash(mypy:*)", - "Bash(python --version)", - "Bash(python -c:*)", - "Bash(ls:*)", - "Bash(dir:*)", - "Bash(where:*)", - "Bash(find C:/Users/carlos/Documents/Paper_CRPTO:*)", - "Bash(git status)", - "Bash(git diff:*)", - "Bash(git log:*)", - "Bash(git branch:*)", - "Bash(git show:*)", - "Bash(pre-commit run:*)", - "Bash(.venv/Scripts/python.exe:*)", - "Bash(C:/Users/carlos/anaconda3/Scripts/uv.exe:*)" - ], - "deny": [ - "Bash(dvc repro crpto.data.splits*)", - "Bash(dvc repro crpto.pd.champion*)", - "Bash(dvc repro crpto.conformal.intervals*)", - "Bash(dvc repro crpto.conformal.validation*)", - "Bash(dvc repro crpto.portfolio.optimization*)", - "Bash(dvc repro crpto.portfolio.bound_exact_eval*)", - "Bash(rm -rf models/*)", - "Bash(rm -rf data/processed/conformal_intervals_mondrian*)", - "Bash(rm -rf data/processed/portfolio_bound_aware*)", - "Bash(git push --force*)", - "Bash(git commit --amend*)", - "Write(//C/Users/carlos/Documents/Paper_CRPTO/EXTRACTION_MANIFEST.json)", - "Write(//C/Users/carlos/Documents/Paper_CRPTO/models/pd_canonical.cbm)", - "Write(//C/Users/carlos/Documents/Paper_CRPTO/models/pd_canonical_calibrator.pkl)", - "Write(//C/Users/carlos/Documents/Paper_CRPTO/models/final_project_promotion.json)", - "Write(//C/Users/carlos/Documents/Paper_CRPTO/models/conformal_policy_status.json)" - ] - }, - "env": { - "QUARTO_LOG_LEVEL": "WARNING", - "PYTHONIOENCODING": "utf-8" - }, - "outputStyle": "default" -} diff --git a/.claude/skills/crpto-claim-sync/SKILL.md b/.claude/skills/crpto-claim-sync/SKILL.md deleted file mode 100644 index ce58460..0000000 --- a/.claude/skills/crpto-claim-sync/SKILL.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -name: crpto-claim-sync -description: Gate de sincronía de claims del paper - corre los tests de sync y coteja los números canónicos pool93 en qmd/tex contra el registro activo. Obligatorio tras cualquier edición de paper o supplement. ---- - -# /crpto-claim-sync - -Verifica que todos los números paper-facing siguen sincronizados con los -artefactos de gobernanza pool93. Fuente de verdad editorial: -`docs/research/active_claims_2026-07-04.md` (o el registro que lo supersede). - -## Pasos - -1. **Tests de sincronía** (los tres deben pasar): - ```powershell - uv run pytest tests/test_pool93_body_claim_sync.py tests/test_crpto_final_sync.py tests/test_supplement_table_sync.py -q - ``` - -2. **Cotejo de números canónicos** en las tres superficies editables - (`paper/CRPTO_ijds.qmd`, `paper/supplement_ijds.qmd`, - `paper/submission/CRPTO_ijds_submission.tex`). Buscar cada valor con Grep - — ojo con el formato LaTeX `184{,}832.48` además de `184,832.48`: - - | Claim | Valor | - | --- | --- | - | Retorno body point | `$184,832.48` | - | V(alpha=0.01) | `0.035350` | - | Gamma_CP(alpha=0.01) | `0.162616` | - | Markov cap | `0.345084` | - | B_u endpoint | `0.245084` | - | Alpha grid | `8/8`, violación exacta `0.0` | - | Return floor | `$170,464.54` (solo como floor, nunca como headline) | - | Frontera consolidada | `50,010` dedup / `27,508` elegibles | - | Búsqueda terminal | `37,068` políticas / `296,544` checks | - | Panel OOT | `276,869` préstamos | - | Endpoint conservador | `$170,467.27`, cap `0.273036` | - | Endpoint económico | `$223,458.14` | - -3. **Divergencia qmd vs tex**: si un claim aparece en el `.qmd` con un valor y - en el `.tex` con otro (o falta en uno), reportarlo — el `.qmd` es la fuente - y el `.tex` debe portarse a mano. - -4. **Reporte**: matriz claim x superficie con OK/FALTA/DIVERGE. Si todo OK, - una sola línea verde. Si algo diverge, bloque con ubicaciones exactas - (archivo:línea) y valor esperado; NO auto-corregir números sin confirmar - con el usuario cuál es la fuente correcta. - -## Notas - -- Los valores canónicos de arriba son los del claim pool93 activo - (2026-07-04). Si el registro activo cambió, actualizar esta tabla junto con - el registro y `CLAUDE.md` en el mismo commit. -- Este gate NO valida hashes de artefactos (eso es `/crpto-validate-champion`); - valida que la prosa no divergió de la gobernanza. diff --git a/.claude/skills/crpto-mrm-card/SKILL.md b/.claude/skills/crpto-mrm-card/SKILL.md deleted file mode 100644 index d200b17..0000000 --- a/.claude/skills/crpto-mrm-card/SKILL.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -name: crpto-mrm-card -description: Actualiza o regenera la model card (skops) del champion en reports/mrm/. Lee del champion congelado. ---- - -# /crpto-mrm-card - -Genera la model card del champion CRPTO para gobernanza MRM (Model Risk Management). Sigue lineamientos SR 11-7. - -## Pasos - -1. **Cargar champion**: - ```python - from catboost import CatBoostClassifier - from pathlib import Path - import json - - model = CatBoostClassifier() - model.load_model("models/pd_canonical.cbm") - promo = json.loads(Path("models/final_project_promotion.json").read_text()) - ``` - -2. **Generar model card via `skops`**: - ```python - from skops import card - model_card = card.Card(model, metadata=card.metadata_from_config(Path("configs/crpto_pd_model.yaml"))) - model_card.add(model_description="PD champion CatBoost — Lending Club 2007-2020Q3") - model_card.add_metrics(**promo["metrics"]) - ``` - -3. **Secciones requeridas (SR 11-7)**: - - Model description (purpose, type, version) - - Intended use (in-scope, out-of-scope) - - Training data (source, time window, feature list, target) - - Performance metrics (AUC, ECE, Brier, KS por cohorte) - - Conformal coverage (90%/95% global + Mondrian por grade) - - Fairness (DPD, EO, DIR por grupo) - - Limitations and recommendations - - Trazabilidad (run tag, DVC commit, artefactos) - -4. **Guardar**: - ```python - output = Path("reports/mrm/model_card_crpto_champion.md") - model_card.save(output) - ``` - -5. **Validar contra EXTRACTION_MANIFEST**: asegurarse de que los hashes citados en la card coinciden con los del manifiesto. - -## Argumentos - -- Sin argumentos: regenera la card del champion. -- `--challenger `: genera card de un modelo challenger para comparación. - -## Notas - -- La card sustituye `reports/mrm/model_card_crpto_champion.md` existente si lo hay. -- NO modifica el modelo, solo lee. -- Pre-check antes de empezar: `uv run python -c "import skops"`. Si falla, `uv add skops`. diff --git a/.claude/skills/crpto-paper-export/SKILL.md b/.claude/skills/crpto-paper-export/SKILL.md deleted file mode 100644 index 803a110..0000000 --- a/.claude/skills/crpto-paper-export/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: crpto-paper-export -description: Pipeline completo de salida journal-ready - tablas, figuras, libro Quarto, journal package. ---- - -# /crpto-paper-export - -Genera todos los artefactos publicables del paper en orden correcto. NO toca el champion congelado. - -> ⚠️ **Alcance pool93**: las tablas A35–A39 y los JSONs de gobernanza pool93 -> son outputs congelados de experimentos FUERA del DAG de DVC — -> `export_crpto_tables.py` NO los regenera (lee el promotion JSON del -> rebaseline congelado y solo regenera las tablas de esa cadena). Nunca -> sobreescribir A35–A39 con este pipeline. Tras cualquier export, correr -> `just validate-champion` para probar que los artefactos protegidos siguen -> bit-idénticos. - -## Pasos - -1. **Validar champion intacto**: - ```powershell - /crpto-validate-champion - ``` - Si falla, abortar y reportar. - -2. **Tablas** (subset regenerable del DAG; hoy hay 43 CSVs + 43 TEX en `reports/crpto/tables/`, de los cuales A35–A39 son congelados pool93): - ```powershell - uv run python scripts/export_crpto_tables.py - ``` - -3. **Figuras** (hoy hay 21 PNGs + 21 PDFs en `reports/crpto/figures/`): - ```powershell - uv run python scripts/generate_crpto_figures.py --paper crpto - ``` - -4. **Evidencia y journal package**: - ```powershell - uv run python scripts/analyze_crpto_evidence.py - uv run python scripts/build_crpto_journal_package.py - ``` - -5. **Render del libro**: - ```powershell - uv run -- quarto render book --to html - ``` - -6. **Estatus final**: leer `models/crpto_journal_package_status.json` y reportar. - -## Argumentos - -- Sin argumentos: pipeline completo. -- `--quick`: solo tablas y figuras (sin journal package ni render). -- `--pdf`: añade `quarto render book --to pdf` al final. - -## Resumen al usuario - -- Tablas generadas: N CSVs (lista los nuevos o cambiados). -- Figuras generadas: N PNGs. -- Libro renderizado: ✅/❌. -- Journal package status: estado del JSON final. -- `just validate-champion` post-export: ✅/❌ (obligatorio; prueba que A35–A39 y demás artefactos protegidos no cambiaron). diff --git a/.claude/skills/crpto-render/SKILL.md b/.claude/skills/crpto-render/SKILL.md deleted file mode 100644 index 8290765..0000000 --- a/.claude/skills/crpto-render/SKILL.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: crpto-render -description: Renderiza el libro Quarto a HTML, abre el resultado y opcionalmente captura screenshots para QA visual. ---- - -# /crpto-render - -Renderiza el libro CRPTO y valida el resultado. - -## Pasos - -1. **Render**: - ```powershell - uv run -- quarto render book --to html - ``` - Captura tiempo de render y warnings. - -2. **Verificación de salida**: confirmar que `book/_book/index.html` existe y que se generaron los 24 capítulos. - -3. **QA visual** (si el usuario tiene Chrome MCP o Playwright MCP): - - Abre `book/_book/index.html`. - - Screenshot de: - - Portada (`index.html`). - - Tabla con `df-print: paged` (cualquier capítulo de resultados, e.g. `04-resultados.html`). - - Callout custom (`.mini-abstract`, `.equation-card`). - - Grid de chapter-cards. - - Lightbox al hacer click en una figura. - - Toggle dark mode (si activado). - - Reporta cualquier anomalía visual. - -4. **QA de contenido**: - - `book/_book/search.json` no vacío (búsqueda funcional). - - No hay 404s en links internos: `find book/_book -name "*.html" | xargs grep -l '404\\|not found'`. - - Bibliografía renderizada en `references.html`. - -5. **Resumen al usuario**: páginas generadas, tiempo total, warnings, archivos cambiados. - -## Argumentos - -- Sin argumentos: render HTML completo. -- `pdf`: ejecuta `uv run -- quarto render book --to pdf` también. -- `preview`: arranca `uv run -- quarto preview book` en background y reporta la URL local. - -## Notas - -- NO usa `QUARTO_PYTHON=.venv/bin/python` (es path Linux y rompe en Windows). `uv run --` resuelve el Python del venv cross-platform. -- Si Quarto no está en PATH: reportar al usuario y abortar. No intentar instalar Quarto desde aquí. -- Si hay errores de chunk Python, correr `just book-clean` (purga `book/_book`, `book/_freeze`, `book/.quarto`) y re-renderizar. diff --git a/.claude/skills/crpto-smoke/SKILL.md b/.claude/skills/crpto-smoke/SKILL.md deleted file mode 100644 index 6427d28..0000000 --- a/.claude/skills/crpto-smoke/SKILL.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: crpto-smoke -description: Smoke tests del proyecto - pytest crítico + dbt parse + dbt test. Rápido (~30s). ---- - -# /crpto-smoke - -Verificación rápida de que el proyecto está sano. Ideal antes de commit o al inicio de sesión. - -## Pasos - -1. **Lint**: - ```powershell - uv run ruff check . - uv run ruff format --check . - ``` - -2. **Tests críticos**: - ```powershell - uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -q - ``` - -3. **DBT sano**: - ```powershell - uv run dbt parse --project-dir dbt_project --profiles-dir dbt_project - uv run dbt test --project-dir dbt_project --profiles-dir dbt_project - ``` - -4. **DVC sin drift**: - ```powershell - uv run dvc status - ``` - Si reporta `changed`, listar los stages afectados pero NO ejecutar `dvc repro`. - -## Resumen al usuario - -- Lint: ✅/❌ (errores si los hay). -- Tests críticos: N pass, M fail. -- DBT: parse + tests OK / fallos. -- DVC drift: stages con cambios pendientes (informativo). - -Si todo verde, decirlo en una sola línea. Si algo falla, mostrar el error específico (no el output completo). diff --git a/.claude/skills/crpto-stage/SKILL.md b/.claude/skills/crpto-stage/SKILL.md deleted file mode 100644 index c46b91d..0000000 --- a/.claude/skills/crpto-stage/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: crpto-stage -description: Ejecuta un stage DVC aislado con dry-run primero. Bloquea stages que tocan el champion congelado. ---- - -# /crpto-stage - -Ejecuta un stage DVC específico con seguridad. - -## Argumentos - -- Nombre del stage (ej. `crpto.paper.export_tables`, `crpto.paper.figures`). - -## Pasos - -1. **Validar stage permitido**. Stages **bloqueados** sin permiso explícito del usuario: - - `crpto.pd.champion` - - `crpto.conformal.intervals` - - `crpto.conformal.validation` - - `crpto.portfolio.optimization` - - `crpto.portfolio.bound_exact_eval` - - `crpto.data.dataset` (lento, regenera todo downstream) - - `crpto.data.features` (regenera train_fe/test_fe/calibration_fe) - - Si el usuario pide alguno de estos, pedir confirmación explícita y avisar de impacto en el champion. - -2. **Dry-run**: - ```powershell - uv run dvc status - uv run dvc repro --dry - ``` - Reporta deps que cambiaron y outputs que se regenerarán. - -3. **Ejecutar** (tras confirmación): - ```powershell - uv run dvc repro - ``` - -4. **Validación post-run**: - - `dvc status ` debe quedar sin cambios. - - Si el stage genera artefactos listados en `EXTRACTION_MANIFEST.json`, ejecutar `/crpto-validate-champion` para confirmar que los hashes coinciden. - -5. **Resumen**: tiempo, outputs generados, líneas de log relevantes. - -## Notas - -- Trabaja siempre desde el root del repo. -- Costo: `crpto.data.dataset` procesa 1.7 GB y los stages PD/search re-puntúan ~514k filas — de minutos a horas. Reportar la estimación antes de ejecutar. -- Si el stage falla, no intentar arreglar automáticamente — reportar al usuario. diff --git a/.claude/skills/crpto-submission-freeze/SKILL.md b/.claude/skills/crpto-submission-freeze/SKILL.md deleted file mode 100644 index 2d6df82..0000000 --- a/.claude/skills/crpto-submission-freeze/SKILL.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -name: crpto-submission-freeze -description: Checklist ejecutable del freeze de submission IJDS - gates de calidad, sincronía de claims, PDFs oficiales, QA de anonimato y límite de páginas. Correr antes de tagear o subir a ScholarOne. ---- - -# /crpto-submission-freeze - -Implementa el checklist de freeze (ventana Ago 9-10 del roadmap -`paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md`). Reporte final -GO/NO-GO por sección; cualquier NO-GO bloquea el freeze. - -## Pasos - -1. **Gates de calidad**: - ```powershell - just lint - just smoke - just validate-champion - ``` - -2. **Sincronía de claims**: correr `/crpto-claim-sync`. Debe salir verde. - -3. **PDFs frescos**: - ```powershell - just paper-submission-pdf - ``` - y el PDF oficial (los PDFs están gitignored — se regeneran localmente): - ```powershell - cd paper/submission - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - Verificar en el log `Output written on CRPTO_ijds_submission.pdf`; el - "up-to-date" sin runs de pdflatex NO cuenta como rebuild. - - Si `latexmk` falla en PowerShell por el wrapper de TinyTeX - (`runscript.tlu` con valor `nil`), usar el fallback probado: - - ```powershell - cd paper/submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - - El fallback cuenta como rebuild oficial si produce - `CRPTO_ijds_submission.pdf` desde el `.tex` sincronizado y el log muestra - `Output written`. Reparación opcional del wrapper TinyTeX: - `tlmgr update --self --all`. - -4. **Límite de páginas**: el body debe quedar en <= 25 páginas excluyendo - referencias (política IJDS). Verificar el total y en qué página empieza la - bibliografía: - ```powershell - uv run --with pypdf python -c "from pypdf import PdfReader; r = PdfReader('paper/submission/CRPTO_ijds_submission.pdf'); print(len(r.pages))" - ``` - -5. **QA de anonimato** (double-anonymous, dblanonrev): buscar en - `CRPTO_ijds_submission.tex`, `CRPTO_ijds.qmd`, `supplement_ijds.qmd` y en el - texto del PDF. Cero ocurrencias permitidas de: - - nombres/correos del autor (`Vergara`, `cavr94`) - - remotes propios (`EigenCharlie`, `github.com/EigenCharlie`, `dagshub.com/`) - - paths locales (`C:\Users`, `C:/Users`, `/home/`) - - `\AUTHOR{...}` no vacío o acknowledgements en el body - -6. **Checklist de venue**: repasar `paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` - punto por punto y reportar los que queden pendientes (cover letter, - disclosure form, title page no anónima van por separado, NO dentro del - packet anónimo). - -7. **Reporte GO/NO-GO**: tabla con las 6 secciones y su estado. Si todo GO, - sugerir el tag de release y recordar que el commit final debe pasar los - hooks (sin `--no-verify`). - -## Notas - -- Este skill NO ejecuta stages DVC ni toca artefactos congelados. -- Si `latexmk` no está en PATH: TinyTeX vive en - `%APPDATA%\TinyTeX\bin\windows\`. diff --git a/.claude/skills/crpto-validate-champion/SKILL.md b/.claude/skills/crpto-validate-champion/SKILL.md deleted file mode 100644 index cbbfed2..0000000 --- a/.claude/skills/crpto-validate-champion/SKILL.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -name: crpto-validate-champion -description: Verifica que los artefactos del champion no han cambiado vs EXTRACTION_MANIFEST.json (hashes SHA256). ---- - -# /crpto-validate-champion - -Verifica que el champion congelado sigue intacto. Sin efectos secundarios — pure read-only. - -Vía rápida: `just validate-champion` corre `tests/test_manifest_regression.py`, -que ya barre todos los `critical_hashes` del manifiesto. Los pasos siguientes -son la versión manual/explicada. - -## Pasos - -1. **Cargar manifiesto**: - ```python - import json, hashlib - from pathlib import Path - manifest = json.loads(Path("EXTRACTION_MANIFEST.json").read_text()) - ``` - -2. **Listar artefactos críticos** del manifiesto (sección `critical_hashes`). - Archivos clave: - - `models/pd_canonical.cbm` - - `models/pd_canonical_calibrator.pkl` - - `models/final_project_promotion.json` - - `models/conformal_policy_status.json` - - `data/processed/conformal_intervals_mondrian.parquet` - - `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_bound_eval.parquet` - - `reports/crpto/tables/crpto_tableA35..A39_pool93_*.csv/.tex` (evidencia pool93) - - `models/experiments/champion_reopen/...__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` - - `models/experiments/champion_reopen/...__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` - -3. **Computar hashes SHA256** de cada archivo y comparar contra el manifiesto: - ```python - def sha256(p: Path) -> str: - h = hashlib.sha256() - with p.open("rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - h.update(chunk) - return h.hexdigest() - ``` - -4. **Reportar**: - - ✅ Artefactos con hash coincidente. - - ❌ Artefactos con drift (path + esperado + actual). - - ⚠️ Artefactos faltantes (en manifiesto pero no en disco). - -5. **Verificar métricas paper (esquema dual-tag)**: - - Cadena upstream congelada: `models/final_project_promotion.json` debe - tener run tag `ijds-rebaseline-2026-06-07` y retorno `$170,464.54`. - - Body claim pool93: `pool93_ijds_claim_governance.json` debe tener run tag - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` y - `declared_return_floor = 170464.54`; el body point en - `pool93_ijds_consolidated_governance.json` (`selected_candidates.paper_body`) - debe reportar retorno `$184,832.48`, `V=0.035350`, `Γ_CP=0.162616`, - Markov cap `0.345084`, alpha grid `8/8`. - - Sincronía completa con el paper: `uv run pytest tests/test_pool93_body_claim_sync.py -q`. - -## Salida - -- Si todo OK: una línea verde "champion intacto, N artefactos verificados". -- Si drift: bloque con cada divergencia, sin sugerir auto-fix. diff --git a/.codex/skills/crpto/SKILL.md b/.codex/skills/crpto/SKILL.md index c03cfe4..ac6762e 100644 --- a/.codex/skills/crpto/SKILL.md +++ b/.codex/skills/crpto/SKILL.md @@ -1,125 +1,169 @@ # CRPTO Skill -Use this skill inside `C:\Users\carlos\Documents\Paper_CRPTO`. CRPTO is a +Use this skill in `C:\Users\carlos\Documents\Paper_CRPTO`. CRPTO is one single-author IJDS paper and reproducibility bundle, not a production service. -Prefer simple code, frozen evidence, and one coherent manuscript claim. - -## Read First - -1. `docs/ACADEMIC_CONTEXT.md` -2. `docs/SCOPE_AND_GOVERNANCE.md` -3. `docs/research/active_claims_2026-07-04.md` -4. `CONTRIBUTING.md` -5. `EXTRACTION_MANIFEST.md` -6. `configs/crpto_publication_targets.yaml` - -Use Windows PowerShell and `uv run`. Do not introduce Unix-only workflow -assumptions. - -## Active IJDS Policy - -- Run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` -- Exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`. -- Decision score: `q=(p+u)/2`. -- Risk tolerance: `tau=0.17`. -- Objective: point-PD expected net return; conformal `q` is the risk guardrail. -- Selector: nine round-number policies on November 2017; five satisfy full - budget, effective-PD, and deterministic `B_u<=0.28` screens. Outcomes are - stored separately from the 12-column selector frame. -- Audit: an outcome-free December replay selects the same policy; opening - outcomes afterward gives weighted default `0.145650` and miscoverage - `0.124925`, so stability is not reported as selected-set validity. -- Full OOT: 276,869 candidates, 308 funded, `$179,327.59` realized return, - `0.039375` weighted default, `0.036875` weighted miscoverage. -- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`, endpoint `0.258051`. -- Markov sensitivity: threshold `0.574279` with probability bound `0.316228` - under weighted funded-set validity. -- Matched point-PD: `$196,369.14`, `0.118400` weighted default, endpoint - `0.921317`, threshold `1.237545`. - -The exact paper-facing source is: - -`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` - -A35--A40 provide exact-alpha, selector, temporal, grade, bootstrap, and matched -comparison evidence. The final selector does not read OOT outcomes, but prior -project development inspected the static OOT corpus. Say "retrospective -lockbox replay," not "preregistered," "prospective," or "untouched holdout." - -Do not revive these as active claims: approximate alpha-0.01 scaling, `8/8`, -the 50,010-policy frontier, `0.345084`, capped/tail-selected policies, or OOT -outcome-selected hyperparameters. - -## Frozen Provenance - -Never overwrite the manifest-protected upstream or historical pool93 files: - -- `models/pd_canonical.cbm` -- `models/pd_canonical_calibrator.pkl` -- `models/final_project_promotion.json` -- `models/conformal_policy_status.json` -- `data/processed/conformal_intervals_mondrian.parquet` -- `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- historical `crpto_tableA35..A40_pool93_*` files -- historical pool93 governance JSON files -- `EXTRACTION_MANIFEST.json` - -New experiments must use a distinct run tag and write under -`data/processed/experiments/champion_reopen/` and -`models/experiments/champion_reopen/`. Never replace frozen paths. - -Protected DVC stages are `crpto.pd.champion`, -`crpto.conformal.intervals`, `crpto.conformal.validation`, -`crpto.portfolio.optimization`, and `crpto.portfolio.bound_exact_eval`. -Experiments may read their outputs. Run a protected stage only with explicit -permission and a drift report. - -## Method Boundary - -The submitted method has one linear policy. Capped, tail, OCE/CVaR, SPO+, -multi-distribution, online, causal, and external-data variants are comparators -or diagnostics, not additional CRPTO methods. - -Keep these distinctions explicit: - -- exact conformal quantile replay versus approximate width scaling; -- point PD in the economic objective versus conformal `q` in the constraint; -- deterministic `weighted outcome <= B_u + V` versus the - assumption-conditional Markov statement; -- calibration-only final ranking versus historical OOT-aware development; -- full-OOT averages versus temporal heterogeneity; -- retrospective contrasts versus causal or universal dominance. - -## Paper Workflow - -Safe paper work may regenerate active A35--A40, figures, Quarto outputs, and the -official IJDS PDF. Keep body, supplement, submission TeX, governance JSON, and -claim-sync tests numerically aligned. - -Standard closeout: + +## Active Contract + +Read first: + +1. `docs/research/active_claims_2026-07-14.md` +2. `docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md` +3. `docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md` +4. `docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md` +5. `docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md` +6. `configs/ijds_active_evidence_sources.yaml` +7. `configs/ijds_claim_ledger.yaml` +8. `docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md` +9. `docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md` +10. `docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md` +11. `docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md` +12. `docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md` +13. `docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md` +14. `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` +15. `docs/ACADEMIC_CONTEXT.md` +16. `docs/SCOPE_AND_GOVERNANCE.md` +17. `CONTRIBUTING.md` +18. `EXTRACTION_MANIFEST.md` + +Active evidence: + +- outcome-free freeze: `ijds-binary-geometry-frontier-v4-2026-07-12-v1`; +- verified evaluation: `ijds-binary-geometry-frontier-v4-2026-07-15-v5`; +- complete residual specification: eight consecutive six-month windows; +- primary OOT: 376,890 candidates in fifteen monthly USD 1 million menus; +- coverage learners: CatBoost/Platt primary plus numeric logistic, monotonic + CatBoost, platform-signal WOE/IV, and pricing-excluded WOE/IV controls; +- portfolio learner: primary CatBoost only; no OOT learner is selected; +- score path: `gamma={0,.25,.50,.75,1}` with endpoint contrast gamma 1 minus + gamma 0; +- objective-matched primary and normalized-score secondary rulers at three + interior coordinates; there is no selector; +- the nine V4 fixed-cap policies are supporting exact-frontier diagnostics; +- C2 comparator: contemporaneous frozen funded point-score cap; +- exact point-cap frontier: 3,067 HiGHS basis/support endpoints; +- unresolved outcomes: retained with sharp common-outcome bounds. +- evaluation-endpoint availability: all lags 0, 3, 6, 8, and 12 months are + reported without selection; the 6-month slice reconciles exactly to the + active evaluation. +- portfolio structure: all 36 budget--purpose-cap--LGD scenarios are reported + without selection; the baseline reconciles exactly to the active evaluation. +- endpoint resolution is partitioned exhaustively into 307,842 fully paid by + cutoff, 56,972 charged off by cutoff, 11,551 nonterminal, 47 terminal after + cutoff, and 478 terminal with missing availability date. +- three missingness encodings and a second retrospective origin are complete, + bounded recurrences; neither selects a representation or model. +- four declared fit-label scenarios are complete; they stress 215 + unavailable fitting labels but are not sharp bounds over all assignments. +- USD 25 floor-with-cash rounding is complete for all 1,440 portfolios and 96 + tracks; it is not an optimized integer policy. + +Headline evidence: + +- under the declared six-month endpoint contract, every CatBoost five-group + OOT upper bound is below 0.90; maximum `0.882597`; +- every logistic-control upper bound is below 0.90; maximum `0.896222`; +- monotonic CatBoost, platform WOE, and pricing-excluded WOE maxima are + `0.886489`, `0.894908`, and `0.897726`; all five fail in all eight windows; +- all `2,925,493` raw rows are audited; the `640,543` active rows exhaust the + declared 36-month population rather than forming a convenience sample; +- all 45 OptBinning problems are optimal; WOE/IV, monotonicity, calibration, + and PSI remain robustness controls rather than central novelty; +- CatBoost stratum 2 crosses prevalence alpha from W7 to W8 and its residual + quantile changes from `0.888435` to `0.111801`; +- C2 match residual is at most `8.33e-17` and reconciles weak plug-in dominance; +- all 216 broad-stress envelopes cross zero; +- default crosses zero in all 72 development-support cells; +- all 27 W8 development-support envelopes cross zero; +- the objective-matched endpoint contrast crosses zero at `.25`, is adverse at + `.50`, and is mostly unidentified for payoff/default at `.75`; +- normalized-score `.25` and `.50` are adverse; `.75` has adverse default and + miscoverage while payoff is adverse in seven windows and crosses in one; +- the six ruler-coordinate tracks are not 48 independent replications; +- every structural scenario retains adverse default and miscoverage cells; + zero scenarios are uniformly favorable or uniformly adverse; +- all 32 overall fit-completion cells remain below nominal, with scenario + maxima from `0.882594` to `0.884669`; the W7--W8 crossing disappears in the + all-default scenario and therefore is not scenario-invariant; +- USD 25 floor rounding changes 2,985 positive exposures; the largest rate + perturbation is `0.001284` percentage points; +- for every binary contrast, identification width is the sum of the unresolved + loan-wise attainable contribution ranges. It reduces to a count times one + coefficient only when those contribution ranges are identical. + +The archive and earlier results were inspected. Each retained evaluation is +protocol-locked before its corresponding outcome join, but the study is not a +preregistration, untouched holdout, prospective confirmation, or model contest. +The superiority paper is NO-GO. The IJDS identification-audit narrative is GO. + +## Claim Boundaries + +Preserve these distinctions: + +- candidate membership versus snapshot outcome resolution; +- clipped binary residual interval versus latent-PD confidence limit; +- continuous interval versus its intersection with `{0,1}`; +- constant-score phase proposition versus varying-score empirical strata; +- overlapping residual windows versus independent replications; +- Platt score and plug-in objective versus true conditional probability; +- candidate coverage versus funded-set validity; +- objective-matched opportunity cost versus normalized-score relaxation; +- finite three-coordinate diagnostics versus a continuous joint frontier; +- standardized payoff versus cash-flow return, IRR, NPV, or welfare; +- sharp identification bounds versus sampling confidence intervals; +- C2 plug-in dominance versus realized-outcome dominance; +- exact declared comparator support versus universal baseline invariance; +- tagged retrospective audit versus preregistration or confirmation. +- conformal-fit label timing versus evaluation-endpoint availability; these + are separate one-factor sensitivities and were not crossed factorially. +- complete structural conditionality versus a selected scenario, universal + adversity, or deployment guidance. +- four declared fit-label scenarios versus a sharp nonlinear region + over all `2^215` assignments. +- deterministic floor-with-cash granularity versus reoptimized integer lending. + +Do not claim a learner, gamma, ruler, coordinate, or policy winner, universal +direction, selected-set validity, Markov/tail certificate, causal effect, live +deployment result, missingness mechanism, or independent temporal replication. + +## Evidence Workflow ```powershell -uv run python scripts/build_ijds_calibration_selected_evidence.py -uv run pytest tests/test_ijds_active_claim_sync.py -q +uv run python scripts/build_ijds_binary_geometry_frontier_v4_evidence.py +just publication-integrity just lint just type-check -just type-advisory-full -just smoke +just type-check-fast +just test just validate-champion -just paper-submission -just paper-submission-official -uv run dvc status --no-updates +just ijds-active-check +just submission-build +just submission-check +just ijds-dvc-status +``` + +The active builder verifies the V4, two-ruler, raw-data, credit-control, +endpoint, structural, fit-label completion, and allocation-granularity +manifests/freezes and every artifact descriptor. The active capsule contains +31 DVC pointers. It emits only +`crpto_ijds_v4_*` tables/figures and +`ijds_binary_geometry_frontier_v4_evidence.json`. Consecutive builds must be +byte-identical. The canonical body is `paper/CRPTO_ijds.qmd`; generate official +TeX with `scripts/build_ijds_submission_tex.py` and never edit it by hand. + +Manual LaTeX fallback: + +```text +pdflatex -> bibtex -> pdflatex -> pdflatex ``` -Run `just drift-gate` after changes to conformal or PD semantics. Do not bypass -hooks, commit secrets, or alter `EXTRACTION_MANIFEST.json`. +## Protected History -## Writing +Never overwrite `EXTRACTION_MANIFEST.json`, canonical PD/calibrator, or other +artifacts protected by that manifest. Protected DVC +stages are `crpto.pd.champion`, `crpto.conformal.intervals`, +`crpto.conformal.validation`, `crpto.portfolio.optimization`, and +`crpto.portfolio.bound_exact_eval`. -- Paper and code identifiers are English; project notes may be Spanish. -- Lead with data, method, decision, and managerial implication. -- Report the price of robustness and temporal failures as prominently as wins. -- Treat reproducibility as evidence quality, not as the sole novelty. -- Keep the main IJDS body within 25 pages; move proofs and diagnostics to the - separate supplement. +Earlier studies are preserved in Git history and `D:\crpto_legacy`. The active +paper explains the final method and evidence, not discarded-version chronology. diff --git a/.env.example b/.env.example index 6260c9a..40752e1 100644 --- a/.env.example +++ b/.env.example @@ -42,12 +42,24 @@ GITHUB_PAT= # ─── CRPTO runtime paths ─────────────────────────────────────────────── # QUARTO_PYTHON is intentionally NOT set here; `uv run -- quarto` resolves it. -CRPTO_DATA_DIR=data +# dbt profiles append artifact filenames directly to this processed-data root. +CRPTO_DATA_DIR=data/processed CRPTO_MODELS_DIR=models CRPTO_DUCKDB_PATH=data/processed/crpto.duckdb +# Legacy dbt inputs; set explicitly only when reproducing the historical marts. CRPTO_PORTFOLIO_DIR=data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734 CRPTO_CONFORMAL_WINNER_DIR=data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1 +# Optional deterministic solver overrides. Defaults shown here are documentation; +# uncomment only for an isolated experiment and record the values in its governance. +# EXACT_THREADS=4 +# HIGHS_RESET_GLOBAL_SCHEDULER=1 +# HIGHS_PRESOLVE=on +# HIGHS_PARALLEL=choose +# HIGHS_SOLVER=choose +# HIGHS_SIMPLEX_STRATEGY=0 +# HIGHS_NATIVE_FALLBACK_SCIPY=1 + # ─── Optuna storage ──────────────────────────────────────────────────── # Examples: journal:/tmp/study.log | sqlite:///optuna.db | empty (default journal) OPTUNA_STORAGE= diff --git a/.github/workflows/book-publish.yml b/.github/workflows/book-publish.yml deleted file mode 100644 index 2bd88df..0000000 --- a/.github/workflows/book-publish.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: book-publish - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - build-and-publish: - runs-on: ubuntu-latest - timeout-minutes: 30 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Set up Python - run: uv python install 3.11 - - - name: Sync deps - run: uv sync --extra dev --frozen - - - name: Install Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: "1.9.35" - - - name: Write build metadata - run: uv run python scripts/write_book_build_info.py - - - name: Render book (full) - run: uv run -- quarto render book --to html --no-execute - - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v5 - with: - path: book/_book - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v5 diff --git a/.github/workflows/tests-full.yml b/.github/workflows/tests-full.yml index 5b3f528..fcff7fb 100644 --- a/.github/workflows/tests-full.yml +++ b/.github/workflows/tests-full.yml @@ -1,13 +1,12 @@ name: tests-full -# Manually-triggered full test suite. Pulls champion artefacts from the -# DagsHub DVC remote and runs the regression hash checks plus the slow +# Manually-triggered full test suite. Pulls protected extraction artifacts from +# the DagsHub DVC remote and runs their hash checks plus the active IJDS suite. # tests. NOT wired to push events — it would be too expensive (and add # zero value to a single-author academic project) to run on every commit. # # Trigger it from the GitHub UI (Actions tab → tests-full → Run workflow) -# whenever you want to verify the champion is still bit-exact against the -# DVC remote (e.g. before a paper submission, after a major refactor). +# Use it before a paper submission or after a major scientific refactor. # # Required repository secrets: # - DAGSHUB_USER_TOKEN @@ -33,7 +32,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.DAGSHUB_TOKEN }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DAGSHUB_TOKEN }} AWS_ENDPOINT_URL: https://dagshub.com/EigenCharlie94/Paper_CRPTO.s3 - MLFLOW_TRACKING_URI: https://dagshub.com/EigenCharlie94/Paper_CRPTO.mlflow CRPTO_REQUIRE_DVC_ARTIFACTS: "1" steps: - uses: actions/checkout@v5 @@ -47,33 +45,25 @@ jobs: run: uv python install 3.11 - name: Sync deps - run: uv sync --extra dev --extra search --frozen + run: uv sync --extra dev --frozen - - name: DVC pull (champion artefacts) + - name: DVC pull (active and protected artifacts) run: uv run dvc pull --no-run-cache - - name: Manifest regression (hash check vs frozen champion) + - name: Protected extraction hash regression run: uv run pytest tests/test_manifest_regression.py -v - - name: Champion drift gate - env: - CRPTO_RUN_CHAMPION_DRIFT: "1" - run: | - mkdir -p reports/ci - uv run pytest tests/test_models/test_conformal_mapie_drift.py -q -s \ - | tee reports/ci/champion-drift-gate.txt + - name: Strict protected-artifact gate + run: just validate-champion-strict + + - name: Active IJDS contract + run: just ijds-active-check + + - name: Active implementation and evidence drift gate + run: just drift-gate - name: Full author tests - run: | - uv run pytest \ - tests/test_models/ \ - tests/test_optimization/ \ - tests/test_utils/ \ - tests/test_features/ \ - tests/test_crpto_final_sync.py \ - tests/test_quarto_book_guardrails.py \ - tests/test_publication_integrity.py \ - -q + run: uv run pytest -q - name: Upload coverage if generated if: always() @@ -83,6 +73,5 @@ jobs: path: | coverage.xml .pytest_cache/ - reports/ci/champion-drift-gate.txt retention-days: 7 if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 109697f..e3eee62 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,13 @@ data/processed/optuna/ !data/processed/*.dvc data/processed/**/* !data/processed/**/*.dvc +!data/processed/experiments/ +!data/processed/experiments/ijds_audit/ +!data/processed/experiments/ijds_audit/*.dvc +!data/processed/experiments/champion_reopen/ +!data/processed/experiments/champion_reopen/champion-reopen-2026-07-10__maturity-safe-locked-bounded-h1h2-v2.dvc +!data/processed/experiments/ijds_prefreeze/ +!data/processed/experiments/ijds_prefreeze/*.dvc # DVC cache, locks, tmp (only `.dvc/config` and `dvc.yaml`/`dvc.lock` belong in git) .dvc/cache/ @@ -130,6 +137,7 @@ mlartifacts/ # Scratch runtime checkpoints are observability, not canonical deliverables. models/*_runtime_checkpoints/ +tmp/ # Optuna local optuna.db @@ -154,6 +162,7 @@ models/experiments/** !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json +!models/experiments/**/*.dvc configs/experiments/ !configs/experiments/ configs/experiments/* @@ -161,3 +170,9 @@ configs/experiments/* !configs/experiments/tabpfn_*.yaml !configs/experiments/champion_reopen*.yaml !configs/experiments/champion_reopen*.txt +!configs/experiments/ijds_maturity_safe_locked_*.yaml +!configs/experiments/ijds_fixed_taxonomy_*.yaml +!configs/experiments/ijds_rolling_origin_*.yaml +!configs/experiments/ijds_decision_active_*.yaml +!configs/experiments/ijds_policy_support_*.yaml +!configs/experiments/ijds_*.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 78d3195..f61a6b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ fail_fast: false repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.21 hooks: - id: ruff args: [--fix] @@ -13,35 +13,27 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: + # These path-bound compatibility files must remain byte-exact. - id: trailing-whitespace - exclude: '^(book/_freeze/|book/_book/|.*\.cbm|.*\.pkl|.*\.parquet)' + exclude: '^(.*\.cbm|.*\.pkl|.*\.parquet)' - id: end-of-file-fixer - exclude: '^(book/_freeze/|book/_book/|.*\.cbm|.*\.pkl|.*\.parquet)' + exclude: '^(.*\.cbm|.*\.pkl|.*\.parquet|models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance\.json)$' - id: check-yaml - exclude: 'dbt_project/profiles\.yml' - id: check-toml - id: check-json - id: check-added-large-files args: [--maxkb=2000] - exclude: '^(EXTRACTION_MANIFEST\.json|models/.*\.cbm|models/.*\.pkl|book/_freeze/|book/_book/|book/apa\.csl|book/assets/figures/editorial/diagrama-crpto\.png)' - id: check-merge-conflict - id: detect-private-key - id: mixed-line-ending args: [--fix=lf] - - - repo: https://github.com/kynan/nbstripout - rev: 0.9.1 - hooks: - - id: nbstripout + exclude: '^book/references\.bib$' - repo: local hooks: - # mypy is intentionally not enforced locally while the legacy codebase is - # gradually annotated. Keep strict checks scoped to the modules that opt in. - - id: dvc-status - name: dvc status (non-blocking drift report) - entry: uv run python scripts/check_dvc_status_warning.py + name: active IJDS DVC status + entry: just ijds-dvc-status language: system pass_filenames: false stages: [pre-push] @@ -54,7 +46,7 @@ repos: stages: [pre-push] - id: validate-champion - name: validate champion artefacts + name: validate protected extraction artifacts entry: just validate-champion language: system pass_filenames: false diff --git a/AGENTS.md b/AGENTS.md index 6bdc9f6..1027822 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,27 +1,63 @@ -# AGENTS.md — Contexto para agentes (Codex y otros) +# AGENTS.md - Agent context -La fuente única de contexto operativo de este repositorio es -[`CLAUDE.md`](CLAUDE.md). Este archivo existía como copia para Codex y quedó -desactualizado (presentaba el rebaseline pre-pool93 como champion activo), así -que se redujo a puntero para eliminar la duplicación. +The single operational source is [`CLAUDE.md`](CLAUDE.md). Read these files in +order before changing scientific code or paper claims: -Lectura obligatoria, en este orden: +1. [`CLAUDE.md`](CLAUDE.md) - repository rules and protected historical stages. +2. [`docs/research/active_claims_2026-07-14.md`](docs/research/active_claims_2026-07-14.md) + - the only active IJDS claim registry. +3. [`docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md`](docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md) + - complete-window V4 protocol and stop rules. +4. [`docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md`](docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md) + - endpoint-corrected V1-freeze to V3-evaluation lineage. +5. [`docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md`](docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md) + - two-ruler research question, estimands, and locked stop rules. +6. [`docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md`](docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md) + - retained outcome-free numerical lineage. +7. [`docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md`](docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md) + - endpoint-corrected finite-grid interpretation boundary. +8. [`reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json`](reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json) + - active full-archive population, maturity, schema, and funding audit. +9. [`models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/credit_risk_controls_summary.json`](models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/credit_risk_controls_summary.json) + - active five-model coverage, WOE/IV, monotonicity, calibration, and PSI evidence. +10. [`docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md`](docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md) + - retrospectively locked fit-label timing sensitivity and retention stop rule. +11. [`docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md`](docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md) + - complete nonselective evaluation-endpoint availability sensitivity. +12. [`docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md`](docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md) + - retained complete budget--purpose-cap--LGD sensitivity and numerical lineage. +13. [`models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/structural_sensitivity_summary.json`](models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/structural_sensitivity_summary.json) + - active complete-grid structural evidence. +14. [`docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md`](docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md) + - observed-only fit plus three completion rules and nonlinear interpretation boundary. +15. [`docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md`](docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md) + - USD 25 floor-with-cash diagnostic and fixed-capital estimands. +16. [`configs/ijds_active_evidence_sources.yaml`](configs/ijds_active_evidence_sources.yaml) + - active lineage identities and exact DVC-pointer authority. +17. [`configs/ijds_claim_ledger.yaml`](configs/ijds_claim_ledger.yaml) + - executable nonnumeric claim and surface contract. +18. [`reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`](reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json) + - the only active paper-facing evidence manifest. +19. [`.codex/skills/crpto/SKILL.md`](.codex/skills/crpto/SKILL.md) - concise + execution and writing guidance. -1. [`CLAUDE.md`](CLAUDE.md) — reglas de operación, champion congelado - (esquema dual-tag pool93), stages prohibidos, comandos y convenciones. -2. [`docs/research/active_claims_2026-07-04.md`](docs/research/active_claims_2026-07-04.md) - — registro vigente de claims del paper (replay exacto al 90%, selector de - calibración 3x3, política lineal 50/50 y reopen gate). -3. [`.codex/skills/crpto/SKILL.md`](.codex/skills/crpto/SKILL.md) — skill de - Codex con el detalle operativo del certificado y sus artefactos. +Minimum rules: -Reglas mínimas que ningún agente puede violar (detalle en `CLAUDE.md`): - -- El champion congelado es ley: los stages `crpto.pd.champion`, - `crpto.conformal.intervals`, `crpto.conformal.validation`, - `crpto.portfolio.optimization` y `crpto.portfolio.bound_exact_eval` no se - ejecutan sin permiso explícito. -- No modificar `EXTRACTION_MANIFEST.json` ni los artefactos que lista. -- Validar con `just validate-champion` antes de cualquier merge; refactors de - la capa conformal/PD exigen `just drift-gate` en verde. -- Windows-first, `uv run` para todo tooling Python, sin secretos en Git. +- Do not run `crpto.pd.champion`, `crpto.conformal.intervals`, + `crpto.conformal.validation`, `crpto.portfolio.optimization`, or + `crpto.portfolio.bound_exact_eval` without explicit permission. +- Do not modify `EXTRACTION_MANIFEST.json` or artifacts protected by it. +- Validate ordinary work with `just validate-champion`; `submission-check` + invokes `just validate-champion-strict` so missing protected artifacts fail. + PD/conformal refactors also require `just drift-gate`. +- Use `uv run` and Windows-first commands. Never commit secrets. +- No selected gamma, ruler, coordinate, structural scenario, policy, winner, + universal guardrail direction, causal effect, or selected-set conformal claim + is active. +- The five learner specifications are coverage controls; only the primary + CatBoost enters portfolio optimization and no OOT model winner is active. +- Historical V1--V3, structural-sensitivity V1--V5, selected-policy, + compact-v7, pool93, and A1--A40 materials are provenance, not evidence for + the active manuscript. +- Edit `paper/CRPTO_ijds.qmd`; generate the official TeX with + `scripts/build_ijds_submission_tex.py` rather than editing it directly. diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index f664d2f..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,147 +0,0 @@ -# Changelog - -All notable changes to **CRPTO** are documented here. The format follows -[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project -adheres to a single-author, paper-driven release cadence — see -`docs/ACADEMIC_CONTEXT.md`. - -## [Unreleased] - -### Changed (feature-config pickle retirement, 2026-06-13) -- Retired `data/processed/feature_config.pkl` from the live DVC DAG and - `EXTRACTION_MANIFEST.json`; the feature contract is now - `feature_config.yml` plus Pandera-validated `feature_config.parquet`. -- Re-materialized `crpto.data.features` once, re-keyed downstream - champion/conformal stage deps without re-running CatBoost, and confirmed - `just drift-gate` remains bit-exact. -- `uv run dvc push -j 2` completed after the cleanup and pushed the one new - DVC object needed by the remote. - -### Removed (docs cleanup, 2026-06-13) -- Retired 27 dated, single-shot research records under `docs/research/` - (audits, backlogs, closures, checklists, dependency/upgrade reports whose - conclusions are already applied in the paper, the code and the manifest). - Kept: foundations/ (perennial technical reference), future_work/, governance - records, the run-tag register, and the records the code reads/writes - (`crpto_p1_evidence`, `crpto_journal_package`, `crpto_bound_tightening_experiment`, - the deep-audit and regret-closure that the Quarto book cites). `docs/research/README.md` - rewritten as a minimal index. - -### Added -- `src/utils/script_helpers.py` — canonical JSON/YAML/table I/O helpers for - the publication scripts, with LF-only idempotent writers that keep - regenerated tables bit-exact against `EXTRACTION_MANIFEST.json` on - Windows (fixes a latent CRLF reproducibility bug in `just tables`). -- `scripts/archive/` — six zero-reference one-shot scripts moved out of the - active tree (`build_concentration_bound_table.py`, - `run_crpto_notebook_suite.py`, and four `search/` helpers); roles recorded - with `status: archived` in `configs/pipeline_registry/script_role_registry.yaml`. - -- `tests/test_manifest_regression.py` — hash-regression tests against - `EXTRACTION_MANIFEST.json` for the three protected champion files - (`pd_canonical.cbm`, `pd_canonical_calibrator.pkl`, - `final_project_promotion.json`) plus a sweep over frozen - model/data/table artefacts. -- `crpto/` package re-exports the public API: `PolicyMode`, - `load_pipeline_state`, `make_study`, `paper_run`, `VennAbersScoreCalibrator`, - etc. -- `book/__init__.py` so `from book._helpers import ...` resolves to a real - package rather than an implicit namespace package. -- `LICENSE-CONTENT` — CC-BY 4.0 for the book/paper text, separated from the - MIT licensed code in `LICENSE`. -- `CONTRIBUTING.md` for reviewers (MRM, journal) describing how to - regenerate the deliverables without disturbing the champion. - -### Changed -- `LICENSE` now contains only the MIT text covering code; the dual-license - note moved out to `LICENSE-CONTENT`. -- `CLAUDE.md` cross-references `docs/SCOPE_AND_GOVERNANCE.md` and - `docs/ACADEMIC_CONTEXT.md` as required reading. - -### Changed (april-lineage unification, 2026-06-10) -- `models/pd_canonical.cbm` and `models/pd_canonical_calibrator.pkl` are now - byte-copies of the April search candidate - (`models/search_pd/pd-hpo-local-2026-04-03-1325`), the exact binaries that - produced the frozen conformal intervals and the funded-set certificate - (drift harness: 0.0 across all columns). The previous canonical files were - later retrains of the same config that never fed the paper's certificate. -- `data/processed/test_predictions.parquet` rebuilt from that bundle via the - new `scripts/rebuild_test_predictions_from_frozen.py` (hard assert: - `pd_calibrated` equals the frozen intervals' `y_pred` exactly). -- Paper-facing PD metrics now come from the certificate lineage: - AUC `0.7127 -> 0.7139`, Brier `0.1546 -> 0.1544`, ECE `0.0062 -> 0.0070` - (table0, paper body/tex, book chapters). The exact certificate - (`$170,464.54`, `V=0.028875`, `Gamma_CP=0.187987`, `45/45`) is unchanged. -- `crpto_tableA5/A9/A10` re-frozen under the current locked stack (their - committed versions were generated in an unrecoverable environment); - `crpto_tableA7/A8` deliberately NOT regenerated — they remain the frozen - per-loan view of the certificate funded set (LP re-solves are degenerate). -- `EXTRACTION_MANIFEST.json` gains an `april_lineage_unification` block and - 14 refreshed hashes; environment-leak paths removed from - `models/threshold_semantics.json` and `models/mrm_report_status.json`. - -### Removed (R0 cleanup, 2026-06-12) -- Second dead-code pass, verified zero live imports/call sites across - `src/`, `scripts/`, `tests/` and configs: `src/data/build_datasets.py`, - `src/utils/mlflow_utils.py`, `src/models/conformal_registry.py`, and - `src/models/_mapie_compat.py` (created during the drift-gate work but - never adopted by any active pipeline call site). Historical provenance - references remain only where explicitly marked as such. -- `_policy_match`/`_policy_matches` consolidated from three scripts into - `src/utils/script_helpers.policy_matches` with parametrizable field tuples; - regenerated tables remained bit-exact. - -### Removed -- Dead modules with no imports anywhere in the repo: - `src/evaluation/encoding_stability.py`, `src/evaluation/monotonicity.py`, - `src/evaluation/slicing_functions.py`, `src/optimization/sda.py`, - `src/optimization/spo_integration.py`. The frozen audit artefacts they - once produced (`models/encoding_stability_status.json`, - `models/monotonicity_audit_status.json`) remain committed and are still - consumed by `scripts/generate_mrm_report.py` and the book. - -## [0.1.0] — 2026-05-11 - -First public release of the standalone Paper_CRPTO repository on GitHub -under `EigenCharlie/Paper_CRPTO`. Frozen against the paper champion: - -- run tag `paper-thesis-final-economic-2026-04-06` -- policy `bound_aware_276k_economic_champion` -- robust return `$170,464.54` -- `V(α=0.01) = 0.03645`, `Γ_CP(α=0.01) = 0.18591` -- robust region `45/45` - -### Highlights -- Quarto book with 24 chapters (manuscript + extended dossier), APA - bibliography, dark mode, lightbox. -- DVC pipeline with 13 stages, frozen champion outputs hashed in - `EXTRACTION_MANIFEST.json`. -- Refactors merged additively without touching the champion: - `PolicyMode` enum, `pipeline_state` aggregator+writer (with protected - files), MAPIE 1.x helper surface, Optuna 4 JournalStorage, MLflow 3 - tracing helpers, Pandera DataFrameModel companions. -- Dual-write helper for `feature_config.pkl` → YAML (companion file - generated, round-trip tested). -- DVC `params:` declared on the four protected stages plus - `crpto.book.render`; baseline absorbed without re-runs. -- 105 author tests green (including Hypothesis property-based tests, - pickle-compat safety net, MAPIE structural invariants on the frozen - parquet). -- GitHub Actions sized for a single-author academic project: - `lint.yml` + `book-publish.yml` only. No branch protection. GitHub - Pages serves the book at https://eigencharlie.github.io/Paper_CRPTO/. - -### Operational decisions documented -- `docs/ACADEMIC_CONTEXT.md`: single-author, static dataset, no - production deployment. Sizes CI/governance overhead accordingly. -- `docs/SCOPE_AND_GOVERNANCE.md`: explicit scope, frozen champion - contract, refactor lanes with execution preconditions. -- `docs/security/SECRETS_AND_REMOTES.md`: secret handling and DagsHub - integration. -- `docs/refactor/`: deferred-refactor plans for MAPIE class extraction, - conformal monolith split and feature-config Parquet migration. - -### Dependabot alert dismissed -- `diskcache <= 5.6.3` (CVE-2025-69872, transitive via `dvc-data`, - severity medium, no upstream fix available) — dismissed as - `no_bandwidth`. Re-evaluate when DVC publishes a fix. diff --git a/CLAUDE.md b/CLAUDE.md index 219e7db..e07d1d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,288 +1,145 @@ -# CLAUDE.md — Contexto para Claude Code en Paper_CRPTO - -## Quién soy y qué es este proyecto - -Soy Carlos Vergara, científico de datos terminando un paper académico de tesis. **CRPTO** (Conformal Robust Predict-Then-Optimize) aplica **conformal prediction** + **optimización robusta de carteras** al dominio de **credit risk** usando datos de Lending Club. Este repositorio es standalone: GitHub, DVC y MLflow apuntan a recursos propios de CRPTO. La historia de extracción está en `docs/PROJECT_HISTORY.md`; el repositorio no debe re-correr el **pipeline de búsqueda** del champion sin permiso explícito. - -**Prefiero código simple y funcional. Sin sobre-ingeniería, sin abstracciones prematuras, sin refactors gratuitos.** - -## Contexto académico (lectura obligatoria) - -Antes de cualquier cambio estructural, lee en este orden: - -1. [`docs/ACADEMIC_CONTEXT.md`](docs/ACADEMIC_CONTEXT.md) — naturaleza single-author / dataset estático / no producción y consecuencias operacionales. -2. [`docs/SCOPE_AND_GOVERNANCE.md`](docs/SCOPE_AND_GOVERNANCE.md) — qué entra en CRPTO, lista explícita de stages prohibidos en `main`, refactor lanes con precondiciones, release checklist. -3. [`CONTRIBUTING.md`](CONTRIBUTING.md) — qué se puede cambiar libremente vs. qué requiere plan de revalidación. -4. [`EXTRACTION_MANIFEST.md`](EXTRACTION_MANIFEST.md) — qué es exactamente lo "congelado" y cómo los tests de regresión lo enforzan. - -Lo crítico de `ACADEMIC_CONTEXT.md`: - -- **Single-author.** Yo soy el único que toca este repo. No hay PR reviews, no hay branch protection necesaria, no hay reviewers que aprueben. Las reglas de operación existen para disciplinar agentes, no para satisfacer un proceso corporativo. -- **Dataset estático.** Lending Club cerró originación retail en 2020. No vamos a recibir datos nuevos. Sin streaming, sin concept drift por cohortes nuevas. Si re-entrenamos, es sobre el mismo histórico. -- **No va a producción.** Output: paper + journal + libro Quarto + MRM dossier. Sin servicio live, sin SLAs, sin on-call. -- **GitHub Actions minimalista.** `book-publish.yml` (Pages) y `lint.yml` corren en push; `tests-full.yml` queda manual para hitos de journal o revalidación con DVC. `test.yml`, `dbt.yml`, `book-build.yml` se retiraron porque el pre-push hook ya valida lo equivalente en local. - -## Re-corrida del champion: matiz importante - -El "champion congelado" se refiere al **pipeline de búsqueda** que produjo las decisiones del paper: - -- ❌ **Prohibido sin permiso explícito**: `crpto.portfolio.bound_exact_eval` (es la búsqueda de 276k políticas — el resultado rank-1 es la contribución del paper). -- ❌ **Prohibido sin permiso explícito**: cualquier HPO Optuna que re-busque hiperparámetros. -- ✅ **Permitido para validación** (drift check requerido): `crpto.pd.champion`, `crpto.conformal.intervals`, `crpto.conformal.validation`, `crpto.portfolio.optimization`. Estos usan hiperparámetros/policies ya elegidos y congelados en `configs/`. Tolerancias documentadas en `ACADEMIC_CONTEXT.md`. -- ✅ **Libre re-corrida**: `crpto.paper.*` y `crpto.book.render`. - -## Champion congelado — NO RE-CORRER - -El modelo PD, calibrador, intervalos y bundle pool93 del manifest permanecen -congelados. El body IJDS usa un replay exacto y una política nueva bajo un run -tag aislado; **no regenera ni sobreescribe ningún artefacto upstream**. - -**Body claim del paper IJDS (activo):** - -| Campo | Valor | -| --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | -| Conformal | exact replay at target `alpha=0.10` (frozen used alpha `0.095`) | -| Policy | `q=(p+u)/2`, `tau=0.17`, point-PD economic objective | -| Selector | 9 policies on Nov 2017; deterministic `B_u<=0.28`; 5 eligible; same policy in outcome-free Dec replay | -| Realized return | `$179,327.59` on a `$1M` budget | -| Weighted default / miscoverage | `0.039375 / 0.036875` | -| Gamma_CP / Gamma_residual | `0.176102 / 0.088051` | -| Endpoint / Markov threshold | `0.258051 / 0.574279` | -| Matched point-PD A40 | return cost `8.678%`; default reduction `7.9025` pp; threshold reduction `66.3266` pp | -| Evidence | exact alpha A35 + split selector/audit A36 + temporal/funded-set/baseline A37--A40 | - -The exact policy-facing quantities come from -`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. -The primary claim is the simple calibration-selected guardrail, deterministic -endpoint screen, independent December audit, and exact funded-set accounting. -December miscoverage `0.124925` documents that stable selection is not -selected-set validity. Markov remains an assumption-conditional sensitivity, -not a selector or headline novelty. - -**Cadena upstream congelada (histórica; su retorno es el return floor declarado del pool93):** - -| Campo | Valor | -| --- | --- | -| Run tag | `ijds-rebaseline-2026-06-07` | -| Policy | `bound_aware_276k_economic_champion` | -| Retorno robusto | `$170,464.54` | -| V(α=0.01) | `0.028875` | -| Γ_CP(α=0.01) | `0.187987` | -| Exact pass | `True` | -| Región robusta | `45/45` | - -Artefactos históricos congelados cuyos hashes están en -`EXTRACTION_MANIFEST.json` y **no se tocan** sin permiso: - -- `models/pd_canonical.cbm` -- `models/pd_canonical_calibrator.pkl` -- `models/final_project_promotion.json` -- `models/conformal_policy_status.json` -- `data/processed/conformal_intervals_mondrian.parquet` -- `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35..A40_pool93_*.csv/.tex` (evidencia pool93) -- `models/experiments/champion_reopen/...__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` -- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` -- `EXTRACTION_MANIFEST.json` - -La sincronía del body claim activo con el paper la vigila -`tests/test_ijds_active_claim_sync.py`. `tests/test_pool93_body_claim_sync.py` -queda limitado a la integridad de procedencia histórica. - -Stages DVC que regeneran estos artefactos (`crpto.pd.champion`, `crpto.conformal.intervals`, `crpto.conformal.validation`, `crpto.portfolio.optimization`, `crpto.portfolio.bound_exact_eval`) **no se ejecutan** sin permiso. Validar con `crpto-validate-champion` antes de cualquier merge. - -## Stack técnico (resumen) - -- **Python** 3.11 con `uv` (no pip, no poetry). `uv.exe` típicamente en `C:\Users\carlos\anaconda3\Scripts\uv.exe`. -- **ML**: CatBoost 1.2.x (PD), MAPIE 1.4 (conformal), Optuna 4 (HPO), fairlearn 0.13, scikit-learn 1.8, Venn-Abers 1.5. -- **Optimización**: Pyomo 6.10 + HiGHS 1.14 (LP/MILP), OR-Tools 9.10, PyEPO 1.1 (SPO+, opcional). -- **Data**: pandas 2.3, numpy 2.4, pyarrow 23, duckdb 1.5, pandera 0.31. -- **Pipeline**: DVC 3.67 (con remote S3), dbt-duckdb 1.10, MLflow 3.12, DagsHub 0.7. -- **Docs**: Quarto 1.9+ (libro de 24 capítulos en español, HTML + PDF). -- **Tooling**: ruff 0.15, pytest 9, pre-commit 4, jupytext 1.19. - -Lista completa: `pyproject.toml`. Versiones efectivas: `uv.lock`. - -## Plataforma y paths - -- **OS**: Windows 11 Pro. Shell por defecto: PowerShell. No usar shells no Windows como flujo operativo del proyecto. -- **Venv**: `.venv/Scripts/python.exe`; todo debe pasar por `uv run` o por el Python de ese entorno. -- **Task runner**: `justfile` (cross-platform). Existió un `Makefile` que se retiró por bug Linux-only. -- **Quarto CLI**: debe estar en PATH. `quarto --version` ≥ 1.9. - -## Comandos clave +# CRPTO Operating Contract + +This repository contains one active IJDS manuscript. Historical searches, +promoted policies, thesis-book material, and earlier manuscript versions are +not active evidence. + +## Read Before Changing Science + +1. `docs/research/active_claims_2026-07-14.md` +2. `docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md` +3. `docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md` +4. `docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md` +5. `docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md` +6. `docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md` +7. `docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md` +8. `docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md` +9. `docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md` +10. `docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md` +11. `configs/ijds_active_evidence_sources.yaml` +12. `configs/ijds_claim_ledger.yaml` +13. `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` +14. `.codex/skills/crpto/SKILL.md` + +The claim registry is the editorial authority. The source registry owns exact +lineage identities, artifact descriptors, and 31 DVC pointers. The V4 evidence +JSON is the only numeric paper-facing manifest. + +## Active Scientific Object + +- One status-independent universe of 640,543 eligible 36-month loans. +- Primary OOT: 376,890 candidates, of which 364,814 are resolved and 12,076 + remain unresolved under the declared six-month availability rule. +- Five frozen coverage specifications and eight complete residual windows. +- CatBoost/Platt is the only score entering portfolio optimization; logistic, + monotonic CatBoost, and two WOE/IV scorecards are coverage controls. +- Five gamma values, two outcome-blind rulers, and three interior coordinates; + none is selected. +- Fifteen separate monthly allocations, sharp common-outcome bounds, exact + point-cap support, and 36 structural scenarios. +- All 40 six-month all-candidate coverage upper bounds are below 0.90. +- All 216 broad-support comparator envelopes cross zero. +- All 32 overall cells across four declared fit-label scenarios remain + below 0.90, but the W7--W8 geometry change is not scenario-invariant. +- USD 25 floor rounding is numerically negligible in the declared archive; it + is a diagnostic of the continuous relaxation, not an integer policy. + +The paper supports archive-specific temporal transport failure, binary +residual geometry near a prevalence threshold, and comparator-dependent +decision identification. It does not support a policy winner. + +## Forbidden Claims + +Do not claim: + +- selected model, missingness encoding, gamma, ruler, coordinate, cap, + structural scenario, comparator, or policy; +- selected-set conformal validity or latent-PD confidence intervals; +- universal favorable or adverse portfolio direction; +- causal, prospective, confirmatory, deployment, fair-lending, or Markov + conclusions; +- cash-flow return, IRR, NPV, welfare, or counterfactual funding effects. + +The five-model coverage result and the CatBoost-only missingness/second-origin +sensitivities are distinct claims. Every first-use 40/40 statement must name +the six-month outcome-availability rule. + +## Code Architecture + +Active reusable modules: + +- `src/ijds_audit/`: protocol, binary geometry, portfolio construction, + evaluation, policy support, evidence loaders, and raw-data controls. Frozen + synthetic mechanism outputs are compatibility material, not active evidence. +- `src/ijds_challengers/`: frozen/evaluated frontier lineage and archive + contracts. +- `src/data/outcome_observability.py`: endpoint reconstruction. +- `src/models/`: maturity-safe PD and binary conformal guardrail components. +- `src/optimization/`: current portfolio, policy, and solver adapters. +- `src/evaluation/`: maturity-safe and paired policy evaluation. + +Active execution is declared in +`configs/crpto_publication_targets.yaml`. Top-level scripts outside that list +may remain only because `dvc.yaml` or `EXTRACTION_MANIFEST.json` fixes their +paths. They are sealed compatibility, not active architecture. + +## Protected Operations + +Never run these without explicit permission: + +- `crpto.pd.champion` +- `crpto.conformal.intervals` +- `crpto.conformal.validation` +- `crpto.portfolio.optimization` +- `crpto.portfolio.bound_exact_eval` + +Never modify `EXTRACTION_MANIFEST.json` or protected historical artifacts. +Do not execute the sealed historical DVC graph merely to make `dvc status` +clean. + +Safe evidence work reads registered roots and writes only the active +`crpto_ijds_v4_*` outputs and evidence JSON. New empirical objects require a +new predeclared protocol, a distinct run tag, contained output paths, and +updated source registration before manuscript use. + +## Commands + +Use Windows PowerShell and `uv run`. ```powershell -# Setup -just setup # uv sync --extra dev --extra search --extra spo -just setup-base # sin pyepo/torch - -# Render del libro -just book # uv run -- quarto render book --to html -just book-pdf # no-op intencional: PDF completo diferido hasta tesis curada -just book-preview # quarto preview book (live reload) - -# Tests y validación -just smoke # pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -just test # pytest completo -just lint # ruff check + format check -just type-check # mypy src scripts (limpio: 0 errores) -just validate-champion # verifica hashes vs EXTRACTION_MANIFEST.json -just drift-gate # recomputa la cadena del certificado y exige diff bit-exacto (CRPTO_RUN_CHAMPION_DRIFT=1) -just bound-audit # re-deriva el menú de bounds A21 + búsquedas bound-aware - -# Paper outputs (re-genera artefactos pero NO toca el champion) -just tables # python scripts/export_crpto_tables.py -just figures # python scripts/generate_crpto_figures.py --paper crpto -just paper-export # tables + figures + book - -# DVC / DBT -just dvc-status # dvc status (sin re-correr) -just dvc-dag # dvc dag --md -just dbt-test # dbt parse + dbt test -``` - -## Variables de entorno (.env) - -Copiar `.env.example` a `.env` y rellenar con tokens reales. Variables clave: - -- `DAGSHUB_OWNER`, `DAGSHUB_USER`, `DAGSHUB_REPO=Paper_CRPTO`, `DAGSHUB_TOKEN` o `DAGSHUB_USER_TOKEN` — standalone CRPTO. -- `MLFLOW_TRACKING_URI`, `MLFLOW_TRACKING_USERNAME`, `MLFLOW_TRACKING_PASSWORD` — DagsHub MLflow. -- `AWS_ENDPOINT_URL`/`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` o `GDRIVE_FOLDER_ID` — DVC remote opcional. -- Variables manuales de entorno Python/Quarto — no setearlas en el flujo normal; `uv run` usa `.venv/Scripts` y resuelve Python para Quarto. - -El archivo `.env` está en `.gitignore`. Nunca commitear tokens. - -## Convenciones de código - -- **Type hints obligatorios** en `src/` y `scripts/`. Verificados con `mypy` (laxo, gradual). -- **`from __future__ import annotations`** al tope de cada módulo nuevo. -- **Docstrings** en funciones públicas (estilo Google o NumPy, consistente con módulo). -- **Logging con `loguru`** — NO `print`, NO `logging` estándar salvo en librerías que ya lo usan. -- **Configs en YAML** bajo `configs/`, validados con Pandera donde aplique. -- **Paths absolutos via `pathlib.Path`**, no strings con `/` o `\\`. -- **Pandera schemas** para contratos de DataFrames en boundaries. -- **Ruff** corre en pre-commit con reglas: `E,F,W,I,UP,B,SIM,C4,ANN,RET,RUF,PERF,FURB` (gradual). - -## Estructura del proyecto - +uv sync --extra dev +just smoke +just test +just lint +just type-check +just type-check-fast +just publication-integrity +just drift-gate +just ijds-active-check +just validate-champion +just submission-build +just submission-check ``` -. -├── book/ # Libro Quarto (24 capítulos ES, HTML+PDF) -│ ├── _quarto.yml # config raíz -│ ├── _brand.yml # paleta + tipografías CRPTO -│ ├── styles.scss # CSS custom -│ ├── chapters/ # 24 .qmd -│ ├── includes/ # snippets reutilizables -│ ├── assets/figures/ # PNG/PDF (editorial, notebooks, publication) -│ ├── _helpers/ # paquete Python con load_artifacts, plot_helpers -│ ├── references.bib # 70 entradas -│ └── apa.csl # estilo APA7 -├── src/ # paquete Python `crpto` -│ ├── data/ # ingesta + cleaning -│ ├── features/ # FE + Pandera contracts -│ ├── models/ # PD + conformal + calibración -│ ├── evaluation/ # métricas + fairness + backtesting -│ ├── optimization/ # Pyomo/HiGHS + robust + SPO -│ └── utils/ # I/O, MLflow, helpers -├── scripts/ # 40+ entry points del pipeline -├── tests/ # 26 pytest (slow / integration markers) -├── configs/ # YAML config (modelos, conformal, optim, fairness) -├── dbt_project/ # 6 modelos (3 staging + 3 marts) sobre crpto.duckdb -├── data/ -│ ├── raw/ # Lending Club CSV (1.7 GB) — NO commit -│ └── processed/ # parquets + DuckDB — DVC tracked -├── models/ # champion + calibrator + status JSONs -├── reports/ -│ ├── crpto/tables/ # 18 CSVs del paper -│ ├── crpto/figures/ # 8 PNGs/PDFs -│ └── mrm/ # Model Risk Management cards (skops) -├── docs/research/ # dossier académico -├── paper/ # manuscrito principal -├── notebooks/ # exploraciones (Jupyter) -├── dvc.yaml / dvc.lock # pipeline de 13 stages -├── pyproject.toml # dependencias y tooling -├── uv.lock # lockfile reproducible -├── justfile # task runner cross-platform -└── EXTRACTION_MANIFEST.json # hashes de artefactos congelados -``` - -## Reglas de operación - -1. **Champion congelado es ley.** Si una tarea sugiere re-correr stages que afecten al champion, parar y preguntar. -2. **No modificar `EXTRACTION_MANIFEST.json` ni artefactos listados** sin permiso. -3. **No subir secretos** (`.env`, tokens, credenciales DagsHub/AWS). -4. **No re-formatear el libro completo** en una sola pasada — preserva freeze cache y diffs limpios. -5. **Render del libro tras cambios a `_quarto.yml`** o capítulos, con QA visual. -6. **Antes de commit**: `just lint && just smoke` deben pasar (pre-commit hook lo enforza). -7. **No bypassar hooks** (`--no-verify`) sin permiso. -8. **Windows-first**: cualquier script o comando nuevo debe correr en Windows PowerShell sin depender de shells Unix. -9. **`uv run`** para invocar herramientas Python (`pytest`, `quarto`, `dbt`, `dvc`, `mlflow`, `optuna`). -10. **Repo público**: `https://github.com/EigenCharlie/Paper_CRPTO`. No subir secretos ni artefactos pesados; usar DVC remote para datos/modelos. -11. **Branch de trabajo**: para código/refactors usa rama y PR. Hotfixes de docs/CI en `main` solo si el usuario lo pide explícitamente. -12. **Drift-gate tras tocar la capa conformal/PD.** Cualquier refactor de `src/models/conformal*.py`, `src/models/optuna_tuning.py`, `scripts/generate_conformal_intervals.py` o `scripts/train_pd_model.py` debe pasar `just drift-gate` (diff bit-exacto vs la cadena del certificado). Un ROJO significa cambio numérico, no refactor: parar y preguntar. Es la red de seguridad que permitió descomponer los `main()` sin tocar el certificado. - -## Qué stages son seguros re-correr - -| Stage DVC | ¿Seguro? | Notas | -| --- | --- | --- | -| `crpto.data.dataset` | ⚠️ Lento (1.7 GB) | Determinista; no rompe champion pero re-corre todo downstream. | -| `crpto.data.splits` | ❌ NO | Regenera `train/test/calibration.parquet` (congelados en el manifest). En la deny-list: un `dvc repro` aquí cascada hasta el champion. | -| `crpto.data.features` | ⚠️ | No rompe champion pero re-corre todo downstream; ahora depende de `calibration.parquet`. | -| `crpto.pd.champion` | ❌ NO | Rompe `pd_canonical.cbm`. Ahora también produce `test_predictions.parquet` como out. | -| `crpto.conformal.intervals` | ❌ NO | Rompe intervalos congelados. | -| `crpto.conformal.validation` | ❌ NO | Rompe `conformal_policy_status.json`. | -| `crpto.portfolio.optimization` | ❌ NO | Rompe `portfolio_allocations.parquet`. | -| `crpto.portfolio.bound_exact_eval` | ❌ NO | Rompe `portfolio_bound_aware_bound_eval.parquet`. | -| `crpto.paper.export_tables` | ✅ Sí | Determinista; regenera CSVs. | -| `crpto.paper.evidence` | ✅ Sí | Determinista; regenera `crpto_evidence_status.json`. | -| `crpto.paper.journal_package` | ✅ Sí | Determinista. | -| `crpto.paper.figures` | ✅ Sí | Determinista. | -| `crpto.paper.spo_stability` | ✅ Sí | Determinista. | -| `crpto.book.render` | ✅ Sí | Render Quarto; output a `book/_book/`. | - -## Scope operativo - -El documento rector es `docs/SCOPE_AND_GOVERNANCE.md`. En corto: - -- Seguro: docs, Quarto no-execute, CI, tests utilitarios, tablas/figuras/evidence/journal package. -- Revisar antes: cambios dbt/DVC/dependencias que afecten contratos de datos. -- No seguro en `main`: PD champion, intervalos conformal, validación conformal, optimización portfolio, exact eval, MAPIE/conformal/feature-config migrations sin drift report. - -## Sub-agentes y MCP útiles - -- `Explore` (built-in) para búsquedas de código y archivos. -- `Plan` (built-in) para diseñar cambios estructurales. -- **Context7 MCP** — docs actualizadas de pandas/sklearn/MAPIE/Quarto/dbt. -- **Chrome/Playwright MCP** — QA visual del libro renderizado. -- **DuckDB MCP** — queries directas a `data/processed/crpto.duckdb`. -- **GitHub MCP** — repo `EigenCharlie/Paper_CRPTO`. - -## Skills custom del proyecto (`.claude/skills//SKILL.md`) -- `/crpto-render` — render del libro + QA visual con Playwright. -- `/crpto-stage` — `dvc repro` de un stage aislado, con dry-run. -- `/crpto-smoke` — smoke tests + dbt parse + dbt test. -- `/crpto-paper-export` — pipeline completo de salida journal. -- `/crpto-validate-champion` — verifica hashes vs `EXTRACTION_MANIFEST.json`. -- `/crpto-mrm-card` — actualiza model cards en `reports/mrm/`. -- `/crpto-claim-sync` — gate de sincronía de claims paper/gobernanza; obligatorio tras editar paper o supplement. -- `/crpto-submission-freeze` — checklist ejecutable del freeze de submission IJDS (gates, PDFs, anonimato, páginas). +`submission-check` must retain `validate-champion-strict`. PD or conformal +implementation changes also require the separately authorized drift gate. -## Plan vigente +## Manuscript Rules -Las fases bootstrap ya fueron publicadas. Los cambios estructurales que quedan -viven como planes explícitos en `docs/refactor/` y no se ejecutan sin validar -drift contra el champion congelado. +- Edit `paper/CRPTO_ijds.qmd` and `paper/supplement_ijds.qmd`. +- Generate official TeX with `scripts/build_ijds_submission_tex.py`; never edit + generated TeX directly. +- Treat sharp bounds as finite-archive identification bounds, not confidence + intervals. +- Distinguish fit-label timing from evaluation-endpoint availability. +- Distinguish five-model controls from CatBoost-only sensitivities. +- Keep iteration history out of the manuscript. +- Keep the submission double-anonymous. -## Cómo me gusta trabajar +## Compatibility And Archive -- Respuestas concisas. Sin resúmenes redundantes al final. -- Tool calls en paralelo cuando son independientes. -- Plan-first para cambios no triviales (3+ archivos). -- Comentarios solo cuando el WHY no es obvio. -- Sin emojis en archivos salvo que los pida. -- En español para el libro/paper/docs; inglés para código/docstrings/identifiers/CI. +The complete pre-consolidation worktree, Git mirror, bundle, and selected +historical snapshots are preserved at `D:\crpto_legacy`. In-repository +historical files are retained only when path-bound by the immutable extraction +manifest or DVC graph. They must not appear in active commands, claims, or +paper narrative. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc3794e..3901d72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,123 +1,87 @@ -# Contributing to CRPTO +# Contributing To CRPTO -This is a single-author academic project (master's thesis + paper + -journal package). It is **not** a community-driven open-source library. -Pull requests from external contributors are not expected; this document -exists so that reviewers (MRM, journal, defense committee) can -reproduce the deliverables and understand the operational guardrails. +CRPTO is a single-author academic repository for one IJDS manuscript, not a +general-purpose Python package. Contributions should improve the validity, +clarity, or reproducibility of that manuscript. -If you are reading this because you are a reviewer or a future agent that -inherited the repo, read in this order: +## Start Here -1. [`README.md`](README.md) — what the repo is and how to run the book. -2. [`docs/ACADEMIC_CONTEXT.md`](docs/ACADEMIC_CONTEXT.md) — why the - tooling is intentionally minimal and how the champion-versus-search - distinction works. -3. [`docs/SCOPE_AND_GOVERNANCE.md`](docs/SCOPE_AND_GOVERNANCE.md) — - what is in scope, what is forbidden in `main`, the release - checklist. -4. [`CLAUDE.md`](CLAUDE.md) — operating rules for AI agents working in - this repo. - -## Reproducing the paper outputs +Read `CLAUDE.md`, the active claim registry, the source registry, and the +publication contract before changing scientific code or prose. ```powershell -# One-time setup (Windows PowerShell) -git clone https://github.com/EigenCharlie/Paper_CRPTO.git -cd Paper_CRPTO -uv venv -uv sync --extra dev --extra search -just smoke # runs artifact-independent guardrail tests +uv sync --extra dev +just smoke ``` -To re-render the Quarto book without re-executing any chunks (uses -`_freeze`): +## Ordinary Changes -```powershell -just book # uv run -- quarto render book --to html --no-execute -``` +The following are normally safe when their tests pass: + +- prose and citation corrections that remain inside the active claim boundary; +- tests and validation code that do not rewrite registered experiment roots; +- deterministic evidence, table, figure, and submission builders; +- CI, formatting, typing, and reproducibility documentation; +- refactors whose outputs are unchanged and whose compatibility paths remain + intact. -To regenerate the paper tables and figures from frozen inputs: +Run: ```powershell -just paper-export # tables + figures + evidence + journal package + book +just test +just lint +just type-check +just type-check-fast +just publication-integrity +just ijds-active-check +just validate-champion ``` -These commands never touch the champion artefacts on disk and never -re-run any DVC stage that performs *search* (the 276k portfolio sweep -or Optuna HPO are out of scope by policy). - -## What you may change freely - -- Documentation, comments, docstrings. -- Quarto prose, glossary entries, chapter ordering. -- Test coverage (especially in `tests/test_utils/`, `tests/test_optimization/`, - `tests/test_features/`). -- Tooling: ruff config, hooks, IDE settings, justfile recipes. -- New scripts that read frozen artefacts without overwriting them. - -## What requires a deliberate revalidation plan +## Scientific Changes -Anything that would change the bytes of these files: +A new estimand, data role, endpoint rule, model, comparator, sensitivity, or +optimization contract is a new scientific object, even if implemented as a +small code edit. Before running it: -- `models/pd_canonical.cbm` -- `models/pd_canonical_calibrator.pkl` -- `models/final_project_promotion.json` -- `models/conformal_policy_status.json` -- `data/processed/conformal_intervals_mondrian.parquet` -- The `portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_*` directory -- `EXTRACTION_MANIFEST.json` +1. state the research question and stop rule; +2. declare the information set and all outcome-blind choices; +3. assign a new run tag and contained output path; +4. separate freeze from outcome evaluation; +5. register exact hashes and DVC pointers before using results in prose; +6. reconcile every reported number against the evidence manifest. -`tests/test_manifest_regression.py` will fail loudly if these drift. +Do not select a model, ruler, coordinate, gamma, scenario, or policy from OOT +outcomes. -A revalidation plan must include: +## Protected Boundary -1. A branch dedicated to the change (never on `main`). -2. A drift report comparing the new artefact to the frozen one with - tolerances documented in `docs/ACADEMIC_CONTEXT.md`: - max abs diff `≤ 1e-6` per loan on conformal intervals, coverage - delta `≤ 5e-4` per Mondrian cell, robust return delta `≤ $1.00`. -3. If the drift is non-zero, the change is not a refactor — it is a - model change and needs a fresh run tag. +Never run the protected `crpto.pd.champion`, `crpto.conformal.intervals`, +`crpto.conformal.validation`, `crpto.portfolio.optimization`, or +`crpto.portfolio.bound_exact_eval` stages without explicit permission. Do not +modify `EXTRACTION_MANIFEST.json` or protected model/data artifacts. -See the three plans under `docs/refactor/` for the pre-written templates -(MAPIE migration, conformal module split, feature_config Parquet). +`dvc.yaml` and manifest-fixed paths form a sealed compatibility capsule. Their +presence does not make them active workflows. The active execution surface is +the allow-list in `configs/crpto_publication_targets.yaml`. -## What is forbidden in `main` +## Paper Workflow -The DVC search stages listed in `docs/ACADEMIC_CONTEXT.md` are blocked -by `.claude/settings.json` and `.codex/skills/crpto/SKILL.md`. Do not -run them from the default branch: +Edit the canonical QMD files, not generated outputs: -- `dvc repro crpto.portfolio.bound_exact_eval` -- Any Optuna HPO that would overwrite the frozen study. - -## Code style - -- Run `just lint` (ruff check + format check) before any commit. -- Run `just smoke` to verify the tests that the pre-push hook will - re-run anyway. -- Mypy strict applies to a small allow-list of new modules - (`src/optimization/policy.py`, `src/utils/pipeline_state.py`, - `src/utils/mlflow_tracing.py`, `src/utils/optuna_storage.py`). - Other modules use a laxer config to absorb research-grade code. -- No new top-level scripts; add to `src/` or `scripts/` according to - whether the code is library or pipeline. -- Spanish for book/paper prose; English for code, docstrings, tests, - CI and changelog. - -## Releasing - -Single-author releases happen by tagging a clean `main`: - -```bash -git tag -a vX.Y.Z -m "release notes here" -git push origin vX.Y.Z +```powershell +just submission-build +just submission-check ``` -The tag triggers the `book-publish` workflow which deploys the latest -HTML rendering of the book to GitHub Pages. +The official TeX is generated from `paper/CRPTO_ijds.qmd`. Keep author identity +out of reviewer-facing files and keep project-version history out of the paper. -## Citing +## Style -See [`CITATION.cff`](CITATION.cff) for the canonical citation block. +- English for code, tests, and manuscript prose. +- Type new public functions and keep comments limited to non-obvious logic. +- Prefer existing modules and structured parsers over new wrappers. +- Avoid one-off scripts when a current library function or registered runner + already owns the behavior. +- Do not add services, dashboards, notebooks, or release machinery that does + not improve the paper or its reproducibility. diff --git a/EXTRACTION_MANIFEST.md b/EXTRACTION_MANIFEST.md index d67cfb1..ab16f8c 100644 --- a/EXTRACTION_MANIFEST.md +++ b/EXTRACTION_MANIFEST.md @@ -12,7 +12,8 @@ exists, and how `tests/test_manifest_regression.py` enforces it. - **Manifest scope**: frozen upstream baseline plus the historical pool93 promotion recorded at extraction time. The current IJDS manuscript policy is newer and is governed separately by - `docs/research/active_claims_2026-07-04.md`. + `docs/research/active_claims_2026-07-14.md`. The retired registry remains + recoverable from Git history. - **187 critical files** are hashed under `critical_hashes` (SHA256 + byte count). - **Historical pool93 claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, diff --git a/README.md b/README.md index 4c61475..5f1475d 100644 --- a/README.md +++ b/README.md @@ -1,253 +1,154 @@ -# CRPTO — Conformal Robust Predict-Then-Optimize - -Pipeline de investigación y libro Quarto que acompañan el paper **CRPTO**, una metodología que integra *conformal prediction* con *optimización robusta de carteras* aplicada a riesgo de crédito (datos de Lending Club, 2007–2020). - -> CRPTO opera como repositorio standalone: GitHub, DVC y MLflow apuntan a recursos propios del paper. La historia de extracción y aprendizajes queda documentada en [`docs/PROJECT_HISTORY.md`](docs/PROJECT_HISTORY.md). - -## Claim IJDS activo - -| Campo | Valor | -| --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | -| Conformal | replay exacto al `90%` (`alpha=0.10`, used `0.095`) | -| Política | `q=(p+u)/2`, `tau=0.17`; PD puntual en el objetivo y `q` en el guardrail | -| Selector | grilla `3x3` en noviembre; cap determinista `B_u<=0.28`; misma política en auditoría de diciembre | -| Retorno realizado | **$179,327.59** | -| Default / miscoverage ponderados | `0.039375 / 0.036875` | -| `Gamma_CP / Gamma_residual` | `0.176102 / 0.088051` | -| Endpoint / contabilidad observada / umbral condicional | `0.258051 / 0.294926 / 0.574279` | -| Baseline A40 | `$196,369.14`; costo de retorno `8.678%`; reducción de default `7.9025` pp | -| Auditoría pre-OOT | diciembre: default `0.145650`, miscoverage `0.124925`; estabilidad no implica cobertura seleccionada | - -Hashes SHA256 de los artefactos críticos están en [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json). Verifica con `just validate-champion` o el skill `/crpto-validate-champion`. -El rebaseline y la frontera pool93 anterior se conservan como procedencia -congelada, no como claims activos del manuscrito IJDS. - -## Requisitos del sistema - -| Herramienta | Versión mínima | Notas | -| --- | --- | --- | -| Python | 3.11 (≤3.12) | Declarado en `.python-version` y `pyproject.toml`. | -| [uv](https://docs.astral.sh/uv/) | 0.4+ | Gestor de dependencias. Reemplaza pip/poetry. | -| [just](https://github.com/casey/just) | 1.28+ | Task runner cross-platform. Reemplaza `make`. Windows: `winget install Casey.Just`. | -| [Quarto CLI](https://quarto.org/docs/get-started/) | 1.9+ | Para renderizar el libro. CI usa 1.9.35. | -| LaTeX (LuaLaTeX) | TeX Live 2024+ | Solo si renderizas el PDF. Opcional para HTML. | -| DuckDB CLI | 1.3+ | Opcional, queries directas a `data/processed/crpto.duckdb`. | -| Git | 2.40+ | Para hooks pre-commit. | - -En Windows, `uv`, `just` y `quarto` deben estar en `PATH`. El venv oficial del -proyecto vive en `.venv/Scripts/`; usa PowerShell como shell normal de trabajo. - -## Setup rápido +# CRPTO + +CRPTO is a research repository for one manuscript targeted to the **INFORMS +Journal on Data Science**. It studies what happens when a binary conformal +score is used as a coefficient in a monthly credit-allocation linear program. +The contribution is an identification audit of the +machine-learning--conformal--optimization interface, not a promoted lending +policy or a new credit-scoring leaderboard. + +## Active Result + +The active design uses the Lending Club 2007--2020Q3 archive and declares all +time roles before evaluation. + +| Quantity | Active value | +|---|---:| +| Raw archive | 2,925,493 rows | +| Eligible 36-month design universe | 640,543 loans | +| Primary OOT candidates | 376,890 loans | +| Resolved / unresolved at the six-month endpoint | 364,814 / 12,076 | +| Coverage controls | 5 frozen learner specifications x 8 windows | +| Largest all-candidate coverage upper bound | 0.897726 | +| Two-ruler optimization solves | 6,240 | +| Exact point-cap frontier | 3,067 caps | +| Broad-support comparator envelopes crossing zero | 216 / 216 | +| Structural sensitivity | 36 complete scenarios | +| Joint fit-label completion stress | 32 / 32 upper bounds below 0.90 | +| USD 25 floor diagnostic | maximum rate perturbation 0.001284 pp | + +Under the declared six-month outcome-availability rule, all 40 sharp +all-candidate coverage upper bounds are below 0.90. Separately, the primary +CatBoost eight-window pattern recurs under three feature-semantics-preserving +missingness encodings and at a later retrospective origin. Portfolio direction +changes with the outcome-blind ruler, coordinate, and comparator support; no +model, encoding, gamma, ruler, coordinate, scenario, or policy is selected. + +These are retrospective, archive-specific identification results. They are not +prospective validity, selected-set conformal coverage, causal lending effects, +cash-flow returns, or deployment evidence. + +## Sources Of Truth + +Read these in order: + +1. [`docs/research/active_claims_2026-07-14.md`](docs/research/active_claims_2026-07-14.md) +2. [`configs/ijds_active_evidence_sources.yaml`](configs/ijds_active_evidence_sources.yaml) +3. [`configs/ijds_claim_ledger.yaml`](configs/ijds_claim_ledger.yaml) +4. [`reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`](reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json) +5. [`paper/CRPTO_ijds.qmd`](paper/CRPTO_ijds.qmd) +6. [`paper/supplement_ijds.qmd`](paper/supplement_ijds.qmd) + +The source registry owns lineage identities and 31 DVC pointers. The evidence +manifest is the only numeric paper-facing manifest. The body QMD is the only +editable source for the official submission TeX. + +## Architecture + +```text +raw archive + frozen experiment roots + | + v + configs/ijds_active_evidence_sources.yaml + | + v + scripts/build_ijds_binary_geometry_frontier_v4_evidence.py + | + +--> reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json + +--> reports/crpto/tables/crpto_ijds_v4_*.csv + +--> reports/crpto/figures/crpto_ijds_v4_*.{png,pdf} + | + v + paper/CRPTO_ijds.qmd + paper/supplement_ijds.qmd + | + v + generated HTML/PDF + official INFORMS TeX/PDF +``` -```powershell -# Una sola vez -just setup # uv sync --extra dev --extra search --extra spo -# O setup ligero (sin pyepo/torch) -just setup-base +Current reusable code is under `src/ijds_audit`, `src/ijds_challengers`, and +the retained data, model, evaluation, and optimization modules they import. +Current experiment entrypoints are exactly the `scripts/experiments/run_ijds_*` +files named by `configs/crpto_publication_targets.yaml`. -# Copia el archivo de entorno y rellena con tus tokens reales -cp .env.example .env -# Edita .env (DagsHub/MLflow/DVC standalone de CRPTO) +`dvc.yaml`, `dvc.lock`, and paths fixed by `EXTRACTION_MANIFEST.json` form a +sealed compatibility capsule. They preserve old hashes and path-bound replay +metadata but are not active workflows or manuscript evidence. The complete +pre-consolidation repository is archived outside the project at +`D:\crpto_legacy`. -# Verifica que todo está sano -just smoke # tests críticos rápidos -``` +## Setup -## Comandos principales +Requirements: Python 3.11 or 3.12, `uv`, `just`, Quarto, Git, and TeX Live. ```powershell -# Libro Quarto -just book # HTML -just book-pdf # no-op: PDF completo diferido hasta tesis curada -just book-preview # live reload -just book-clean # borra _book/, _freeze/, .quarto/ - -# Pipeline de paper (no toca el champion) -just paper-export # tablas + figuras + evidence + journal + libro -just ijds-evidence # A35--A40 y gobernanza de la política IJDS activa -just tables # solo CSVs -just figures # solo PNGs/PDFs - -# Calidad -just lint # ruff check + format check -just fmt # ruff fix + format -just type-check # mypy src scripts -just type-advisory # ty sobre ruta activa IJDS, no bloqueante -just type-advisory-full # ty sobre src/scripts completos; bloquea el cierre IJDS -just api-docs-core # pdoc local para modulos core, salida ignorada -just hooks-check # valida hooks con pre-commit y prek -just smoke # tests críticos rápidos -just test # suite completa -just submission-check # cierre IJDS: claims, lint, type, suite completa, champion y PDF oficial - -# DVC -just dvc-status # drift detection -just dvc-dag # imprime el DAG en markdown - -# dbt -just dbt-test # parse + tests -just dbt-build # marts materializadas - -# Listado completo -just help +uv sync --extra dev +just smoke ``` -## Alcance y reglas de operación - -El repositorio es público y está dedicado solo a CRPTO. El alcance operativo, -qué se puede regenerar, qué no debe tocarse en `main`, cómo manejar secretos y -qué refactors requieren revalidación están documentados en -[`docs/SCOPE_AND_GOVERNANCE.md`](docs/SCOPE_AND_GOVERNANCE.md). - -Regla corta: documentación, CI, tablas, figuras y journal package son seguros; -reentrenar PD, recalcular intervalos conformal o reoptimizar el champion no lo -es sin una rama de revalidación y drift report. - -## Estrategia de publicación +Use Windows PowerShell and `uv run` for Python commands. -La decisión editorial activa es escribir primero para **INFORMS Journal on Data -Science** y mantener **European Journal of Operational Research** como pivote -principal si el manuscrito termina leyendo más como OR aplicada que como data -science reproducible. La decisión vive en -[`configs/crpto_publication_targets.yaml`](configs/crpto_publication_targets.yaml) -y está explicada en -[`docs/research/crpto_publication_strategy_2026-05-12.md`](docs/research/crpto_publication_strategy_2026-05-12.md). - -Los borradores de extracción están en [`paper/`](paper/): - -- `paper/CRPTO_ijds.qmd`: cuerpo anónimo IJDS. -- `paper/supplement_ijds.qmd`: online supplement con A3--A18, reproducibilidad, - MRM y fairness. -- `paper/CRPTO.qmd`: entrada genérica. - -Comandos rápidos: +## Main Commands ```powershell -just paper-ijds -just paper-ijds-supplement -just paper-submission -``` - -## Estructura - +just test # complete retained test suite +just lint # Ruff check and format check +just type-check # mypy +just type-check-fast # blocking ty check on the active surface +just publication-integrity # source, claim, and artifact contracts +just drift-gate # read-only PD/conformal/evidence regression +just ijds-active-check # scientific and manuscript synchronization +just submission-build # evidence, HTML, TeX, PDFs, previews +just submission-check # all read-only submission gates +just submission-closeout # build, check, and remote DVC verification ``` -. -├── CLAUDE.md # Contexto para Claude Code -├── .claude/ # Configuración Claude Code + skills CRPTO -│ ├── settings.json # Permisos pre-aprobados (compartido) -│ └── skills/ # 6 skills custom (/crpto-render, /crpto-smoke, ...) -├── .codex/ # Skill local CRPTO para Codex -├── .github/workflows/ # CI/CD (lint, book-publish, tests-full manual) -├── .pre-commit-config.yaml # ruff + nbstripout + dvc-status + smoke -├── pyproject.toml # Deps y tooling (ruff, pytest, mypy) -├── uv.lock # Lockfile reproducible -├── justfile # Task runner cross-platform -├── dvc.yaml / dvc.lock # Pipeline de 13 stages -├── EXTRACTION_MANIFEST.json # Hashes SHA256 de artefactos congelados -├── book/ # Libro Quarto (24 capítulos en español) -│ ├── _quarto.yml -│ ├── _brand.yml # Paleta + tipografías (Fase 3) -│ ├── styles.scss # CSS custom con dark mode + print media -│ ├── chapters/ # 24 .qmd (manuscrito + dossier) -│ ├── includes/ # snippets reutilizables -│ ├── assets/figures/ # editorial/, notebooks/, publication/ -│ ├── _helpers/ # paquete Python con load_artifacts, plot_helpers -│ ├── references.bib # 70 entradas -│ └── apa.csl # estilo APA 7 -├── crpto/ # paquete público mínimo (`import crpto`) -├── src/ # módulos fuente históricos (data, features, models, optimization, evaluation, utils) -├── scripts/ # entry points; ver scripts/README.md para rutas IJDS vs históricas -├── tests/ # 26 archivos pytest (markers slow / integration) -├── configs/ # YAML (pd_model, conformal, optimization, fairness, mrm) -├── dbt_project/ # 3 staging + 3 marts sobre crpto.duckdb -├── data/ -│ ├── raw/ # Lending Club CSV (1.7 GB) — DVC tracked -│ └── processed/ # parquets + DuckDB -├── models/ # champion + calibrator + status JSONs -├── reports/ -│ ├── crpto/tables/ # 18 CSVs del paper -│ ├── crpto/figures/ # 8 PNGs/PDFs -│ └── mrm/ # Model risk cards (skops) -├── docs/research/ # dossier académico -├── paper/ # manuscrito principal + versión IJDS + supplement -└── notebooks/ # exploraciones Jupyter -``` - -## Qué stages son seguros re-correr -| Stage DVC | Seguro | Notas | -| --- | --- | --- | -| `crpto.data.dataset` | ⚠️ Lento | 1.7 GB; regenera todo downstream. | -| `crpto.data.features` | ⚠️ | Regenera train_fe/test_fe/calibration_fe. | -| `crpto.pd.champion` | ❌ | Rompe `pd_canonical.cbm`. | -| `crpto.conformal.intervals` | ❌ | Rompe intervalos congelados. | -| `crpto.conformal.validation` | ❌ | Rompe `conformal_policy_status.json`. | -| `crpto.portfolio.optimization` | ❌ | Rompe `portfolio_allocations.parquet`. | -| `crpto.portfolio.bound_exact_eval` | ❌ | Rompe `portfolio_bound_aware_bound_eval.parquet`. | -| `crpto.paper.export_tables` | ✅ | Determinista; regenera CSVs. | -| `crpto.paper.evidence` | ✅ | Determinista. | -| `crpto.paper.journal_package` | ✅ | Determinista. | -| `crpto.paper.figures` | ✅ | Determinista. | -| `crpto.paper.spo_stability` | ✅ | Determinista. | -| `crpto.book.render` | ✅ | Render Quarto. | +The manual fallback for official TeX compilation is intentionally +`pdflatex -> bibtex -> pdflatex -> pdflatex`: the first pass writes citation +and cross-reference metadata, BibTeX writes the bibliography, and the final two +passes stabilize references and pagination. -## Troubleshooting +## Protected Boundary -**`quarto: command not found`** — instala Quarto CLI desde https://quarto.org/docs/get-started/ y reabre PowerShell. +Do not run these historical DVC stages without explicit permission: -**`uv: command not found`** — `winget install --id=astral-sh.uv` en Windows, o `pip install uv`. +- `crpto.pd.champion` +- `crpto.conformal.intervals` +- `crpto.conformal.validation` +- `crpto.portfolio.optimization` +- `crpto.portfolio.bound_exact_eval` -**Render del libro falla con `ModuleNotFoundError`** — los chunks Python requieren el venv activo. `uv run -- quarto render book` resuelve esto. +Do not modify `EXTRACTION_MANIFEST.json` or its protected model/data artifacts. +Use `just validate-champion` for ordinary work and +`just validate-champion-strict` when every protected artifact is available. -**El entorno Python parece incorrecto** — borra variables locales que cambien el -entorno de `uv` o Quarto, recrea con `uv venv && uv sync --extra dev --extra -search`, y trabaja desde PowerShell en `C:\Users\carlos\Documents\Paper_CRPTO`. +## Paper Editing -**`dvc status` muestra muchos cambios** — probablemente el lockfile cambió. Ejecuta `just dvc-status` para ver detalle. Si los stages del champion están afectados, NO repro: documenta y consulta antes. - -**Pre-commit bloquea un commit** — corre `just fmt && just smoke` para arreglar lint/format y verificar tests. Si el hook `dvc-status` falla, hay drift inesperado. - -**Render PDF falla** — los PDFs activos son los borradores IJDS -(`just paper-submission-pdf`) y, más adelante, un PDF APA de tesis curado. El -PDF completo del libro no se mantiene como artefacto rutinario; ver -[`docs/THESIS_PDF_SCOPE.md`](docs/THESIS_PDF_SCOPE.md). - -## Herramientas interactivas para reviewers - -Comandos opcionales útiles para inspeccionar el champion sin tocar el pipeline: +Edit `paper/CRPTO_ijds.qmd`, then generate the official TeX with: ```powershell -just duckdb # REPL DuckDB sobre data/processed/crpto.duckdb -just datasette # UI web sobre el warehouse (requiere datasette + datasette-duckdb) -just dbt-docs # UI dbt en http://localhost:8088 -just optuna-dashboard # Optuna Dashboard sobre el journal de HPO -just pipeline-state # Snapshot JSON de todos los status del pipeline +just paper-tex ``` -Ninguno modifica artefactos congelados — solo leen `data/processed/` y `models/`. - -## Documentación adicional - -- [`CLAUDE.md`](CLAUDE.md) — Contexto operativo para Claude Code (champion, comandos, convenciones). -- [`CONTRIBUTING.md`](CONTRIBUTING.md) — Cómo reproducir las salidas y qué requiere plan de revalidación. -- [`CHANGELOG.md`](CHANGELOG.md) — Historial de cambios. -- [`docs/ACADEMIC_CONTEXT.md`](docs/ACADEMIC_CONTEXT.md) — Single-author, dataset estático, sin producción. -- [`docs/SCOPE_AND_GOVERNANCE.md`](docs/SCOPE_AND_GOVERNANCE.md) — Alcance CRPTO, límites del repo público y reglas de refactor. -- [`docs/refactor/`](docs/refactor/) — Planes de refactor diferido (MAPIE, conformal split, feature_config Parquet). -- [`docs/research/`](docs/research/) — Dossier académico (conformal prediction readme, audit, integrations). -- [`paper/README.md`](paper/README.md) — Workspace de manuscrito, target IJDS y - comandos de render. -- [`docs/security/SECRETS_AND_REMOTES.md`](docs/security/SECRETS_AND_REMOTES.md) — Variables de entorno, secretos y remotes DVC/MLflow para Windows. -- [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json) + [`EXTRACTION_MANIFEST.md`](EXTRACTION_MANIFEST.md) — Hashes y narrativa de la extracción. - -## Citar este trabajo +Never edit `paper/submission/CRPTO_ijds_submission.tex` directly. The paper is +double-anonymous; author-identifying material belongs only in the separate +submission forms. -Ver [`CITATION.cff`](CITATION.cff). Resumen: +## Data And Literature -> Vergara Rojas, C. A. (2026). *CRPTO: predict-then-optimize con conformal prediction para riesgo de crédito* [Master's thesis]. https://github.com/EigenCharlie/Paper_CRPTO +The raw CSV and experiment roots are DVC-managed and excluded from Git. The +local `Papers_tesis/` corpus is also excluded because it contains copyrighted +PDFs; bibliographic metadata belongs in `paper/references.bib`. -## Licencia +## License And Citation -- Código fuente: [MIT](LICENSE). -- Texto del libro/paper, figuras y tablas: [CC BY 4.0](LICENSE-CONTENT). +Code is MIT licensed. See [`CITATION.cff`](CITATION.cff) for citation metadata. diff --git a/book/__init__.py b/book/__init__.py deleted file mode 100644 index 4f84e71..0000000 --- a/book/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""``book`` package marker. - -Quarto chunks under ``book/chapters/*.qmd`` import the artifact loaders and -plot helpers as ``from book._helpers import ...``. Marking this directory as -a regular package (instead of an implicit namespace package) keeps the -import path deterministic when the chunks run from ``book/_book/`` cache -locations and avoids surprises with editable installs. - -There is no public top-level API here; consumers should reach into the -``book._helpers`` submodule directly. -""" diff --git a/book/_brand.yml b/book/_brand.yml deleted file mode 100644 index 25f8d49..0000000 --- a/book/_brand.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Quarto 1.6+ brand configuration — single source of truth for palette and typography. -# Picked up automatically by `_quarto.yml`. Overrides values from `styles.scss` -# while remaining backwards-compatible (styles.scss continues to drive components). - -meta: - name: CRPTO - subtitle: Conformal Robust Predict-Then-Optimize - -color: - palette: - crpto-primary: "#1f6feb" - crpto-primary-soft: "#f6f8fb" - crpto-accent: "#5fa8ff" - crpto-surface: "#f8fafc" - crpto-ink: "#152238" - crpto-muted: "#526173" - crpto-band-train: "#e7f0ff" - crpto-band-cal: "#eef7e9" - crpto-band-test: "#fff2e3" - foreground: crpto-ink - background: "#ffffff" - primary: crpto-primary - secondary: crpto-accent - success: "#059669" - warning: "#d97706" - danger: "#dc2626" - info: "#0891b2" - link: crpto-primary - light: crpto-surface - dark: crpto-ink - -typography: - fonts: - - family: Inter - source: google - weight: [300, 400, 500, 600, 700] - style: [normal] - - family: Source Serif 4 - source: google - weight: [400, 600, 700] - style: [normal, italic] - - family: JetBrains Mono - source: google - weight: [400, 500] - style: [normal] - base: - family: Inter - weight: 400 - line-height: 1.55 - headings: - family: Inter - weight: 600 - line-height: 1.25 - monospace-inline: - family: JetBrains Mono - monospace-block: - family: JetBrains Mono - link: - color: crpto-primary - decoration: underline - -logo: - small: assets/logo-small.svg - medium: assets/logo.svg - large: assets/logo.svg - -defaults: - bootstrap: - defaults: - border-radius: 6px - enable-shadows: true diff --git a/book/_extensions/README.md b/book/_extensions/README.md deleted file mode 100644 index 824c903..0000000 --- a/book/_extensions/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# Quarto extensions for the CRPTO book - -This directory holds Quarto extensions installed via `quarto add `. The -project currently relies on Quarto built-ins (lightbox, code-fold, hover refs, -etc.) and does not require any extension to render. - -The list below documents the **recommended** extensions for an academic book of -this kind. Install only the ones you need; each one adds metadata to -`_extensions//_extension.yml` and an entry in `.gitignore`-tracked -directories. - -## Recommended - -```bash -# Run from the repository root. - -# Glossary support — useful for CRPTO terminology (PD, ECE, V(α), Γ_CP, MRM, SR 11-7). -quarto add coatless/quarto-glossary - -# Embed snippets from src/ directly into chapters without copy-paste. -quarto add quarto-ext/include-code-files - -# Iconify icons for chapter cards and callouts (decorative). -quarto add mcanouil/quarto-iconify - -# Pretty typesetting of LaTeX, BibTeX, X̄ symbols. -quarto add quarto-ext/fancy-text -``` - -## Journal templates (install ONE when you choose a target journal) - -```bash -quarto add quarto-journals/elsevier -quarto add quarto-journals/acm -quarto add quarto-journals/springer -``` - -After installing a template, add the appropriate format block to -`_quarto.yml`, e.g. `format: { elsevier-pdf: default }`. - -## Why we did not install these automatically - -`quarto add` requires: - -1. The Quarto CLI to be in `PATH` (it should be). -2. Internet access to fetch the extension from GitHub. -3. A clean git working tree (it modifies files and the commit history). - -Because the project is being prepared for a first GitHub push, we are leaving -the extension installation to the user so the initial commit history remains -predictable. - -## Built-ins already in use - -- **Lightbox**: `lightbox: auto` in `_quarto.yml` (Quarto 1.5+). -- **Hover citations / cross-refs**: `citations-hover: true`, `crossrefs-hover: true`. -- **Code folding and tools**: `code-fold: show`, `code-tools: { toggle: true }`. -- **APA bibliography**: `csl: apa.csl` + `bibliography: references.bib`. -- **Dark mode**: `theme: { light: ..., dark: darkly }`. -- **`df-print: paged`**: paged tables in HTML. diff --git a/book/_helpers/__init__.py b/book/_helpers/__init__.py deleted file mode 100644 index f553b75..0000000 --- a/book/_helpers/__init__.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Book helpers — artifact loading and plotting utilities for Quarto chunks. - -Re-exports the most commonly used functions so chapters can write simply:: - - from book._helpers import load_json, load_parquet, PALETTE, apply_publication_style - -without dipping into submodules. For less common helpers, import directly from -``book._helpers.load_artifacts`` or ``book._helpers.plot_helpers``. -""" - -from __future__ import annotations - -from .load_artifacts import ( - CONFIGS_DIR, - DATA_DIR, - MODEL_DIR, - NOTEBOOK_IMAGE_DIR, - PUBLICATION_FIGURES_DIR, - REPO_ROOT, - REPORTS_DIR, - load_parquet, - try_load_parquet, -) -from .plot_helpers import PALETTE - -# Optional: bring publication-style helpers if present. -try: - from .plot_helpers import apply_publication_style # type: ignore -except ImportError: # pragma: no cover - apply_publication_style = None # type: ignore - -# Optional JSON loader (some chapters expect it from the unified namespace). -try: - from .load_artifacts import load_json # type: ignore -except ImportError: # pragma: no cover - import json - from pathlib import Path - - def load_json(name: str) -> dict: # type: ignore[no-redef] - """Fallback JSON loader. Resolves ``name`` against repo root or DATA_DIR.""" - candidates = [ - REPO_ROOT / name, - REPO_ROOT / f"{name}.json", - DATA_DIR / name, - DATA_DIR / f"{name}.json", - MODEL_DIR / name, - MODEL_DIR / f"{name}.json", - ] - for c in candidates: - if c.is_file(): - return json.loads(Path(c).read_text(encoding="utf-8")) - raise FileNotFoundError(name) - - -__all__ = [ - "CONFIGS_DIR", - "DATA_DIR", - "MODEL_DIR", - "NOTEBOOK_IMAGE_DIR", - "PALETTE", - "PUBLICATION_FIGURES_DIR", - "REPORTS_DIR", - "REPO_ROOT", - "apply_publication_style", - "load_json", - "load_parquet", - "try_load_parquet", -] diff --git a/book/_helpers/load_artifacts.py b/book/_helpers/load_artifacts.py deleted file mode 100644 index cd1bd67..0000000 --- a/book/_helpers/load_artifacts.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Artifact loading utilities for Quarto book code cells. - -Adapted from streamlit_app/utils.py — no Streamlit dependency. -All paths resolve relative to the repository root. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pandas as pd - -# --------------------------------------------------------------------------- -# Path resolution -# --------------------------------------------------------------------------- - -REPO_ROOT = Path(__file__).resolve().parents[2] -DATA_DIR = REPO_ROOT / "data" / "processed" -MODEL_DIR = REPO_ROOT / "models" -REPORTS_DIR = REPO_ROOT / "reports" -NOTEBOOK_IMAGE_DIR = REPORTS_DIR / "notebook_images" -PUBLICATION_FIGURES_DIR = REPORTS_DIR / "paper_material" / "figures_publication" -CONFIGS_DIR = REPO_ROOT / "configs" - -# --------------------------------------------------------------------------- -# Loaders -# --------------------------------------------------------------------------- - - -def load_parquet(name: str) -> pd.DataFrame: - """Load a parquet file from ``data/processed/.parquet``.""" - path = DATA_DIR / f"{name}.parquet" - return pd.read_parquet(path) - - -def try_load_parquet(name: str, default: pd.DataFrame | None = None) -> pd.DataFrame: - """Load parquet if available, otherwise return *default* or empty DF.""" - path = DATA_DIR / f"{name}.parquet" - if not path.exists(): - return default.copy() if isinstance(default, pd.DataFrame) else pd.DataFrame() - try: - return pd.read_parquet(path) - except Exception: - return default.copy() if isinstance(default, pd.DataFrame) else pd.DataFrame() - - -def load_json(name: str, directory: str = "data", search_models: bool = False) -> dict: - """Load a JSON artifact. - - Args: - name: File name **without** extension. - directory: ``'data'`` → ``data/processed/``, ``'models'`` → ``models/``. - """ - use_models = directory == "models" or search_models - path = MODEL_DIR / f"{name}.json" if use_models else DATA_DIR / f"{name}.json" - return json.loads(path.read_text(encoding="utf-8")) - - -def try_load_json( - name: str, - directory: str = "data", - default: dict | None = None, - search_models: bool = False, -) -> dict: - """Load JSON if available, otherwise return *default* or empty dict.""" - use_models = directory == "models" or search_models - path = MODEL_DIR / f"{name}.json" if use_models else DATA_DIR / f"{name}.json" - if not path.exists(): - return dict(default or {}) - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return dict(default or {}) - - -def try_load_report_parquet(subdir: str, name: str) -> pd.DataFrame: - """Load parquet from ``reports//.parquet``.""" - path = REPORTS_DIR / subdir / f"{name}.parquet" - if not path.exists(): - return pd.DataFrame() - try: - return pd.read_parquet(path) - except Exception: - return pd.DataFrame() - - -def load_yaml(name: str) -> dict: - """Load a YAML config from ``configs/.yaml``.""" - import yaml - - path = CONFIGS_DIR / f"{name}.yaml" - return yaml.safe_load(path.read_text(encoding="utf-8")) - - -# --------------------------------------------------------------------------- -# Formatters -# --------------------------------------------------------------------------- - - -def format_pct(value: float, decimals: int = 2) -> str: - """Format a float as percentage string, e.g. 0.9257 → '92.57%'.""" - return f"{value * 100:.{decimals}f}%" - - -def format_number(value: float | int, decimals: int = 0) -> str: - """Format a number with thousands separator.""" - if decimals == 0: - return f"{int(value):,}" - return f"{value:,.{decimals}f}" - - -def format_money(value: float, decimals: int = 0) -> str: - """Format as USD, e.g. 1003000 → '$1,003,000'.""" - if abs(value) >= 1e9: - return f"${value / 1e9:,.{decimals}f}B" - if abs(value) >= 1e6: - return f"${value / 1e6:,.{decimals}f}M" - if abs(value) >= 1e3: - return f"${value / 1e3:,.{decimals}f}K" - return f"${value:,.{decimals}f}" diff --git a/book/_helpers/plot_helpers.py b/book/_helpers/plot_helpers.py deleted file mode 100644 index d5526cc..0000000 --- a/book/_helpers/plot_helpers.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Publication-quality figure helpers for the Quarto book. - -Provides consistent styling for matplotlib and plotly figures across all -chapters. Uses a project-standard color palette and font sizes suitable -for both HTML (interactive) and PDF (static, vector) output. -""" - -from __future__ import annotations - -from pathlib import Path - -import matplotlib.pyplot as plt - -# --------------------------------------------------------------------------- -# Project palette -# --------------------------------------------------------------------------- - -PALETTE = { - "primary": "#2563EB", - "secondary": "#7C3AED", - "success": "#059669", - "warning": "#D97706", - "danger": "#DC2626", - "info": "#0891B2", - "muted": "#6B7280", - "grades": { - "A": "#059669", - "B": "#0891B2", - "C": "#2563EB", - "D": "#7C3AED", - "E": "#D97706", - "F": "#DC2626", - "G": "#BE185D", - }, -} - -GRADE_COLORS = list(PALETTE["grades"].values()) -GRADE_ORDER = list(PALETTE["grades"].keys()) - -# --------------------------------------------------------------------------- -# Matplotlib defaults -# --------------------------------------------------------------------------- - -_RC_DEFAULTS = { - "figure.figsize": (8, 5), - "figure.dpi": 150, - "savefig.dpi": 300, - "savefig.format": "svg", - "savefig.bbox": "tight", - "axes.titlesize": 13, - "axes.labelsize": 11, - "xtick.labelsize": 10, - "ytick.labelsize": 10, - "legend.fontsize": 10, - "font.family": "sans-serif", - "font.sans-serif": ["Source Sans 3", "Inter", "DejaVu Sans", "Arial"], - "axes.spines.top": False, - "axes.spines.right": False, - "axes.grid": True, - "grid.alpha": 0.3, - "axes.facecolor": "#FFFFFF", - "figure.facecolor": "#FFFFFF", - "text.color": "#111827", - "axes.labelcolor": "#111827", - "axes.titlecolor": "#111827", - "xtick.color": "#374151", - "ytick.color": "#374151", -} - - -def apply_style() -> None: - """Apply project-standard matplotlib style in-place.""" - plt.rcParams.update(_RC_DEFAULTS) - - -def get_grade_color(grade: str) -> str: - """Return the color for a credit grade letter.""" - return PALETTE["grades"].get(grade, PALETTE["muted"]) - - -def save_figure( - fig: plt.Figure, - output_path: str | Path, - *, - transparent: bool = False, - close: bool = False, -) -> Path: - """Save a matplotlib figure using project defaults. - - Defaults to SVG-first output so Quarto can render publication-quality vector - graphics in HTML and PDF with a single asset. - """ - apply_style() - path = Path(output_path) - path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(path, transparent=transparent) - if close: - plt.close(fig) - return path - - -# --------------------------------------------------------------------------- -# Convenience figure constructors -# --------------------------------------------------------------------------- - - -def bar_chart( - labels: list[str], - values: list[float], - *, - title: str = "", - xlabel: str = "", - ylabel: str = "", - color: str | list[str] | None = None, - horizontal: bool = False, - figsize: tuple[float, float] = (8, 5), -) -> plt.Figure: - """Create a styled bar chart and return the Figure.""" - apply_style() - fig, ax = plt.subplots(figsize=figsize) - c = color or PALETTE["primary"] - if horizontal: - ax.barh(labels, values, color=c) - ax.set_xlabel(ylabel) - ax.set_ylabel(xlabel) - else: - ax.bar(labels, values, color=c) - ax.set_xlabel(xlabel) - ax.set_ylabel(ylabel) - if title: - ax.set_title(title) - fig.tight_layout() - return fig - - -def metric_table_md(data: dict[str, str | float], title: str = "") -> str: - """Return a markdown table from a flat dict for inline rendering.""" - lines = [] - if title: - lines.append(f"**{title}**\n") - lines.append("| Métrica | Valor |") - lines.append("|---------|-------|") - for k, v in data.items(): - lines.append(f"| {k} | {v} |") - return "\n".join(lines) diff --git a/book/_quarto.yml b/book/_quarto.yml index 48d1714..0c2f249 100644 --- a/book/_quarto.yml +++ b/book/_quarto.yml @@ -6,23 +6,23 @@ quarto-required: ">=1.9.0" book: title: "CRPTO" - subtitle: "Conformal Robust Predict-Then-Optimize aplicado a credit risk" + subtitle: "Auditoría de identificación en ML, conformal prediction y asignación crediticia" author: "Carlos Alfredo Vergara Rojas" date: today date-format: long - description: "Libro técnico y dossier reproducible de CRPTO: conformal prediction, optimización robusta de portafolio y credit risk sobre Lending Club." + description: "Companion técnico del CRPTO activo: diseño temporal, geometría conformal binaria, asignación y evidencia reproducible sobre Lending Club." site-url: "https://eigencharlie.github.io/Paper_CRPTO/" favicon: assets/logo-small.svg - image: "https://eigencharlie.github.io/Paper_CRPTO/assets/figures/editorial/diagrama-crpto.png" - image-alt: "Diagrama maestro del pipeline CRPTO con datos, PD calibrada, conformal Mondrian, optimización robusta y política promovida." + image: "https://eigencharlie.github.io/Paper_CRPTO/assets/logo.svg" + image-alt: "Identidad visual de CRPTO." open-graph: - image: "https://eigencharlie.github.io/Paper_CRPTO/assets/figures/editorial/diagrama-crpto.png" - image-alt: "Diagrama maestro del pipeline CRPTO." + image: "https://eigencharlie.github.io/Paper_CRPTO/assets/logo.svg" + image-alt: "Identidad visual de CRPTO." locale: es_CO twitter-card: - image: "https://eigencharlie.github.io/Paper_CRPTO/assets/figures/editorial/diagrama-crpto.png" - image-alt: "Diagrama maestro del pipeline CRPTO." - card-style: summary_large_image + image: "https://eigencharlie.github.io/Paper_CRPTO/assets/logo.svg" + image-alt: "Identidad visual de CRPTO." + card-style: summary search: location: sidebar type: textbox @@ -54,46 +54,10 @@ book: right: "[GitHub](https://github.com/EigenCharlie/Paper_CRPTO) · [DagsHub](https://dagshub.com/EigenCharlie94/Paper_CRPTO) · [Quarto](https://quarto.org)" chapters: - index.qmd - - chapters/00-crpto-en-una-pagina.qmd - - chapters/00b-prologo-tesis.qmd - - part: "Manuscrito CRPTO y evidencia paper-ready" + - part: "Companion del manuscrito IJDS activo" chapters: - - chapters/01-introduccion.qmd - - chapters/02-marco-teorico.qmd - - chapters/03-metodologia.qmd - - chapters/04-resultados.qmd - - chapters/30-replicacion-multidataset.qmd - - chapters/05-discusion.qmd - chapters/06-blueprint-manuscrito.qmd - chapters/06b-guia-editorial-claims.qmd - - chapters/07-apendice-robustez.qmd - - chapters/08-ablacion-mondrian.qmd - - chapters/09-spo-regret.qmd - - chapters/10-fair-lending.qmd - - chapters/11-mrm.qmd - - chapters/12-funded-set.qmd - - chapters/13-trazabilidad.qmd - - chapters/14-release.qmd - - part: "Dossier extendido para journal y appendix" - chapters: - - chapters/15-fundamentos-riesgo-ml.qmd - - chapters/16-fundamentos-conformal-optimizacion.qmd - - chapters/17-pipeline-datos-features.qmd - - chapters/18-pd-calibracion-champion.qmd - - chapters/19-conformal-dossier.qmd - - chapters/20-portafolio-policy.qmd - - chapters/21-gobernanza-explicabilidad-dataset.qmd - - chapters/22-literatura-trazabilidad-entorno.qmd - - chapters/24-bibliografia-crpto-actualizada.qmd - - chapters/25-reviewer-map.qmd - - chapters/23-apendices-regulatorios-y-future-work.qmd - - part: "Cierre de tesis" - chapters: - - chapters/28-resultados-auxiliares.qmd - - chapters/29-ecl-incertidumbre.qmd - - chapters/26-sintesis-tesis.qmd - - chapters/27-conclusiones-tesis.qmd - - chapters/glosario.qmd - references.qmd lang: es diff --git a/book/assets/figures/editorial/ale_fico_score.png b/book/assets/figures/editorial/ale_fico_score.png deleted file mode 100644 index 72e757a..0000000 Binary files a/book/assets/figures/editorial/ale_fico_score.png and /dev/null differ diff --git a/book/assets/figures/editorial/ale_int_rate.png b/book/assets/figures/editorial/ale_int_rate.png deleted file mode 100644 index e8c0acd..0000000 Binary files a/book/assets/figures/editorial/ale_int_rate.png and /dev/null differ diff --git a/book/assets/figures/editorial/alpha_eligible_loans.png b/book/assets/figures/editorial/alpha_eligible_loans.png deleted file mode 100644 index eea8380..0000000 Binary files a/book/assets/figures/editorial/alpha_eligible_loans.png and /dev/null differ diff --git a/book/assets/figures/editorial/alpha_pareto_frontier.png b/book/assets/figures/editorial/alpha_pareto_frontier.png deleted file mode 100644 index 04c9b97..0000000 Binary files a/book/assets/figures/editorial/alpha_pareto_frontier.png and /dev/null differ diff --git a/book/assets/figures/editorial/conformal_coverage_by_grade.png b/book/assets/figures/editorial/conformal_coverage_by_grade.png deleted file mode 100644 index 847459d..0000000 Binary files a/book/assets/figures/editorial/conformal_coverage_by_grade.png and /dev/null differ diff --git a/book/assets/figures/editorial/conformal_width_vs_target.png b/book/assets/figures/editorial/conformal_width_vs_target.png deleted file mode 100644 index 8bc8000..0000000 Binary files a/book/assets/figures/editorial/conformal_width_vs_target.png and /dev/null differ diff --git a/book/assets/figures/editorial/diagrama-crpto.png b/book/assets/figures/editorial/diagrama-crpto.png deleted file mode 100644 index dd77ee4..0000000 Binary files a/book/assets/figures/editorial/diagrama-crpto.png and /dev/null differ diff --git a/book/assets/figures/editorial/pd_pr_curve.png b/book/assets/figures/editorial/pd_pr_curve.png deleted file mode 100644 index f15822c..0000000 Binary files a/book/assets/figures/editorial/pd_pr_curve.png and /dev/null differ diff --git a/book/assets/figures/editorial/pd_reliability_curve.png b/book/assets/figures/editorial/pd_reliability_curve.png deleted file mode 100644 index 490dd2b..0000000 Binary files a/book/assets/figures/editorial/pd_reliability_curve.png and /dev/null differ diff --git a/book/assets/figures/editorial/pd_roc_curve.png b/book/assets/figures/editorial/pd_roc_curve.png deleted file mode 100644 index 7a998f0..0000000 Binary files a/book/assets/figures/editorial/pd_roc_curve.png and /dev/null differ diff --git a/book/assets/figures/editorial/shap_family_mass.png b/book/assets/figures/editorial/shap_family_mass.png deleted file mode 100644 index 033e7e9..0000000 Binary files a/book/assets/figures/editorial/shap_family_mass.png and /dev/null differ diff --git a/book/assets/figures/editorial/uncertainty_baselines_tradeoff.png b/book/assets/figures/editorial/uncertainty_baselines_tradeoff.png deleted file mode 100644 index be26f11..0000000 Binary files a/book/assets/figures/editorial/uncertainty_baselines_tradeoff.png and /dev/null differ diff --git a/book/assets/figures/notebooks/ale_curves.png b/book/assets/figures/notebooks/ale_curves.png deleted file mode 100644 index e7b344f..0000000 Binary files a/book/assets/figures/notebooks/ale_curves.png and /dev/null differ diff --git a/book/assets/figures/notebooks/calibration_curves.png b/book/assets/figures/notebooks/calibration_curves.png deleted file mode 100644 index 07921f1..0000000 Binary files a/book/assets/figures/notebooks/calibration_curves.png and /dev/null differ diff --git a/book/assets/figures/notebooks/default_rate_by_grade.png b/book/assets/figures/notebooks/default_rate_by_grade.png deleted file mode 100644 index e432590..0000000 Binary files a/book/assets/figures/notebooks/default_rate_by_grade.png and /dev/null differ diff --git a/book/assets/figures/notebooks/default_rate_grade_x_term.png b/book/assets/figures/notebooks/default_rate_grade_x_term.png deleted file mode 100644 index d24a309..0000000 Binary files a/book/assets/figures/notebooks/default_rate_grade_x_term.png and /dev/null differ diff --git a/book/assets/figures/notebooks/feature_family_decomposition.png b/book/assets/figures/notebooks/feature_family_decomposition.png deleted file mode 100644 index 877da00..0000000 Binary files a/book/assets/figures/notebooks/feature_family_decomposition.png and /dev/null differ diff --git a/book/assets/figures/notebooks/interval_width_distribution.png b/book/assets/figures/notebooks/interval_width_distribution.png deleted file mode 100644 index 9c919e8..0000000 Binary files a/book/assets/figures/notebooks/interval_width_distribution.png and /dev/null differ diff --git a/book/assets/figures/notebooks/iv_ranking.png b/book/assets/figures/notebooks/iv_ranking.png deleted file mode 100644 index 4b1afcb..0000000 Binary files a/book/assets/figures/notebooks/iv_ranking.png and /dev/null differ diff --git a/book/assets/figures/notebooks/monotonicity_verification.png b/book/assets/figures/notebooks/monotonicity_verification.png deleted file mode 100644 index 1e57410..0000000 Binary files a/book/assets/figures/notebooks/monotonicity_verification.png and /dev/null differ diff --git a/book/assets/figures/notebooks/portfolio_time_series_overview.png b/book/assets/figures/notebooks/portfolio_time_series_overview.png deleted file mode 100644 index 6d0b440..0000000 Binary files a/book/assets/figures/notebooks/portfolio_time_series_overview.png and /dev/null differ diff --git a/book/assets/figures/notebooks/roc_curves.png b/book/assets/figures/notebooks/roc_curves.png deleted file mode 100644 index a8eeda4..0000000 Binary files a/book/assets/figures/notebooks/roc_curves.png and /dev/null differ diff --git a/book/assets/figures/notebooks/shap_bar_importance.png b/book/assets/figures/notebooks/shap_bar_importance.png deleted file mode 100644 index e664fa4..0000000 Binary files a/book/assets/figures/notebooks/shap_bar_importance.png and /dev/null differ diff --git a/book/assets/figures/notebooks/shap_beeswarm.png b/book/assets/figures/notebooks/shap_beeswarm.png deleted file mode 100644 index d91f68d..0000000 Binary files a/book/assets/figures/notebooks/shap_beeswarm.png and /dev/null differ diff --git a/book/assets/figures/notebooks/shap_vs_permutation.png b/book/assets/figures/notebooks/shap_vs_permutation.png deleted file mode 100644 index 3ed9cf4..0000000 Binary files a/book/assets/figures/notebooks/shap_vs_permutation.png and /dev/null differ diff --git a/book/assets/figures/notebooks/target_distribution.png b/book/assets/figures/notebooks/target_distribution.png deleted file mode 100644 index 3438d02..0000000 Binary files a/book/assets/figures/notebooks/target_distribution.png and /dev/null differ diff --git a/book/assets/figures/notebooks/uncertainty_sets.png b/book/assets/figures/notebooks/uncertainty_sets.png deleted file mode 100644 index d44f565..0000000 Binary files a/book/assets/figures/notebooks/uncertainty_sets.png and /dev/null differ diff --git a/book/assets/figures/notebooks/woe_binning_top6.png b/book/assets/figures/notebooks/woe_binning_top6.png deleted file mode 100644 index 9aea609..0000000 Binary files a/book/assets/figures/notebooks/woe_binning_top6.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig10_cqr_per_grade.pdf b/book/assets/figures/publication/crpto_fig10_cqr_per_grade.pdf deleted file mode 100644 index 778abfb..0000000 Binary files a/book/assets/figures/publication/crpto_fig10_cqr_per_grade.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig10_cqr_per_grade.png b/book/assets/figures/publication/crpto_fig10_cqr_per_grade.png deleted file mode 100644 index 7667c34..0000000 Binary files a/book/assets/figures/publication/crpto_fig10_cqr_per_grade.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig11_crpto_stability.pdf b/book/assets/figures/publication/crpto_fig11_crpto_stability.pdf deleted file mode 100644 index e089e38..0000000 Binary files a/book/assets/figures/publication/crpto_fig11_crpto_stability.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig11_crpto_stability.png b/book/assets/figures/publication/crpto_fig11_crpto_stability.png deleted file mode 100644 index f710579..0000000 Binary files a/book/assets/figures/publication/crpto_fig11_crpto_stability.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf b/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf deleted file mode 100644 index 106f664..0000000 Binary files a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.png b/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.png deleted file mode 100644 index 786510d..0000000 Binary files a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf b/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf deleted file mode 100644 index 370e4a9..0000000 Binary files a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.png b/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.png deleted file mode 100644 index 1a8c519..0000000 Binary files a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf b/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf deleted file mode 100644 index 28f5f57..0000000 Binary files a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.png b/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.png deleted file mode 100644 index 23c63e3..0000000 Binary files a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf b/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf deleted file mode 100644 index 7e7d9de..0000000 Binary files a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.png b/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.png deleted file mode 100644 index ffe2cb2..0000000 Binary files a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.pdf b/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.pdf deleted file mode 100644 index eadee36..0000000 Binary files a/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.png b/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.png deleted file mode 100644 index 8c33ec0..0000000 Binary files a/book/assets/figures/publication/crpto_fig16_tail_risk_frontier.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.pdf b/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.pdf deleted file mode 100644 index 5d15094..0000000 Binary files a/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.png b/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.png deleted file mode 100644 index 69d2b92..0000000 Binary files a/book/assets/figures/publication/crpto_fig17_tail_risk_lgd.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.pdf b/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.pdf deleted file mode 100644 index b97d23e..0000000 Binary files a/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.png b/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.png deleted file mode 100644 index 6b254cd..0000000 Binary files a/book/assets/figures/publication/crpto_fig18_tail_constrained_frontier.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig19_online_coverage_aci.pdf b/book/assets/figures/publication/crpto_fig19_online_coverage_aci.pdf deleted file mode 100644 index 2da7d2d..0000000 Binary files a/book/assets/figures/publication/crpto_fig19_online_coverage_aci.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig19_online_coverage_aci.png b/book/assets/figures/publication/crpto_fig19_online_coverage_aci.png deleted file mode 100644 index 4b9b09d..0000000 Binary files a/book/assets/figures/publication/crpto_fig19_online_coverage_aci.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf b/book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf deleted file mode 100644 index 8084242..0000000 Binary files a/book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig1_journal_pipeline.png b/book/assets/figures/publication/crpto_fig1_journal_pipeline.png deleted file mode 100644 index 2e737d9..0000000 Binary files a/book/assets/figures/publication/crpto_fig1_journal_pipeline.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig1_journal_pipeline.svg b/book/assets/figures/publication/crpto_fig1_journal_pipeline.svg deleted file mode 100644 index f86b5ac..0000000 --- a/book/assets/figures/publication/crpto_fig1_journal_pipeline.svg +++ /dev/null @@ -1,167 +0,0 @@ - - - - - - - - -1 - -2 - -3 - -4 - -Frozen calibrated -PD - - -Mondrian conformal -intervals - - -Robust portfolio -optimization - - -Promoted auditable -policy - - - - - - - -u_i(α) → robust PD input - - - - - - - - - - -0.0 - -0.5 - -1.0 - -0.0 - -0.5 - -1.0 - - - - - - - - - -Predicted PD -Observed -default rate -Lending Club OOT: n = 276,869 -CatBoost + Venn-Abers -AUC 0.714, ECE 0.007 - - - -0.0 - -0.1 - -0.2 - -0.3 - -1 - -3 - -5 - -7 - -9 - -10 - - - - - - - - - - - - - - -point PD - -90% band -PD -Score decile -score-decile partition -90% coverage: 0.9297 -min-group coverage: 0.9190 - - -max Σ_i x_i r_i -s.t. -budget -robust PD cap -x_i ≥ 0 - - - - - - - -$ - - - - - - - -$1M budget -point PD + conformal upper risk -τ = 0.175, γ = 0.45 - - - - - - - - - - - - - - -robust return: $170.5K -V(α = 0.01) = 0.028875 -Γ_CP = 0.187987 -exact pass; 45/45 safe policies - -Prediction uncertainty becomes an auditable robust portfolio constraint. - diff --git a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf b/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf deleted file mode 100644 index 9162529..0000000 Binary files a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.png b/book/assets/figures/publication/crpto_fig20_bound_claim_layers.png deleted file mode 100644 index 6b9ad48..0000000 Binary files a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig21_end_to_end_arc.pdf b/book/assets/figures/publication/crpto_fig21_end_to_end_arc.pdf deleted file mode 100644 index 803b76b..0000000 Binary files a/book/assets/figures/publication/crpto_fig21_end_to_end_arc.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig21_end_to_end_arc.png b/book/assets/figures/publication/crpto_fig21_end_to_end_arc.png deleted file mode 100644 index 7166b5a..0000000 Binary files a/book/assets/figures/publication/crpto_fig21_end_to_end_arc.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig22_external_replication.pdf b/book/assets/figures/publication/crpto_fig22_external_replication.pdf deleted file mode 100644 index 85dc8fe..0000000 Binary files a/book/assets/figures/publication/crpto_fig22_external_replication.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig22_external_replication.png b/book/assets/figures/publication/crpto_fig22_external_replication.png deleted file mode 100644 index 6847fa5..0000000 Binary files a/book/assets/figures/publication/crpto_fig22_external_replication.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.pdf b/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.pdf deleted file mode 100644 index 2f739af..0000000 Binary files a/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.png b/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.png deleted file mode 100644 index 665d390..0000000 Binary files a/book/assets/figures/publication/crpto_fig23_external_candidate_sensitivity.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.pdf b/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.pdf deleted file mode 100644 index 9ff4e1b..0000000 Binary files a/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.png b/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.png deleted file mode 100644 index 70cb668..0000000 Binary files a/book/assets/figures/publication/crpto_fig24_freddie_all_candidate_certificate.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf deleted file mode 100644 index 2c0bf99..0000000 Binary files a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png deleted file mode 100644 index 966ab67..0000000 Binary files a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.pdf b/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.pdf deleted file mode 100644 index 97a81ec..0000000 Binary files a/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.png b/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.png deleted file mode 100644 index d7840fd..0000000 Binary files a/book/assets/figures/publication/crpto_fig7_uncertainty_baselines.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig8_alpha_pareto.pdf b/book/assets/figures/publication/crpto_fig8_alpha_pareto.pdf deleted file mode 100644 index 5b8f044..0000000 Binary files a/book/assets/figures/publication/crpto_fig8_alpha_pareto.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig8_alpha_pareto.png b/book/assets/figures/publication/crpto_fig8_alpha_pareto.png deleted file mode 100644 index a473826..0000000 Binary files a/book/assets/figures/publication/crpto_fig8_alpha_pareto.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig9_spo_regret.pdf b/book/assets/figures/publication/crpto_fig9_spo_regret.pdf deleted file mode 100644 index ff10938..0000000 Binary files a/book/assets/figures/publication/crpto_fig9_spo_regret.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig9_spo_regret.png b/book/assets/figures/publication/crpto_fig9_spo_regret.png deleted file mode 100644 index 7c1ce3e..0000000 Binary files a/book/assets/figures/publication/crpto_fig9_spo_regret.png and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.pdf b/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.pdf deleted file mode 100644 index b5ba800..0000000 Binary files a/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.pdf and /dev/null differ diff --git a/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.png b/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.png deleted file mode 100644 index 770403c..0000000 Binary files a/book/assets/figures/publication/crpto_fig_alpha_gamma_bound.png and /dev/null differ diff --git a/book/assets/logo-small.svg b/book/assets/logo-small.svg deleted file mode 100644 index 83db397..0000000 --- a/book/assets/logo-small.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - CRPTO - - CR - diff --git a/book/assets/logo.svg b/book/assets/logo.svg deleted file mode 100644 index ad279a9..0000000 --- a/book/assets/logo.svg +++ /dev/null @@ -1,18 +0,0 @@ - - - CRPTO - CRPTO — Conformal Robust Predict-Then-Optimize - - - - - - - - - CRPTO - - CONFORMAL ROBUST · PREDICT-THEN-OPTIMIZE - - - diff --git a/book/chapters/01-introduccion.qmd b/book/chapters/01-introduccion.qmd index 6b6bf54..bcc9979 100644 --- a/book/chapters/01-introduccion.qmd +++ b/book/chapters/01-introduccion.qmd @@ -2,7 +2,7 @@ La mayoría de los pipelines de crédito siguen una receta implícita: predecir una PD puntual y optimizar decisiones como si esa predicción fuera exacta. El problema es conocido y profundo: pequeñas desviaciones de calibración o cambios de régimen pueden inducir políticas frágiles, justo en la capa donde se asigna capital. -Este paper propone una respuesta operativa y científicamente defendible: usar intervalos conformales como conjuntos de incertidumbre para optimización robusta de portafolio. El foco no es "otro modelo de scoring", sino un puente explícito entre *uncertainty quantification* y *predict-then-optimize* que denominamos **Conformal Robust Predict-then-Optimize** (CRPTO). La industria suele colapsar predicciones en valores esperados para la optimización; nosotros resolvemos esto inyectando los intervalos de Conformal Prediction directamente como conjuntos de incertidumbre del optimizador. +Este paper estudia cómo auditar un intervalo conformal cuando su endpoint entra en un optimizador de portafolio. El foco no es "otro modelo de scoring", sino el contrato entre *uncertainty quantification*, comparadores y *predict-then-optimize*, que denominamos **Conformal Risk-Aware Predict-then-Optimize** (CRPTO). El intervalo aporta un score de decisión auditable; no hereda automáticamente cobertura temporal, validez sobre el conjunto financiado ni una garantía de robustez del portafolio. ### Contexto en la literatura {#sec-crpto-lit-context} @@ -55,7 +55,7 @@ El paper actual es fuerte como **CRPTO post-hoc auditable**: toma una PD calibra ### Contribuciones del paper -1. Formaliza el enfoque **Conformal Robust Predict-then-Optimize** (CRPTO), estableciendo un puente directo entre la garantía de cobertura/riesgo conformal y el presupuesto de robustez de [@bertsimas2004] para selección de portafolio crediticio. +1. Formaliza el enfoque **Conformal Risk-Aware Predict-then-Optimize** (CRPTO), separando cobertura candidata, geometría score-cap, comparadores y resultados financiados en la selección de portafolio crediticio. 2. Compara CRPTO contra baselines alternativos de incertidumbre (bootstrap, BMA) y contra decision-focused learning (SPO+), con métricas que abarcan cobertura, anchura, regret económico y auditabilidad regulatoria. 3. Cuantifica la frontera entre cobertura, ancho y retorno económico bajo una misma trazabilidad experimental, proporcionando una herramienta operativa para gestores de portafolio. 4. Demuestra que la ventaja competitiva de CRPTO frente a SPO+ no es el regret (donde SPO+ es superior por diseño) sino la **trazabilidad**: cobertura verificable por grupo, auditabilidad ante SR 11-7, y respuesta automática ante cambios de régimen vía ensanchamiento natural de intervalos. diff --git a/book/chapters/03-metodologia.qmd b/book/chapters/03-metodologia.qmd index 7ac96ac..a6abe89 100644 --- a/book/chapters/03-metodologia.qmd +++ b/book/chapters/03-metodologia.qmd @@ -51,15 +51,19 @@ pd.DataFrame( ``` ::: {.callout-note} -## Por qué WOE+OptBinning estabiliza inputs para conformal Mondrian - -CatBoost soporta features categóricas nativamente, así que el champion usa los valores crudos sin pre-binning. Sin embargo, el pipeline pre-conformal del proyecto **sí** computa WOE/IV mediante OptBinning para tres propósitos auditables: - -1. **Diagnóstico monotónico**. Los IV scores y los binarios de WOE permiten verificar que cada feature monotónica del champion respeta la dirección regulatoria (mayor `dti` → mayor PD, etc.) sin ambigüedad de modelo. -2. **Estabilidad por grupo conformal**. Los grupos Mondrian del winner (`score_decile_mondrian`) se construyen sobre el score calibrado, no sobre features crudas. Pero la calibración Venn-Abers del score depende de que la transformación PD → score sea monotónica y bien comportada, y eso se verifica con los binarios WOE: si las celdas de WOE son monótonas y los IV están dentro del rango estable, la calibración inductiva es bien-definida. -3. **Interpretabilidad regulatoria**. Para el comité de riesgo y el reporte MRM (@sec-p1-mrm-approval), los WOE bins de cada feature top dan una explicación `(rango → riesgo)` que un revisor puede inspeccionar directamente, mientras que la importancia CatBoost cruda es opaca. - -Las tablas WOE/IV detalladas viven en `notebooks/02_feature_engineering.ipynb` y los chapters 05a-05c del libro principal. +## Rol correcto de WOE/IV en la evidencia actual + +CatBoost soporta categorías nativamente y el score primario activo no usa bins +WOE. El pipeline histórico conserva encoders WOE train-only por cuantiles; no +son OptBinning. El audit IJDS añade dos scorecards OptBinning aislados, uno con +señales de grade/pricing y otro borrower-only, para probar si el fallo temporal +de cobertura depende de CatBoost o de señales de la plataforma. + +Los 45 problemas de binning alcanzan estado `OPTIMAL`, pero los scorecards no +entran al optimizador ni compiten por promoción OOT. WOE/IV aporta +interpretabilidad y un diagnóstico de dependencia de especificación; no prueba +estabilidad conformal, no valida la calibración y no constituye la novedad +central del paper. ::: #### Calibración Venn-Abers: por qué se eligió frente a Platt, Isotonic y Beta {#sec-p1-vennabers} diff --git a/book/chapters/05-discusion.qmd b/book/chapters/05-discusion.qmd index 5fe0588..8cbe047 100644 --- a/book/chapters/05-discusion.qmd +++ b/book/chapters/05-discusion.qmd @@ -180,7 +180,7 @@ La mejor continuacion no es expandir alcance indiscriminadamente, sino endurecer ### Conclusión general {#sec-crpto-conclusion} -En síntesis, el CRPTO demuestra que el enfoque **Conformal Robust Predict-then-Optimize** (CRPTO) puede trascender el rol tradicional de los intervalos conformales como herramienta diagnostica para convertirlos en un insumo operativo de decisión financiera. La combinación de una PD monotónica calibrada, intervalos con cobertura verificable y un programa lineal robusto parametrizado por esos intervalos produce un sistema que es simultáneamente más defensible ante auditoría y más sensible ante cambios en el régimen de riesgo. +En síntesis, **Conformal Risk-Aware Predict-then-Optimize** (CRPTO) convierte el uso de intervalos conformales en un objeto de auditoría de decisión. El resultado activo no certifica una política robusta: muestra que la cobertura candidata falla bajo transporte temporal y que la dirección del portafolio depende del timing residual, del comparador y de restricciones operacionales vinculantes. Su aporte es hacer esas dependencias reproducibles y explícitas. La comparación con SPO+ ilumina la contribución real: CRPTO no compite en regret (donde SPO+ es superior por diseño: `49.09%` en A19/Figura 15 y `48.51%` en el closeout PyEPO pareado) sino en **auditabilidad y gobernanza** --- dimensiones que la literatura de decision-focused learning ha tratado como secundarias pero que son centrales en crédito regulado. Aunque las limitaciones identificadas --- dataset estático, clase de activo única, holgura de la cota teórica --- exigen cautela en la generalización, la evidencia empírica presentada sostiene algo más fuerte que en la versión previa: el teorema ya no está solo “bien armado”, sino sincronizado con una policy promovida exacta y con una región robusta completa en el OOT total. Con eso, el paper queda bien posicionado como pieza bandera de la agenda de maestría. diff --git a/book/chapters/06-blueprint-manuscrito.qmd b/book/chapters/06-blueprint-manuscrito.qmd index 70b1ce4..0a1a882 100644 --- a/book/chapters/06-blueprint-manuscrito.qmd +++ b/book/chapters/06-blueprint-manuscrito.qmd @@ -5,283 +5,214 @@ page-layout: article ## Blueprint Journal-Ready {#sec-p1-manuscript-blueprint} -Esta página traduce el dossier largo del libro en una arquitectura de paper. -No reemplaza las secciones metodológicas previas: las comprime en la forma del -manuscrito objetivo para **INFORMS Journal on Data Science (IJDS)**. La regla -editorial es mantener una tesis clara: **CRPTO no compite por ganar AUC; compite -por convertir incertidumbre conformal en una decisión de portafolio auditable**. - -### Venue objetivo - -El venue está decidido: el manuscrito se escribe para **INFORMS Journal on Data -Science (IJDS)**, cuyo encaje es directo --- data science reproducible al -servicio de una decisión de crédito --- y queda versionado en -`configs/crpto_publication_targets.yaml` y desarrollado en -`docs/research/crpto_publication_strategy_2026-05-12.md`. Lo que IJDS pide ya -existe en el proyecto: pipeline reproducible (DVC/DagsHub/MLflow, companion -Quarto, cadena PD -> CP -> LP), tablas y figuras regenerables, y una -contribución de decisión auditable. El cuerpo activo ya vive en -`paper/CRPTO_ijds.qmd`; el trabajo editorial restante es mantenerlo quirurgico, -anonimo y sincronizado con el online supplement. - -El borrador vigente se escribe entonces como **paper IJDS**: cuerpo anonimo, -compacto, con evidencia principal suficiente para el revisor y appendix online -separado para robustez extendida. Si una revisión interna concluyera que la -historia lee más como OR aplicada que como data science, *European Journal of -Operational Research* (EJOR) queda como único pivote de respaldo, sin cambiar -champion ni artefactos. - -Los esqueletos activos son `paper/CRPTO_ijds.qmd` y `paper/supplement_ijds.qmd`. -El archivo `paper/CRPTO.qmd` queda como entrada generica, no como formato final. +Esta página describe el manuscrito IJDS activo. No es una lista de versiones ni +un registro de resultados atractivos: fija la pregunta, el estimand, la +secuencia argumental y la fuente de cada afirmación. -### Abstract operativo +### Tesis editorial + +CRPTO estudia una cadena integrada de aprendizaje, incertidumbre conformal y +asignación. Su aporte no es una policy ganadora. Es una auditoría de +identificación que muestra cuándo una propiedad del score no transporta al +periodo OOT y cuándo la comparación downstream depende del ruler que define +stringency común. -Un abstract posible, todavía largo para paper, sería: +El encaje con **INFORMS Journal on Data Science (IJDS)** descansa en cuatro +rasgos: una pregunta de decisión, un diseño temporal auditable, teoría que +separa objetos predictivos y de optimización, y evidencia reproducible que +incluye resultados negativos. El body es `paper/CRPTO_ijds.qmd`; el detalle +matemático y empírico vive en `paper/supplement_ijds.qmd`. -> We study credit portfolio selection when default probabilities are calibrated -> but decisión uncertainty remains material. We propose Conformal Robust -> Predict-then-Optimize (CRPTO), a post-hoc auditable framework that maps -> Mondrian conformal prediction intervals into robust portfolio constraints. -> On a 276,869-loan out-of-time Lending Club evaluation, the selected pool93 body -> point earns `$184.8K` on a `$1M` budget while passing the declared eight-level -> alpha grid (`V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, exact -> Markov threshold `0.345084`, zero realized risk-tolerance excess). The consolidated finite policy-grid frontier contains 50,010 -> deduplicated semantic policies, with 27,508 all-alpha above-floor policies, -> showing that the result is not a single-point artifact. The contribution is -> not higher AUC, but a reproducible bridge from calibrated probabilistic -> learning to auditable robust credit decisions. +El estudio es retrospectivo y usa un archivo previamente inspeccionado. Cada +lineage activo fue protocol-locked antes de su outcome join correspondiente y +se reporta completo, pero esto no es preregistration, confirmación prospectiva +ni un holdout intacto. -Este abstract ya contiene la diferencia central: el valor está en la decisión -robusta verificable, no en otro leaderboard predictivo. +### Abstract operativo -### Claims numerados del paper +> We audit identification at the interface of credit-risk prediction, binary +> conformal intervals, and constrained monthly allocation. From a 2,925,493-row +> Lending Club archive, a status-independent protocol retains all 640,543 +> eligible 36-month loans and evaluates 376,890 primary OOT candidates. Five +> calibrated score specifications and all eight eligible six-month residual +> windows place the sharp all-candidate coverage upper bound below 0.90. A +> prevalence crossing in one fixed score stratum coincides with a discontinuous +> change in binary absolute-residual geometry. Downstream, objective-matched and +> normalized-score rulers produce different realized contrast directions, and +> exact common-outcome envelopes expose unresolved endpoints and comparator +> support. Across a complete 36-scenario budget--concentration--LGD grid, every +> scenario retains adverse default and miscoverage cells, but none is uniformly +> favorable or adverse. No model, gamma, ruler, coordinate, scenario, or policy is selected. CRPTO's +> contribution is a reproducible identification audit of the +> machine-learning--conformal--optimization handoff. + +### Claims editoriales activos + +Los identificadores I1--I8 son claves de escritura, no hipótesis independientes +ni nombres de comparadores. ::: {#tbl-crpto-numbered-claims} -| Claim | Formulación para paper | Estatus | Evidencia primaria | +| Clave | Formulación defendible | Evidencia primaria | Límite | |---|---|---|---| -| C1 | La base PD calibrada es suficientemente estable para alimentar incertidumbre conformal. | Empírico | AUC `0.7139`, Brier `0.1544`, ECE `0.0070`, tests de calibración | -| C2 | La capa conformal Mondrian produce cobertura útil y trazable para decisión. | Empírico-conformal | Coverage 90 `92.97%`, min group coverage `91.90%`, Winkler `1.111` | -| C3 | El intervalo conformal se puede mapear a un conjunto de incertidumbre usado por un LP robusto. | Metodológico | Definiciones `u_i(alpha)`, `Gamma_CP`, policy modes | -| C4 | El bound controla no-cobertura ponderada del funded set bajo supuestos distribution-free. | Teórico | `thm-conformal-feasibility`, Markov, `V <= sqrt(alpha)` | -| C5 | La policy oficial es el punto pool93 body/default, no el endpoint de máximo retorno. | Empírico-editorial | A35/A40, retorno `$184.8K`, umbral Markov `0.345084`, `alpha_grid_pass=8/8` | -| C6 | La frontera finita A35 muestra que el resultado no es un punto aislado. | Empírico | 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | -| C7 | La evidencia CRPTO y journal-package muestra robustez adicional sin cambiar la dirección del paper. | Robustez | A3--A36, Figuras 12--25 | - -Claims numerados para transformar el libro en manuscrito. +| I1 | El diseño usa toda la población elegible bajo fechas, plazo, esquema y observabilidad declarados. | 2,925,493 filas crudas; 640,543 elegibles; sampling `none`. | No todos los registros comparten target u horizonte. | +| I2 | La cobertura candidata no transporta bajo las cinco especificaciones y ocho ventanas declaradas. | 40/40 upper bounds por debajo de 0.90. | No prueba invalidez conformal universal ni selected-set coverage. | +| I3 | La geometría del residual absoluto binario es sensible a prevalencia. | En CatBoost stratum 2, W7--W8 cambia prevalencia 0.101703→0.097147 y cuantil 0.888435→0.111801. | Es un mecanismo compatible con el dato, no atribución causal completa. | +| I4 | Score, ruler y coordenada definen conjuntamente la comparación de portafolios. | Dos rulers, tres coordenadas interiores, ocho ventanas y 48 contrastes de ventana. | Ningún ruler es neutral ni seleccionado por outcomes. | +| I5 | El endpoint reconstruido cambia la interpretación de direcciones previamente aparentes. | 376,890 candidatos; 364,814 resueltos y 12,076 no resueltos. | El archivo no es un snapshot administrativo verificado. | +| I6 | Los bounds sharp y el exact support cuantifican identificación parcial sobre outcomes y caps declarados. | 216 broad-support envelopes cruzan cero; terminal default cruza cero en 72/72 celdas de development support. | No son intervalos de confianza ni una frontera continua universal. | +| I7 | La ausencia de dirección universal sobrevive una sensibilidad estructural completa. | 36/36 escenarios; mínimo 17 celdas de default y 21 de miscoverage adversas por escenario; 0 escenarios uniformemente favorables o adversos. | No identifica un escenario ganador ni prueba daño universal. | +| I8 | La evidencia es reproducible y outcome-isolated. | Registry hash-locked, freezes/evaluations DVC, manifest paper-facing y tests de publicación. | Reproducibilidad no convierte el estudio en prospectivo. | + +: Claims editoriales del manuscrito activo. ::: -La secuencia C1--C7 también evita una tentacion peligrosa: contar la historia -como si el champion apareciera por magia al final. El paper debe mostrar que el -sistema aprendio a mover el cuello de botella desde predicción, hacia -incertidumbre, hacia composición del funded set. - -### Estructura del manuscrito +### Arquitectura argumental ::: {#tbl-crpto-paper-outline} -| Sección | Papel en el manuscrito | Material fuente | +| Sección | Pregunta que resuelve | Material esencial | |---|---|---| -| 1. Introduction | Gap, pregunta, contribuciones C1--C7 | 14a, 14f, esta página | -| 2. Related work | CP/CRC/RCPS, RO, Predict-then-Calibrate, DFL/SPO+ | 14a, 14f, referencias [1]--[17] | -| 3. Method | PD calibrada -> CP Mondrian -> uncertainty set -> LP robusto | 14b, 14c | -| 4. Theory | `Gamma_CP`, `V`, Markov, tightening condicional como appendix | 14b, appendix condicional | -| 5. Experimental design | Lending Club OOT, artifacts, compatible leaderboards | 14c, docs de sincronizacion | -| 6. Results | Champion, región robusta, comparadores, SPO+ y alpha frontier | 14d | -| 7. Robustness | nested/temporal holdout, funded set, shift, tail risk, re-opt de cola, multi-distribución/online y réplica externa | 14d, 14h, A3--A36 | -| 8. Discussion | Limitaciones, governance, future work sin overclaim | 14e | -| Appendix | Pruebas, tablas A3--A36, reproducibility checklist | 14b, 14h | - -Skeleton recomendado del paper. +| 1. Introduction | ¿Qué se rompe al inferir una policy a partir de cobertura candidata? | Gap, contribuciones I1--I8 y límites. | +| 2. Related Work | ¿Qué aportan y qué no transfieren los vecinos más cercanos? | Decision quality, CP bajo shift, conformal RO, crédito y madurez. | +| 3. Data and Locked Evaluation Design | ¿Cuál era la información físicamente disponible? | Universo status-independent, roles temporales, outcome isolation y endpoint. | +| 4. Method | ¿Cómo se construyen scores, intervalos, payoff y LP? | CatBoost primario, cuatro controles, Mondrian binario y asignación mensual. | +| 5. Audit Theory and Estimands | ¿Qué puede ordenarse sin observar outcomes? | Equivalencia afín, nesting, C2, geometría binaria y bounds sharp. | +| 6. Results | ¿Transporta cobertura y es invariante la dirección downstream? | 40 celdas, W7--W8, two-ruler, estructura y exact support. | +| 7. Discussion | ¿Qué aprende un sistema decision-focused de este fallo de transporte? | Estimand, información y separación de claims. | +| 8. Limitations | ¿Qué diseño sería necesario para afirmar más? | Snapshot, censura, selected set, cash flows y validez externa. | +| 9. Reproducibility | ¿Cómo se reconstruye cada cifra? | Registry, DVC, evidence manifest, tablas, figuras y tests. | +| 10. Conclusion | ¿Cuál es la contribución mínima que sobrevive? | Auditoría de identificación, sin policy winner. | + +: Arquitectura del paper IJDS. ::: -### Page-budget ledger +### Page-budget de trabajo -IJDS limita el manuscrito inicial a 25 páginas. El ledger fija páginas objetivo -por sección y la regla de compresión. Cualquier subsection que no sobreviva esta -tabla se mueve al online supplement o a la tesis. +No hay freeze de submission. Este ledger es una restricción editorial móvil, +útil para evitar que provenance y pruebas largas desplacen el argumento. ::: {#tbl-crpto-page-budget} -| Sección | Páginas objetivo | Regla de compresión | +| Bloque | Páginas objetivo | Regla | |---|---:|---| -| 1. Introduction | 2.0 | Gap, pregunta, contribuciones y preview del resultado. | -| 2. Related Work | 3.0 | Solo vecinos necesarios: CP/CRC, RO, PtO/DFL, credit governance. | -| 3. Method | 5.0 | Definiciones, pipeline, LP robusto y notación reutilizable. | -| 4. Theory | 3.0 | `Gamma_CP`, `V`, bound principal; pruebas largas al supplement. | -| 5. Experimental Design | 2.5 | Dataset, split OOT, leakage controls y artifacts. | -| 6. Results | 5.0 | Champion, región robusta, ablation y comparadores. | -| 7. Robustness | 1.5 | Compacta; el grueso A3--A36 vive en el supplement. | -| 8. Discussion | 2.0 | Implicaciones, límites, MRM/fairness proxy y futuro. | -| Buffer editorial | 1.0 | Títulos, figuras, transiciones y ajustes finales. | - -Page-budget ledger del cuerpo IJDS (suma objetivo 25 páginas). +| Introduction | 2.0 | Pregunta, gap, contribuciones y resultado principal. | +| Related Work | 2.5 | Solo vecinos que cambian el límite de claim. | +| Data and Design | 3.0 | Dar espacio a madurez, endpoint e information boundary. | +| Method | 4.0 | Una notación, un payoff y un LP. | +| Theory | 3.0 | Enunciados en body; pruebas completas al supplement. | +| Results | 5.0 | Cobertura, mecanismo, rulers y support. | +| Discussion + Limitations | 3.0 | Implicación y frontera, sin repetir resultados. | +| Reproducibility | 1.0 | Contrato ejecutable, no inventario de archivos. | +| Buffer | 1.0 | Figuras, tablas y transiciones. | + +: Page-budget de trabajo previo al freeze. ::: -### Figuras y tablas finales - -El libro puede guardar más material que el paper. Para el manuscrito, la -selección sugerida es: +### Figuras y tablas ::: {#tbl-crpto-manuscript-figures} -| Elemento | Ubicación probable | Razon | Artefacto | +| Elemento | Ubicación | Función | Fuente | |---|---|---|---| -| Figura 1: pipeline CRPTO | Cuerpo | Explica el aporte en una sola vista IJDS | `crpto_fig1_journal_pipeline.png` | -| Figura bound: pila de claim | Teoría | Separa endpoint conformal, identidad determinística, supuesto ponderado y certificado exacto | `crpto_fig20_bound_claim_layers.png` | -| Figura 2: alpha -> `Gamma_CP` -> funded set | teoría/método | Une parámetro conformal y decisión | `crpto_fig13_alpha_gamma_funded_set.png` | -| Tablas A35/A40 | Resultados | Muestran la frontera finita y el baseline point-PD emparejado | `crpto_tableA35_pool93_ijds_frontier.csv`, `crpto_tableA40_pool93_point_baseline.csv` | -| Tabla 1: métricas core | Resultados | Fija PD, CP y portfolio sin mezclar familias | `crpto_table0_key_metrics.csv` | -| Tabla 2: champion y comparadores | Resultados | Economic vs theorem-tight vs balanced | `crpto_table1_champion_policy.csv` y A13 | -| Tabla 3: robustez P1 | Apéndice corto | Post-selección, temporal, selector, shift | A3--A11 | -| Tabla 4: robustness journal | Appendix largo | Tail risk, bootstrap, dependencia, LGD/caps, regret-auditability, re-opt de cola, multi-distribución, online y réplica externa | A12--A34 | - -Selección de figuras y tablas para el manuscrito. +| Figura 1 | Body | Cobertura resuelta y sharp bounds por ventana. | `crpto_ijds_v4_fig1_coverage` | +| Figura 2 | Body | Prevalencia y cuantil de stratum 2 en W1--W8. | `crpto_ijds_v4_fig2_phase_transition` | +| Tabla de controles | Body | Cinco modelos como coverage controls, no leaderboard. | `crpto_ijds_v4_table6_credit_controls.csv` | +| Tabla two-ruler | Body | Seis tracks y sus hulls endpoint-corrected. | `crpto_ijds_v4_table5_two_ruler_tracks.csv` | +| Figura 3 | Supplement | Dirección de envelopes sobre development support. | `crpto_ijds_v4_fig3_envelopes` | +| Tabla endpoint timing | Supplement | Sensibilidad separada del fit-label lag. | `crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv` | +| Tabla structural sensitivity | Supplement, resumen en body | Censo no selectivo de los 36 escenarios. | `crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv` | + +: Selección activa de figuras y tablas. ::: -El paper no necesita mostrar todas las tablas en el cuerpo. El cuerpo debe -proteger tres mensajes: la metodología, el punto pool93 promovido y la frontera -finita A35 y la baseline A40. -El appendix puede cargar la evidencia de stress y trazabilidad. - -### Paquetes del online supplement (A--F) +El paquete activo contiene tres figuras, cada una en PNG y PDF. Decir “seis +figuras” confunde formatos con objetos visuales. -El online supplement absorbe todo lo que fortalece el paper sin romper el límite -de 25 páginas. Se organiza en seis paquetes; `paper/supplement_ijds.qmd` es la -fuente de escritura y el detalle por tabla está en -[`07-apendice-robustez.qmd`](07-apendice-robustez.qmd) y -[`14-release.qmd`](14-release.qmd). +### Arquitectura del online supplement ::: {#tbl-crpto-supplement-packages} -| Paquete | Contenido | Fuente en el libro | +| Apéndice | Contenido | Razón para separarlo | |---|---|---| -| A. Proof details | Teorema Markov y tightening Hoeffding/Bernstein como condicional. | Cap. 02 (`thm-conformal-feasibility`); apéndice condicional. | -| B. Conformal ablation | Ranks 1--3, `grade` vs `score_decile_mondrian`, coverage por grupo. | Cap. 08 (ablación Mondrian); A10. | -| C. Robustness A3--A36 | Nested/temporal holdouts, synthetic shift, tail diagnostics, bootstrap, dependency, tail-constrained challenger, multi-distribución, online ACI, réplica externa y precio de robustez cross-dataset. | Cap. 07 (apéndice journal) y @sec-crpto-multidataset. | -| D. Funded set | Composición periodo × grade y loan-level export resumido. | Cap. 12 (funded set); A7--A8. | -| E. Fairness/MRM | Proxy audit, controles SR 11-7 y source-governance caveats. | Cap. 10 (fair lending); cap. 11 (MRM). | -| F. Reproducibility | Claim -> artifact -> script -> test, paths DVC/MLflow y data/code disclosure. | Cap. 13 (trazabilidad); cap. 14 (release). | - -Paquetes A--F del online supplement IJDS. +| A | Universo, cronología, endpoint y information boundary. | Auditoría completa del dato. | +| B | Cinco learners, WOE/IV, monotonicidad, PSI, ventanas y label lag. | Censo técnico de cobertura y geometría. | +| C | Payoff, LP, rulers y named comparators. | Definición exacta del contrafactual. | +| D | Enunciados y pruebas. | Mantiene legible el body. | +| E | Censo two-ruler, endpoint recovery, estructura y exact support. | Reporte completo sin elegir celdas o escenarios. | +| F | Simulación de mecanismo. | Aísla explicación geométrica de claims empíricos de policy. | +| G | Lineage, runs detenidos y replay. | Procedencia reproducible. | +| H | Claim boundary y limitaciones. | Evita que una tabla favorable amplíe el claim. | +| I | Closest-work positioning. | Delimita novedad sin afirmar que cada ingrediente es nuevo. | + +: Paquetes A--I del online supplement. ::: -### Mapa claim -> artifact -> test -> paper location +### Claim, artefacto y guardrail ::: {#tbl-crpto-claim-artifact-test-location} -| Claim | Artifact canónico | Test/guardrail | Donde va | +| Objeto | Autoridad | Guardrail principal | Uso | |---|---|---|---| -| C1 | `reports/dvc/metrics_summary.json`, `data/processed/pipeline_summary.json` | Doc guardrails de baseline operacional | Resultados, Tabla 1 | -| C2 | `conformal_upstream.winner_metrics`, `alpha_sweep_pareto_mondrian.parquet` | Tests de CRPTO y guardrails conformales | método/resultados | -| C3 | `scripts/run_portfolio_bound_aware_search.py`, `champion_portfolio_policy.json` | Consistency tests de champion | Método | -| C4 | `scripts/validate_alpha_gamma_bound.py`, bound eval `276k` | `test_crpto_champion_artifacts_agree` | Teoría y appendix | -| C5 | `models/final_project_promotion.json` | `test_crpto_champion_artifacts_agree` | Resultados | -| C6 | `portfolio_bound_aware_shortlist.parquet`, A18 | Journal package guardrail | resultados/appendix | -| C7 | A3--A36, status P1, journal package y multidataset | P1 evidence + journal package + external replication tests | Robustness | - -Mapa compacto para escribir y auditar el paper. +| Claim registry | `docs/research/active_claims_2026-07-14.md` | `test_ijds_active_claim_sync.py` | Lenguaje permitido y prohibido. | +| Claim ledger | `configs/ijds_claim_ledger.yaml` | `test_claim_ledger.py` | Alcance, superficies y forbidden inferences ejecutables. | +| Evidence registry | `configs/ijds_active_evidence_sources.yaml` | `test_publication_sources.py` | Identidad, rol y DVC de cada lineage. | +| Manifest numérico | `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` | `test_ijds_v4_claim_sync.py` | Única autoridad paper-facing de números. | +| Data contract | `ijds-raw-data-contract-2026-07-14-v2/evidence.json` | Publication integrity | Población, esquema y madurez. | +| Credit controls | `ijds-credit-risk-controls-2026-07-14-v3/credit_risk_controls_summary.json` | Publication integrity | Cinco modelos, calibración, IV y PSI. | +| Binary geometry | V4 summary + execution receipt | V4 scientific invariants | Cobertura, fase, support y solver. | +| Two-ruler | V3 verified evaluation manifest | Endpoint reconciliation tests | Contrastes y sharp hulls. | +| Endpoint sensitivity | V1 endpoint sensitivity summary | Sensitivity evidence tests | Timing del endpoint, separado de fit labels. | +| Structural sensitivity | V6 freeze + evaluation summary | Structural evidence tests | 36 escenarios completos, baseline reconciliado y ningún selector. | + +: Mapa de autoridad y verificación. ::: -### Notacion única +### Notación única ::: {#tbl-crpto-notation-unified} -| Simbolo | Significado | Donde se usa | Regla editorial | -|---|---|---|---| -| `Y_i` | Default observado o target acotado evaluado | Bound, exact eval | No llamarlo PD latente sin supuesto adicional | -| `p_hat_i` | PD puntual calibrada | PD, CP, LP | Viene de CatBoost + Venn-Abers | -| `u_i(alpha)` | Cota superior conformal | Uncertainty set | Entrada prudente del optimizador | -| `x_i` | Decisión o fraccion de asignación | LP | Puede ser continua en el solver operativo | -| `a_i` | Monto del préstamo | Presupuesto | Normaliza exposición | -| `w_i` | Peso de portafolio financiado | Bound | `w_i = x_i a_i / sum_j x_j a_j` | -| `tau` | Techo de riesgo ponderado | LP robusto | Se reporta como `risk_tolerance` | -| `Gamma_CP` | Prima conformal ponderada | teoría/resultados | No mezclar con `gamma` del solver | -| `gamma` | Parámetro de policy del robust LP | Search de portafolio | Se reporta junto con `risk_tolerance` | -| `V` | No-cobertura ponderada del funded set | Exact eval | Cantidad principal del bound empírico | - -Notacion canónica para evitar drift entre 14b, 14c y 14d. +| Símbolo | Significado | Regla editorial | +|---|---|---| +| $Y_i$ | Default terminal observado o no resuelto al cutoff. | No llamarlo PD latente. | +| $s_i$ | Score point-PD calibrado. | Solo CatBoost primario entra al LP. | +| $u_i$ | Upper score conformal binario. | No implica validez del funded set. | +| $a_i$ | Fracción financiada. | Se decide dentro de un menú mensual. | +| $b$ | Cap del score ponderado. | Su escala depende del score. | +| $\gamma$ | Mezcla guardrail--point en el diagnóstico two-ruler. | No se selecciona ni se promueve. | +| $\rho$ | Coordenada común del ruler. | Debe nombrarse junto con el ruler. | +| $\Delta$ | Guardrail menos point para un endpoint. | Se reporta como sharp hull si hay outcomes no resueltos. | + +: Notación canónica del manuscrito activo. ::: -La distinción más delicada es `Gamma_CP` vs `gamma`. `Gamma_CP` es una métrica -inducida por los intervalos y pesos del funded set; `gamma` es un parámetro de -la policy robusta. En el texto final deben aparecer juntos solo cuando se -explique como una policy con cierto `gamma` genera cierto `Gamma_CP`. - -### Related work compacto con referencias numeradas - -El related work del paper puede comprimirse en cinco bloques: - -1. **Conformal prediction y risk control.** La validez finita de CP viene de - Vovk et al. [1], de la exposición moderna de Angelopoulos y Bates [2] y de la - monografia de fundamentos [18]. RCPS, LTT y CRC amplian el lenguaje hacia - control de riesgos definidos por el usuario [3]--[5], con la extensión a - pérdidas no monotonas [24]. Los límites de la inferencia condicional [20], la - cobertura por grupos [21] y la versión localizada [22] explican por qué solo - prometemos cobertura marginal/Mondrian, y la robustez al ruido de etiquetas - [23] sostiene los stress de flips. -2. **Robust optimization y conformal RO.** Bertsimas y Sim dan el lenguaje del - precio de la robustez [6]. Johnstone y Cox [7], la versión contextual [8] y la - optimización-satisficing conformal [27] muestran como pasar de sets - conformales a decisiones robustas con cobertura calibrada. -3. **Predict-then-calibrate y DFL.** Predict-then-Calibrate es el vecino más - cercano por su separación post-hoc entre predicción, calibración y LP [9]. - SPO+ y Online DFL son comparadores decision-focused que optimizan regret, - pero no entregan el mismo tipo de cobertura auditable [15], [16]; la raiz - task-based [28] y los regret robustos [29] completan esa familia. -4. **Frontera reciente y medidas de cola.** CROMS, end-to-end conformal - calibration, conformal risk training, MDCP y online CP vía universal - portfolios apuntan a la versión siguiente del proyecto [10]--[14], junto al - equilibrio de gradiente online [30]. La re-optimización con cota de cola se - apoya en CVaR [25] y OCE [26]. Sirven para reforzar actualidad, no para - cambiar el champion actual. -5. **Crédito, fairness y datos.** El contexto del caso de uso se ancla en el - desempeno y equidad de los scores [31], el rol de datos alternativos en - LendingClub [32], el proxy BISG [33] y el marco de fairness de FinRegLab [34]. - -La lista numerada completa está en `docs/research/crpto_editorial_claims_references.qmd`. En el paper -final, estos números se reemplazan por citas BibTeX normales; en el libro se -mantienen porque ayudan a escribir el related work como argumento y no como -inventario. - -### Contribuciones redactadas para introducción - -Una versión fuerte y honesta de las contribuciones sería: - -1. Proponemos **CRPTO**, un framework post-hoc que conecta PD calibrada, - intervalos conformales Mondrian y optimización robusta de portafolio. -2. Derivamos un bound distribution-free sobre no-cobertura ponderada del funded - set y separamos explícitamente el tightening condicional de la garantía - principal. -3. Mostramos que el enfoque produce una policy pool93 promovida con retorno - `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral - Markov `0.345084`, exceso realizado cero y pass `8/8`. -4. Documentamos una frontera finita consolidada con 50,010 políticas semánticas - y 27,508 all-alpha above-floor policies, por lo que el resultado no depende - de un único punto elegido después de ver los datos. -5. Entregamos trazabilidad reproducible desde artefactos DVC/MLflow/DagsHub - hasta tablas de paper, funded set loan-level y guardrails documentales. - -Lo que no decimos: que CRPTO domina SPO+ en regret, que Markov es una cota -apretada en todos los regímenes o que A6/A11 equivalen por sí solos a validación -externa. A25--A34 cubren réplica estática, no despliegue prospectivo. -Esa cautela es una fortaleza, no una debilidad. - -### Checklist de aceptacion del borrador - -Antes de convertir esto en paper `.tex` o `.qmd` independiente, el borrador debe -pasar este checklist: - -- El champion oficial es siempre `bound_aware_276k_economic_champion`. -- El retorno oficial sale de `final_project_promotion.json`, no de tablas - diagnosticas repriced. -- `theorem-tight` aparece como comparador, nunca como champion. -- `Y_i` se define como target observado/acotado; PD latente queda como lectura - adicional bajo supuesto. -- Las familias de métricas no se mezclan en un solo leaderboard. -- Las tablas A12--A34 se presentan como robustness/journal package, no como una - nueva búsqueda de champion. -- El related work incluye CP/CRC/RCPS, RO, Predict-then-Calibrate, DFL, CROMS, - MDCP, online CP y conformal risk training. -- El appendix contiene los comandos exactos para regenerar tablas, renderizar - Quarto y correr guardrails. +### Núcleo teórico + +El body necesita nueve afirmaciones exactas y sus fronteras: + +1. equivalencia de caps bajo transformaciones globales positive-affine; +2. invariancia del ruler normalizado a esas transformaciones; +3. nesting mecánico cuando se copia el mismo cap; +4. dominancia del objetivo plug-in bajo el comparador C2; +5. discontinuidad del cuantil residual binario en el modelo de score constante; +6. identidad loan-wise de miscoverage binario; +7. bounds sharp con una asignación común de outcomes no resueltos; +8. suficiencia de endpoints de bases LP sobre el soporte declarado; +9. interpretación del envelope como partial identification, no como intervalo de confianza. + +Ninguna de estas propiedades ordena por sí sola payoff realizado, default, +miscoverage o bienestar. + +### Regla de decisión editorial + +No existe una policy oficial del manuscrito activo. Tampoco existe un learner, +ruler, coordenada o gamma ganador. Un resultado solo entra al body si: + +1. está en el evidence manifest activo; +2. pertenece a una grid completa declarada; +3. conserva unresolved outcomes o usa bounds sharp; +4. no fue seleccionado por su signo OOT; +5. lleva la limitación que impide una lectura más fuerte. + +Los bundles `pool93`, `compact-v7`, selected-policy y A1--A40 siguen protegidos +para procedencia, pero no son fuentes de esta arquitectura. diff --git a/book/chapters/06b-guia-editorial-claims.qmd b/book/chapters/06b-guia-editorial-claims.qmd index 5bbca67..d7d5a29 100644 --- a/book/chapters/06b-guia-editorial-claims.qmd +++ b/book/chapters/06b-guia-editorial-claims.qmd @@ -1,397 +1,185 @@ - - # Guía Editorial, Claims y Referencias {#sec-crpto-editorial-guide} -Esta página existe porque el libro cumple una función distinta al paper. El -paper final debe ser breve, selectivo y persuasivo; el libro puede ser más -generoso: explica por qué se tomó cada decisión, qué evidencia sostiene cada -claim, qué preguntas de reviewer ya están respondidas y qué partes todavía son -agenda futura. En otras palabras, esta sección es el puente entre el proyecto -vivo y el manuscrito publicable. - -Para facilitar la escritura posterior, esta página usa referencias numeradas -locales del tipo `[1]`. La bibliografía global del libro sigue viviendo en -`book/references.bib`; los números de esta página son una guía editorial para -el CRPTO, no un reemplazo del sistema BibTeX del libro. - -### Tesis defendible en una frase - -CRPTO demuestra que una banda conformal calibrada puede dejar de ser un -diagnóstico posterior y convertirse en una restricción operativa para asignar -capital crediticio bajo incertidumbre, con una policy económica promovida, -artefactos reproducibles y un bound que separa cuidadosamente garantía -distribution-free de evidencia empírica post-selección. - -La frase anterior tiene cuatro piezas: - -1. **banda conformal calibrada**: se apoya en la teoría conformal y de control de - riesgo [1]--[5]; -2. **restricción operativa**: conecta esos intervalos con robust optimization y - price of robustness [6]--[9]; -3. **policy económica promovida**: usa el cierre rebaselined `ijds-rebaseline-2026-06-07` - y no reabre el champion; -4. **cautela post-selección**: separa el teorema Markov del tightening - condicional y de las validaciones empíricas A7--A11. - -### Escalera de claims {#sec-crpto-claim-ladder} - -El paper se vuelve más fuerte cuando los claims se ordenan por nivel de -ambición. No todos tienen el mismo estatus. Algunos son teóricos, otros son -empíricos, otros son de ingeniería reproducible. - -| Nivel | Claim | Evidencia actual | Lo que no debe prometer | -|---|---|---|---| -| 1 | La PD base es suficientemente calibrada para alimentar incertidumbre | AUC `0.7139`, Brier `0.1544`, ECE `0.0070`, Venn-Abers | No afirmar que AUC es el aporte principal | -| 2 | Los intervalos conformales tienen cobertura útil para decisión | Coverage 90 `92.97%`, min group coverage `91.90%` | No afirmar cobertura condicional perfecta | -| 3 | El intervalo puede convertirse en conjunto de incertidumbre | Definición de `u_i(alpha)` y `Gamma_CP` | No tratar `Gamma_CP` como presupuesto ad hoc | -| 4 | El bound controla no-cobertura ponderada del funded set | @sec-thm-feasibility y `V` | No decir que controla directamente una PD latente sin supuesto adicional | -| 5 | Existe una frontera finita robusta en OOT | A35: 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | No decir que eso elimina toda incertidumbre post-selección ni que certifica una región continua | -| 6 | El punto oficial es pool93 body/default, no el endpoint económico extremo | Retorno `$184.8K`, `V=0.035350`, `gamma_cp=0.162616`, `gamma_res=0.073584`, umbral Markov `0.345084`, pass `8/8` | No mezclarlo con el max-return endpoint de `$223K` ni con el punto de threshold mínimo | -| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A7--A11, apéndice condicional, A25--A34 | No venderlo como validación live o universal | +Esta guía gobierna cómo escribir el CRPTO activo. Su propósito es impedir tres +saltos frecuentes: de calidad predictiva a calidad de decisión, de cobertura +candidata a validez del selected set, y de una comparación bajo cierto ruler a +dominancia universal. + +## Regla de autoridad + +1. `docs/research/active_claims_2026-07-14.md` decide qué afirmaciones están + activas. +2. `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` decide qué + números pueden aparecer como paper-facing. +3. `configs/ijds_claim_ledger.yaml` convierte los claims cualitativos y sus + límites en un contrato ejecutable sobre las superficies editoriales. +4. `configs/ijds_active_evidence_sources.yaml` decide qué lineages y hashes + sustentan el manifest. +5. Body, supplement y TeX generado deben concordar con esas cuatro capas. + +Un memo, una tabla histórica o un resultado visualmente favorable no puede +ampliar el claim. Los bundles `pool93`, `compact-v7`, selected-policy y A1--A40 +son procedencia, no evidencia activa. + +## Escalera de claims {#sec-crpto-claim-ladder} + +| Nivel | Claim defendible | Evidencia | Frontera obligatoria | +|---:|---|---|---| +| 1 | El universo activo es status-independent y exhaustivo bajo el contrato declarado. | 640,543 préstamos elegibles de 2,925,493 filas crudas. | No generalizar a otros plazos, esquemas o plataformas. | +| 2 | La cobertura candidata no transporta en la especificación completa. | Cinco learners × ocho ventanas: 40/40 upper bounds < 0.90. | No afirmar invalidez conformal universal. | +| 3 | La geometría del intervalo binario es sensible a prevalencia. | Cruce W7--W8 en stratum 2 y resultado de score constante. | Mecanismo, no prueba causal del caso variable-score. | +| 4 | Copiar un cap o igualar una coordenada no define el mismo contrafactual. | Equivalencia afín, same-cap nesting, C2 y normalized invariance. | La teoría ordena objetos plug-in específicos, no outcomes realizados. | +| 5 | La dirección downstream depende de ruler y coordenada. | Seis tracks, 48 contrastes de ventana y 144 celdas metric-direction. | No seleccionar gamma, ruler, coordenada o signo. | +| 6 | La incertidumbre de endpoint y comparator support puede acotarse exactamente. | 12,076 outcomes no resueltos; sharp common-outcome bounds; 3,067 caps. | Partial identification, no confidence interval ni frontera universal. | +| 7 | La dirección sigue siendo condicional al variar estructura de portafolio. | 36 escenarios completos; todos conservan celdas adversas de default y miscoverage; ninguno es uniformemente favorable o adverso. | No seleccionar escenario ni afirmar daño universal. | +| 8 | El sistema completo es el objeto científico. | Score, intervalo, LP, endpoint y soporte hash-linked. | No separar una cifra favorable de su protocolo. | : Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} -La regla de escritura es simple: el cuerpo principal del paper debe vivir en los -niveles 1--6; el nivel 7 alimenta appendix, respuesta a reviewers y versión -journal. Si una frase no cabe en esta escalera, probablemente pertenece a la -agenda futura o a otro paper. - -### Aportes por audiencia +La contribución más fuerte es negativa y constructiva. El paper no dice que +conformal prediction o robust optimization fallen en general; muestra qué no +queda identificado al conectarlos sin fijar transporte, endpoint y comparador. -El mismo resultado se lee distinto según quién revise el manuscrito. +## Traducción por audiencia -| Audiencia | Qué le importa | Qué debe ver rápido | Artefacto o sección | -|---|---|---|---| -| OR / Management Science | Decisión bajo incertidumbre y trade-off robustez-retorno | CRPTO como puente CP -> RO -> portfolio | @sec-alpha-gamma-bound, @sec-crpto-results | -| ML / conformal prediction | Garantía distribution-free y límites post-selección | Markov como claim principal, tightening condicional separado | @sec-thm-feasibility, @sec-lem-conditional-tightening | -| Crédito / riesgo | Auditabilidad, funded set, governance | Policy congelada, composición del funded set, segment checks | @sec-crpto-p1-evidence | -| Reviewer escéptico de DFL | Por qué no basta SPO+ | SPO+ reduce regret, CRPTO compra cobertura auditable | @fig-p1-spo-regret, @sec-crpto-comparison | -| Reproducibilidad | Trazabilidad y no cherry-picking | Mapa claim -> artifact -> test | @tbl-crpto-claim-artifact-test | - -: Traducción del aporte por tipo de reviewer {#tbl-crpto-reviewer-audiences} - -Esta tabla es útil para escribir la introducción del paper. El paper no debe -presentarse como "ganamos todo"; debe presentarse como "resolvemos una dimensión -que otros enfoques dejan débil: decisión auditable bajo incertidumbre". - -### Qué aprendimos de la literatura - -La literatura conformal fundacional nos da el permiso teórico para construir -intervalos con cobertura finita bajo intercambiabilidad [1], [2]. RCPS, LTT y -CRC enseñan una lección más general: la predicción conformal puede controlar -riesgos definidos por el usuario, no solamente coverage marginal promedio [3]-- -[5]. Esa es la puerta conceptual para nuestro bound: el riesgo relevante no es -solo "falló el intervalo", sino "falló el intervalo en préstamos que realmente -financié". - -La robust optimization clásica enseña que la robustez tiene precio y que ese -precio puede medirse [6]. Los trabajos conformal robust optimization muestran -que los conjuntos conformales pueden alimentar problemas de optimización [7], -[8]. Predict-then-Calibrate es especialmente cercano porque separa predicción y -calibración para LP contextuales robustos [9]. Nuestra diferencia práctica es el -aterrizaje: crédito minorista, funded set ponderado, región finita exacta y -artefactos reproducibles. - -La literatura más reciente marca hacia dónde crecería una versión futura: -CROMS sugiere selección de modelos conformales con criterio de robust -optimization [10]; end-to-end conformal calibration y conformal risk training -apuntan a entrenar la calibración con la decisión aguas abajo [11], [12]; -multi-distribution robust CP y online CP vía universal portfolios apuntan a -shift y streaming [13], [14]. Online DFL abre el comparador dinámico para -decisiones repetidas [16]. Ninguna de esas líneas invalida el paper actual; más -bien confirma que la dirección del paper es contemporánea y fértil. - -La biblioteca de tesis integrada aterriza, además, varias de esas líneas en -evidencia concreta del proyecto. La monografía de fundamentos [18] y los límites -de la inferencia condicional [20] fijan por qué solo prometemos cobertura -marginal/Mondrian; covariate shift [19], group-weighted CP [21] y localized CP -[22] sostienen el diagnóstico multi-distribución A23, mientras que la robustez al -ruido de etiquetas [23] respalda el stress de flips A11. En la capa de decisión, -el conformal risk control para pérdidas no monótonas [24] junto con CVaR [25] y -OCE [26] son el soporte teórico de la re-optimización con cota de cola A22, y la -optimización-satisficing conformal [27] enmarca A13/A20. El decision-focused -learning queda anclado en su raíz task-based [28] y en los regret robustos [29], -que complementan a SPO+. El equilibrio de gradiente online [30] da la lectura de -recalibración secuencial de A24. Por último, el contexto de crédito y fairness ---- desempeño y equidad de scores [31], datos alternativos en LendingClub [32], -proxy BISG [33] y el marco de política de FinRegLab [34] --- encuadra el -checkpoint MRM/fair-lending sin convertirlo en un claim legal. - -### Tabla de traducción conceptual - -| Concepto estadístico | Traducción en CRPTO | Lectura de negocio | +| Audiencia | Pregunta principal | Respuesta CRPTO | |---|---|---| -| `alpha` | Nivel de tolerancia a no-cobertura | Cuánto riesgo de cola acepta el comité | -| `PD_low`, `PD_high` | Intervalo conformal por préstamo | Rango defendible de riesgo | -| `u_i(alpha)` | Cota superior usada por el LP | PD prudente para decisión | -| `Gamma_CP` | Prima conformal ponderada | Costo promedio de comprar robustez | -| `V` | No-cobertura ponderada del funded set | Cuánto default realizado quedó fuera de la cota | -| `violation` | Exceso de riesgo ponderado sobre `tau` | Incumplimiento del límite de portafolio | -| `alpha01_exact_pass` | Check exacto de bound en OOT | Semáforo para promoción paper-facing | -| `price_of_robustness` | A40 empareja candidatos y restricciones: costo realizado `5.875%`, reducción default/V `8.305` pp y reducción de threshold `43.55` pp | Costo económico de la robustez, solo con comparadores semánticamente equivalentes | - -: Diccionario operativo del CRPTO {#tbl-crpto-operational-dictionary} +| Data science | ¿La incertidumbre predictiva conserva significado downstream? | No automáticamente: se auditan transporte y geometría antes del LP. | +| Conformal prediction | ¿La cobertura marginal autoriza un claim sobre funded set? | No; candidate coverage y selected-set validity son objetos distintos. | +| Optimization | ¿Qué significa comparar dos scores con caps diferentes? | El ruler y la coordenada son parte del estimand; exact support revela sensibilidad. | +| Credit risk | ¿El resultado depende de CatBoost o de señales de pricing? | Cinco controles, incluidos WOE/IV y monotonic CatBoost, rechazan esa explicación simple. | +| Reproducibilidad | ¿Se eligió una combinación después de ver outcomes? | No se promueve ninguna; grids completas, freezes outcome-free y joins estrictos. | +| Governance | ¿Puede desplegarse la policy? | No; es una auditoría retrospectiva sin fair-lending ni validación prospectiva. | -Esta traducción es una de las fortalezas del libro. En el paper final, varios de -estos términos pueden aparecer en una sola figura o tabla; aquí conviene -mantenerlos explícitos porque ayudan a que el lector no confunda métricas -predictivas, conformales y de portafolio. - -### Preguntas de reviewer y respuesta corta - -**¿Por qué no basta con reportar AUC?** -Porque el aporte está downstream. La PD es un insumo; la contribución ocurre -cuando la incertidumbre entra al optimizador y cambia el funded set. - -**¿SPO+ no es mejor porque reduce regret?** -SPO+ es un comparador fuerte y reduce regret en el benchmark. CRPTO no pretende -dominarlo en regret; compra auditabilidad, cobertura verificable y una región -robusta exacta. Es otro eje de valor [15], [16]. - -**¿El bound controla PD latente o default observado?** -El bound principal controla un target acotado observado `Y_i in [0,1]` y su -no-cobertura ponderada. Interpretarlo como PD latente requiere un supuesto -adicional. Esa cautela está explícita en @sec-alpha-gamma-bound. - -**¿Hay riesgo de post-selección?** -Sí, por eso el paper no vende el cierre `276k` como una garantía conformal nueva. -Lo presenta como evidencia empírica exacta de la policy promovida y lo refuerza -con A3, A9 y el apéndice condicional. - -**¿Por qué rank 2 y rank 3 no son winners si pasan exact eval?** -Porque el selector no solo mira bound de portfolio. Ranks 2 y 3 pasan la eval -exacta de portfolio, pero fallan el gate conformal por cobertura mínima de -grupo. Esa es precisamente la utilidad de A10: muestra que el selector no es una -excusa post-hoc. - -**¿Los stress tests reemplazan un dataset externo?** -No. A6 y A11 fortalecen la evidencia bajo reponderación y flips adversariales, -pero no reemplazan la réplica externa. Esa capa ya queda cubierta por A25--A34 -en Prosper y Freddie/Mendeley; lo que sigue fuera es una validación live o -prospectiva con protocolo sellado. - -### Reviewer-defense bank gobernado {#sec-crpto-reviewer-defense-bank} - -Las preguntas anteriores cubren las objeciones más frecuentes en prosa. La tabla -siguiente es el **banco gobernado completo** (13 objeciones) que se usa para -preparar el rebuttal IJDS y la defensa de tesis. Cada fila fija la objeción -probable, la respuesta gobernada y la evidencia canónica. La regla es no -improvisar: si llega una objeción nueva, primero se busca aquí. - -| Objeción probable | Respuesta gobernada | Evidencia | -|---|---|---| -| "Conformal no garantiza cobertura condicional perfecta." | Correcto; el claim usa cobertura marginal/Mondrian y reporta límites explícitos. | @sec-thm-feasibility; cap. 07; ablación Mondrian (cap. 08). | -| "AUC no basta para decidir portafolios." | Precisamente: el paper separa ranking, calibración, incertidumbre y valor económico. | Tabla 0 (`crpto_table0_key_metrics.csv`); escalera de claims (esta página). | -| "SPO+/DFL podría reducir regret." | Sí; CRPTO no compite como regret-minimizer universal, sino como puente auditable con restricciones conformales. | Cap. 09 (`@fig-p1-spo-regret`); registro de negativos. | -| "Lending Club no es banco regulado." | Se presenta como dataset público de originación, no como validación contractual bancaria. | `docs/ACADEMIC_CONTEXT.md`; limitaciones del cuerpo. | -| "Fair lending no está probado." | No se hace claim legal; solo proxy governance y source/fairness boundary. | Cap. 10; `models/fairness_audit_status.json`. | -| "IFRS9/SICR parece otro paper." | En IJDS entra como implicación/limitación; la tesis puede expandirlo. | Cap. 23 (señal SICR conformal). | -| "El resultado podría ser un punto aislado." | La frontera A35 muestra 50,010 políticas semánticas y 27,508 políticas all-alpha above-floor. | `crpto_tableA35_pool93_ijds_frontier.csv`; @sec-crpto-results. | -| "¿La validación PD se está tratando como clasificación?" | No. CRPTO separa risk differentiation, risk quantification, calibración y valor económico antes de optimizar portafolio. | Tabla 0; cap. 02; `crpto_metrics_triage_2026-05-18.md`. | -| "¿El backtesting multi-periodo es confiable por defecto?" | No se sobrepromete: las pruebas multi-periodo dependen del estimador, tamaño efectivo y autocorrelación; quedan como caveat o appendix controlado. | Cap. 07 (A14--A15); `crpto_metrics_triage_2026-05-18.md`. | -| "¿Usar ML en IRB implica perder auditabilidad?" | No. ML queda como soporte selectivo para imputación, factores, residuos o challengers; el champion no se reabre sin gate reproducible. | Cap. 13 (freeze policy); `crpto_metrics_triage_2026-05-18.md`. | -| "¿WOE/scorecards son estables bajo monitoreo?" | Se tratan como inputs auditables con caveats: IV no es soberano y WoE puede volverse inestable si se recalcula bajo datos etiquetados por el propio modelo. | `crpto_metrics_triage_2026-05-18.md`. | -| "¿Los umbrales IV/PSI/correlación prueban calidad del modelo?" | No. Son heurísticas operativas útiles, no evidencia estadística suficiente; los umbrales tipo 0.02/0.1 son sensibles a desbalance y deben gobernarse como incertidumbre estimable. | `crpto_metrics_triage_2026-05-18.md`. | -| "¿Los árboles sobre residuos contradicen el scorecard auditable?" | No. Quedan como validación de segmentos y búsqueda de factores omitidos; no reemplazan el champion ni la calibración oficial. | Agenda extendida CRPTO/tesis; `crpto_metrics_triage_2026-05-18.md`. | +: Traducción del aporte por tipo de reviewer {#tbl-crpto-reviewer-audiences} -: Banco gobernado de respuestas a reviewers del CRPTO {#tbl-crpto-reviewer-defense-bank} +## Vocabulario gobernado -### Qué puede entrar al paper final +Usar: -| Material del libro | Ubicación probable en paper | Motivo | -|---|---|---| -| Diagrama CRPTO | Figura 1 | Explica el método en una sola vista | -| Escalera alpha -> `Gamma_CP` -> funded set | Teoría / método | Une conformal y robust optimization | -| Tablas A35/A40 | Resultados principales | Demuestran frontera finita y trade-off contra point-PD emparejado | -| Holdout temporal estricto CRPTO | Appendix o robustness checks | Responde post-selección | -| Funded-set composition | Appendix | Útil para reviewers de crédito y riesgo | -| A10 exact finalists | Appendix | Defiende el selector conformal | -| A11 enhanced shift | Appendix | Fortalece robustness interna; A25--A34 cubren réplica externa estática | -| A25--A34 external replication | Cuerpo corto y appendix | Responde la crítica de un solo dataset sin reabrir el champion | -| Referencias numeradas | Related work del paper | Ayuda a pasar de libro a manuscrito | +- **retrospectively protocol-locked before the corresponding outcome join**; +- **complete declared grid**; +- **sharp common-outcome bound**; +- **partial identification over declared support**; +- **coverage control**, no model challenger ganador; +- **standardized payoff**, no cash return o welfare; +- **risk-aware allocation audit**, no certified robust policy. -: Destino editorial probable del material del libro {#tbl-crpto-paper-placement} +Evitar: -### Paquete journal-ready CRPTO +- *preregistered*, *confirmatory*, *prospective* o *untouched holdout*; +- *selected-set conformal guarantee*; +- *policy winner*, *optimal gamma* o *best ruler*; +- *universal guardrail benefit*; +- *causal mechanism* para W7--W8; +- *verified September 2020 snapshot*; +- *continuous-frontier uniqueness*; +- *deployment-ready* o *fair-lending compliant*. -Las páginas @sec-p1-manuscript-blueprint y @sec-p1-journal-appendix convierten -esta guía editorial en un paquete de escritura más cercano al manuscrito. El -blueprint define venue target, abstract operativo, claims C1--C7, notacion única -y plan de figuras/tablas. El appendix journal agrega A12--A34 y las figuras -paper-grade generadas desde artefactos congelados. +## Diccionario operativo -| Nuevo material | Papel editorial | Artefacto | +| Término | Definición activa | No significa | |---|---|---| -| Blueprint del manuscrito | Estructura paper-ready y mapa claim -> artifact -> test -> location | `06-blueprint-manuscrito.qmd` | -| Figura CRPTO limpia | Figura 1 IJDS | `crpto_fig1_journal_pipeline.png` | -| Figura pila del bound | Puente visual entre endpoint conformal, identidad, supuesto y certificado | `crpto_fig20_bound_claim_layers.png` | -| Figura alpha -> `Gamma_CP` -> funded set | Puente teoría-método | `crpto_fig13_alpha_gamma_funded_set.png` | -| Frontera pool93 A35 | Evidencia tabular de no-aislamiento | `crpto_tableA35_pool93_ijds_frontier.csv` | -| A12--A34 | Robustness appendix journal | `scripts/build_crpto_journal_package.py`; `scripts/build_tail_satisficing_challenger_audit.py`; `scripts/build_tail_constrained_reoptimization.py`; `scripts/build_distribution_robustness_diagnostics.py`; `scripts/build_multidataset_external_replication.py`; `scripts/build_price_of_robustness_cross_dataset.py` | -| A22--A24 (re-opt de cola, multi-distribución y online) | Cierre P2 sin reabrir champion | `scripts/build_tail_constrained_reoptimization.py`; `scripts/build_distribution_robustness_diagnostics.py` | -| A25--A34 (Prosper/Freddie) | Réplica externa sin reabrir Lending Club | `scripts/build_multidataset_external_replication.py`; `scripts/build_price_of_robustness_cross_dataset.py` | -| Status del paquete | Trazabilidad y fuente canónica | `models/crpto_journal_package_status.json`; `models/crpto_tail_constrained_reopt_status.json`; `models/crpto_distribution_robustness_status.json`; `models/crpto_multidataset_external_status.json` | - -: Material journal-ready agregado al CRPTO {#tbl-crpto-journal-ready-package} - -La regla de jerarquia sigue intacta: si una tabla diagnostica nueva entra en -tension con `models/final_project_promotion.json`, gana la promoción oficial. -A12--A34 fortalecen robustez y escritura journal, pero no re-promueven el -champion: A22 reporta un *tail-constrained challenger* y A25--A34 reportan -réplica externa, no un nuevo champion. - -### Auditoría Papers_tesis 2026-06-06 {#sec-crpto-papers-tesis-deep-audit} - -El source pack de tesis queda auditado con una regla evidence-gated: leer mucho, -promover poco y no cambiar el champion sin un gate capaz de modificar C1--C7. -La auditoría cubre los 61 PDFs locales de `Papers_tesis/`: 21 en `paper/`, 28 -en `supplement/` y 12 en `tesis/`. - -| Artefacto | Uso editorial | -|---|---| -| `docs/research/papers_tesis_deep_audit_2026-06-06.md` | Memo paper-by-paper con conceptos, claims, método, conclusiones, figuras/tablas útiles, limitaciones, destino editorial y acción requerida. | -| `reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv` | Matriz fuente para decidir `promote`, `append`, `park` o `future_gate` sin depender de memoria editorial. | -| `reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv` | Índice de figuras/tablas detectadas en PDFs para seleccionar visual sinks sin copiar figuras ajenas. | -| `reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv` | Lista corta de sinks visuales que pueden inspirar figuras propias del paper, supplement o tesis. | - -: Artefactos canónicos de la auditoría Papers_tesis 2026-06-06 {#tbl-crpto-papers-tesis-audit-artifacts} - -La regla de absorción es conservadora. En el cuerpo CRPTO solo entran fuentes que -sostienen directamente CP/CRC/LTT, robust optimization, CP -> RO / -Predict-then-Calibrate, contexto Lending Club y DFL/SPO+ como comparador. Las -fuentes de Hu CRC, utility-directed CP, group-weighted/localized/source-shift, -decision-risk certificates, IFRS9 y fairness/equity quedan como related work, -appendix o future work. Ese material fortalece límites de claim y defensa ante -reviewers; no reabre la policy promovida ni convierte diagnósticos en método -oficial. - -### Referencias numeradas locales - -[1] Vovk, Gammerman and Shafer (2005), *Algorithmic Learning in a Random -World*. Fundamento original de conformal prediction. - -[2] Angelopoulos and Bates (2023), *Conformal Prediction: A Gentle Introduction*. -Entrada pedagógica moderna para explicar cobertura distribution-free. - -[3] Bates et al. (2021), *Distribution-Free, Risk-Controlling Prediction Sets*. -Base RCPS para hablar de control de riesgo más allá de intervalos clásicos. - -[4] Angelopoulos et al. (2025), *Learn then Test*. Marco para calibrar -algoritmos predictivos con control de riesgo. - -[5] Angelopoulos et al. (2024), *Conformal Risk Control*. Generaliza control de -riesgo conformal para pérdidas monotónicas acotadas. - -[6] Bertsimas and Sim (2004), *The Price of Robustness*. Lenguaje clásico de -presupuesto de robustez y costo de protección. - -[7] Johnstone and Cox (2021), *Conformal Uncertainty Sets for Robust -Optimization*. Puente temprano entre CP y robust optimization. +| `point score` | PD calibrada usada por el LP base. | Probabilidad causal individual. | +| `upper score` | Endpoint superior del intervalo binario. | Garantía de riesgo del funded set. | +| `coverage bound` | Sharp all-candidate completion bound con outcomes no resueltos. | Intervalo de confianza. | +| `objective-matched ruler` | Mantiene un floor común de objetivo plug-in. | Igual oportunidad económica realizada. | +| `normalized-score ruler` | Mantiene una relajación relativa positive-affine-invariant. | Mismo opportunity cost. | +| `C0` | Copia el mismo cap numérico. | Comparador justo por construcción. | +| `C2` | Iguala el momento point-score del allocation guardrail. | Igual feasible set o outcomes. | +| `standardized payoff` | Payoff coherente `(1-p)r-pLGD` y su endpoint realizado simplificado. | IRR, discounted cash flow o welfare. | +| `support envelope` | Extremos sharp sobre caps outcome-free declarados. | Universalidad sobre cualquier comparador. | +| `endpoint sensitivity` | Cambia cuándo un outcome terminal se considera observable. | Sensibilidad del label usado para fit. | +| `fit-label sensitivity` | Cambia el lag de disponibilidad del label de calibración. | Cambio del cutoff de evaluación. | +| `structural sensitivity` | Reporta el producto completo budget--purpose-cap--LGD sin seleccionar escenario. | Robustez universal, policy recommendation o búsqueda de ganador. | -[8] Patel, Rayan and Tewari (2024), *Conformal Contextual Robust Optimization*. -Extiende la conexión CP-RO en problemas contextuales. - -[9] Sun, Liu and Li (2024), *Predict-then-Calibrate*. Comparador cercano para LP -contextuales robustos con calibración posterior. - -[10] Bao et al. (2025), *Optimal Model Selection for Conformalized Robust -Optimization (CROMS)*. Motiva el selector conformal decision-aware. - -[11] Yeh et al. (2025/2026), *End-to-End Conformal Calibration for Optimization -Under Uncertainty*. Dirección futura para calibración conformal integrada a -decisión. - -[12] Yeh et al. (2025), *Conformal Risk Training*. Motiva OCE/CVaR y control de -cola como extensión P2. - -[13] Yang and Jin (2026), *Multi-Distribution Robust Conformal Prediction*. -Motiva validación robusta ante múltiples fuentes o distribuciones. - -[14] Liu, Dobriban and Orabona (2026), *Online Conformal Prediction vía -Universal Portfolio Algorithms*. Motiva recalibración online bajo streaming. - -[15] Elmachtoub and Grigas (2022), *Smart Predict, then Optimize*. Base de SPO+ -y decision-focused learning. - -[16] Capitaine et al. (2026), *Online Decision-Focused Learning*. Extiende DFL a -decisiones online y drift. - -[17] Powell (2026), *Sequential Decisión Analytics and Modeling*. Da el lenguaje -CFA/VFA/DLA para clasificar CRPTO como política parametrizada auditable. - -La biblioteca de tesis (`Papers_tesis`) añade las siguientes fuentes, ya -integradas en `book/references.bib` y citadas en los capítulos donde aplican. - -**Fundamentos conformales y límites de cobertura.** - -[18] Angelopoulos, Barber and Bates (2024), *Theoretical Foundations of -Conformal Prediction*. Monografía de referencia para la validez finita bajo -intercambiabilidad; respalda [1]--[2]. - -[19] Tibshirani et al. (2019), *Conformal Prediction Under Covariate Shift*. -Reponderación por likelihood ratio bajo shift; base teórica de los stress A6 y -del diagnóstico multi-distribución A23. - -[20] Barber et al. (2021), *The Limits of Distribution-Free Conditional -Predictive Inference*. Justifica por qué el paper no promete cobertura -condicional perfecta, solo marginal/Mondrian. - -[21] Bhattacharyya and Barber (2026), *Group-Weighted Conformal Prediction*. -Cobertura por grupos cuando el shift está determinado por grupos; vecino directo -de la cobertura Mondrian y del diagnóstico A23. - -[22] Guan (2023), *Localized Conformal Prediction*. Marco de inferencia local; -alternativa a la partición Mondrian para cobertura condicional aproximada. - -[23] Einbinder et al. (2024), *Label Noise Robustness of Conformal Prediction*. -Cuándo los conjuntos cubren la etiqueta limpia bajo ruido; sustenta el stress de -flips adversariales A11. - -**Control de riesgo y medidas de cola.** - -[24] Angelopoulos (2026), *Conformal Risk Control for Non-Monotonic Losses*. -Extiende CRC a pérdidas no monótonas y multidimensionales; soporte teórico -directo de la re-optimización por CVaR/OCE de A22. - -[25] Rockafellar and Uryasev (2000), *Optimization of Conditional Value-at-Risk*. -Base de CVaR; define la cantidad de cola usada en A12 y A22. +: Diccionario operativo del CRPTO {#tbl-crpto-operational-dictionary} -[26] Ben-Tal and Teboulle (2007), *An Old-New Concept of Convex Risk Measures -(OCE)*. Base del optimized certainty equivalent entrópico de A12 y A22. +## Banco de defensa ante reviewers {#sec-crpto-reviewer-defense-bank} -**Optimización robusta conformal y decision-focused.** +| Objeción probable | Respuesta gobernada | Evidencia canónica | +|---|---|---| +| “El resultado es negativo.” | Sí, y es informativo: localiza dos fallos de inferencia en una cadena decision-focused y ofrece un protocolo para detectarlos. | Body, Discussion y claim registry. | +| “Entonces CRPTO ya no conecta ML, CP y optimización.” | Sí los conecta. Precisamente por tratar el sistema completo muestra que las propiedades no se heredan entre capas. | Method + Audit Theory. | +| “¿Por qué no elegir la mejor ventana o policy?” | Porque los mismos outcomes usados para evaluar no pueden seleccionar el objeto sin otro procedimiento de validez. | Outcome-free freezes y no-selector boundary. | +| “¿Los cinco modelos forman un leaderboard?” | No. Son controles de cobertura; solo CatBoost primario entra al LP. | Tabla de credit controls. | +| “¿WOE/IV mejora la auditabilidad?” | Aporta una especificación inspeccionable y monotonicidad, pero no es novedad ni ganador. | 45 fits OptBinning óptimos y métricas descriptivas. | +| “¿Split conformal no garantiza 90%?” | Bajo exchangeability, sí para el objeto correspondiente. Aquí el transporte temporal falla en las 40 celdas evaluadas. | Coverage tables y teoría de shift. | +| “¿Por qué conservar outcomes no resueltos?” | Excluirlos reconstruiría retrospectivamente el menú. Los sharp bounds mantienen el universo original. | Endpoint census y common-outcome bounds. | +| “¿El archivo representa septiembre de 2020?” | No. La observabilidad se reconstruye conservadoramente y 12,076 casos quedan no resueltos. | Raw-data audit y endpoint protocol. | +| “¿W7--W8 prueba una phase transition causal?” | No. El teorema de score constante da un mecanismo y el stratum observado es evidencia compatible. | Binary geometry proposition + label-lag sensitivity. | +| “¿El ruler normalizado es el correcto?” | Es scale-invariant, pero no iguala opportunity cost. Se reporta junto al objective-matched ruler sin seleccionar uno. | Two-ruler grid. | +| “¿Exact support resuelve toda la comparación?” | Solo sobre el soporte de caps declarado; no identifica una frontera conjunta continua. | 3,067 caps y basis-endpoint theorem. | +| “¿El resultado depende del presupuesto, concentración o LGD?” | Las proporciones cambian, pero cada uno de los 36 escenarios retiene celdas adversas de default y miscoverage; también hay celdas favorables en muchos escenarios. | Structural sensitivity V6, Tables S9D--S9E. | +| “¿Puede llamarse robusta la policy?” | No se promueve policy. `Risk-aware` describe la entrada de incertidumbre sin atribuir garantía downstream. | Claim boundary. | +| “¿El payoff es retorno económico?” | Es un endpoint estandarizado que omite timing, prepago, recoveries, fees y capital. | Method y Limitations. | +| “¿Hay evidencia externa o fair-lending?” | No para el claim activo. Un archivo histórico de aceptados no identifica esos objetos. | Ethical implications y Limitations. | -[27] Zhao, Jiang and Qi (2026), *Conformal Robust Optimization and Satisficing*. -Equivalencia CRO $\equiv$ CRS y cobertura finita $O(n^{-1/2})$; motiva el bound y -las tablas de satisficing A13/A20. +: Banco gobernado de respuestas a reviewers del CRPTO {#tbl-crpto-reviewer-defense-bank} -[28] Donti, Amos and Kolter (2017), *Task-based End-to-End Model Learning*. Raíz -del decision-focused learning que SPO+ y CRPTO toman como punto de partida. +## Qué entra al paper -[29] Schutte, Postek and Yorke-Smith (2024), *Robust Losses for Decision-Focused -Learning*. Regret robusto frente a incertidumbre aleatoria/epistémica; -complementa el comparador SPO+. +| Material | Destino | Motivo | +|---|---|---| +| Universo, roles y endpoint | Body | Sin information boundary no hay estimand defendible. | +| Cinco-model coverage audit | Body | Establece que el fallo no es exclusivo de CatBoost. | +| Binary geometry W1--W8 | Body | Explica por qué ancho y cobertura pueden cambiar abruptamente. | +| Two-ruler endpoint tracks | Body | Resultado downstream principal sin seleccionar policy. | +| Exact statements | Body, con proofs al supplement | Separa resultados matemáticos de outcomes. | +| WOE/IV, PSI, label lag y endpoint timing | Supplement | Controles y sensibilidades, no contribuciones autónomas. | +| Structural sensitivity | Body y censo completo en supplement | Evita que el baseline estilizado cargue por sí solo la conclusión downstream. | +| Named comparators y exact cap support | Supplement, resumen en body | Evita que la grid finita se lea como familia cerrada. | +| Simulation | Supplement | Ilustra geometría; no sostiene claims de portafolio. | +| Lineage y runs detenidos | Supplement/editor-only | Procedencia necesaria, no narrativa central. | -**Multi-distribución y online.** +: Destino editorial probable del material del libro {#tbl-crpto-paper-placement} -[30] Angelopoulos, Jordan and Tibshirani (2025), *Gradient Equilibrium in Online -Learning*. Debiasing post-hoc de predicciones black-box bajo shift arbitrario; -respalda la lectura online de A24 junto a [14]. +## Paquete journal-ready -**Crédito, fairness y datos.** +| Capa | Objeto activo | Papel editorial | +|---|---|---| +| Body | `paper/CRPTO_ijds.qmd` | Argumento mínimo y resultados centrales. | +| Supplement | `paper/supplement_ijds.qmd` | Pruebas, grids completas, sensibilidades y lineage. | +| TeX | `scripts/build_ijds_submission_tex.py` | Genera la fuente oficial; no se edita a mano. | +| Evidence | `ijds_binary_geometry_frontier_v4_evidence.json` | Autoridad numérica única. | +| Claim ledger | `ijds_claim_ledger.yaml` | Claims cualitativos, alcance y forbidden inferences. | +| Registry | `ijds_active_evidence_sources.yaml` | Identidades, roles, hashes y punteros DVC. | +| Tables | `crpto_ijds_v4_table*.csv` | Exportaciones derivadas, nunca autoridad independiente. | +| Figures | Tres objetos `crpto_ijds_v4_fig*` en PNG/PDF | Cobertura, geometría y envelopes. | +| Tests | Active claim sync, V4 claim sync, publication integrity | Rechazan drift de texto, grids y fuentes. | -[31] Albanesi and Vamossy (2024), *Credit Scores: Performance and Equity*. -Benchmark de un score frente a un modelo ML de default; contexto de desempeño y -equidad para la lectura de crédito. +: Material journal-ready activo del CRPTO {#tbl-crpto-journal-ready-package} -[32] Jagtiani and Lemieux (2019), *The Roles of Alternative Data and Machine -Learning in Fintech Lending* (LendingClub). Contexto del dataset y del rol de -datos alternativos en el originador estudiado. +## Uso del corpus de literatura {#sec-crpto-papers-tesis-deep-audit} -[33] Consumer Financial Protection Bureau (2014), *Using Publicly Available -Information to Proxy for Race and Ethnicity (BISG)*. Método proxy que sustenta la -auditoría de fair lending sin atributos protegidos directos. +`Papers_tesis` es una biblioteca de contraste, no una votación de claims. Cada +fuente entra por la pregunta que ayuda a delimitar: -[34] FinRegLab (2023), *Explainability and Fairness in Machine Learning for -Credit Underwriting*. Marco de política para explicabilidad y fairness que -encuadra el checkpoint MRM/fair-lending. +| Familia | Uso en CRPTO | Lo que no se importa | +|---|---|---| +| Decision quality y cost-aware calibration [@fernandezloria2022causaldecision; @yang2025costaware] | Separa calidad de score y calidad de acción. | Identificación causal para este archivo. | +| Conformal bajo exchangeability y shift [@vovk2005; @angelopoulos2023; @barber2021limits] | Define el objeto de cobertura y sus límites. | Cobertura condicional o temporal automática. | +| Valid selection y decision-risk calibration [@hegazy2025valid_selection_conformal_sets; @bao2025croms; @zhou2026creme] | Muestra qué diseño requeriría seleccionar un set o decisión. | Sus garantías con solo 11 menús de desarrollo. | +| Conformal robust optimization [@johnstone2021; @patel2024] | Sitúa el puente CP--OR. | Selected-set validity para CRPTO. | +| Credit scoring, profit y madurez [@lessmann2015; @ausset2022censoring; @li2023online_loans] | Motiva payoff, censoring y observabilidad. | IRR, survival o cash-flow claims no estimados. | +| WOE/IV y monotonicidad [@navaspalencia2020] | Construye controles auditables. | Superioridad de scorecard. | +| AI--OR [@wiberg2025ai_or] | Justifica estudiar el sistema completo. | Novedad separada de ML, CP o LP. | + +: Uso gobernado del corpus local {#tbl-crpto-papers-tesis-audit-artifacts} + +No se fija aquí un conteo de PDFs porque el corpus puede crecer. La literatura +se promueve al related work solo si cambia el gap, el estimand, una limitación o +la interpretación de evidencia activa. + +## Checklist antes de aceptar una frase + +1. ¿El sustantivo nombra el objeto correcto: candidato, allocation, endpoint o + support? +2. ¿El número existe en el evidence manifest activo? +3. ¿La frase distingue fit-label timing de evaluation-endpoint timing? +4. ¿Nombra ruler y coordenada cuando habla de dirección downstream? +5. ¿Conserva outcomes no resueltos o declara el bound usado? +6. ¿Evita selección por signo, causalidad, prospective language y deployment? +7. ¿La limitación aparece en la misma sección, no escondida al final? + +Si alguna respuesta es no, la frase todavía no está lista para IJDS. diff --git a/book/chapters/17-pipeline-datos-features.qmd b/book/chapters/17-pipeline-datos-features.qmd index b30ceee..58d1f48 100644 --- a/book/chapters/17-pipeline-datos-features.qmd +++ b/book/chapters/17-pipeline-datos-features.qmd @@ -56,16 +56,16 @@ Las herramientas se eligieron con tres criterios: import pandas as pd ml_stack = [ - {"Librería": "CatBoost 1.2.8", "Rol": "Modelo PD (gradient boosting)", + {"Librería": "CatBoost 1.2.10", "Rol": "Modelo PD (gradient boosting)", "Alternativas": "XGBoost, LightGBM", "Justificación": "Manejo nativo de categorías y NaN, ordered boosting reduce overfitting, mejor calibración out-of-the-box para credit scoring"}, - {"Librería": "scikit-learn 1.6.1", "Rol": "Baseline LR, calibración, métricas", + {"Librería": "scikit-learn 1.9.0", "Rol": "Baseline LR, calibración, métricas", "Alternativas": "—", "Justificación": "Estándar de facto para ML en Python. Provee CalibratedClassifierCV, IsotonicRegression, y todas las métricas de evaluación"}, - {"Librería": "Optuna 4.7", "Rol": "HPO (320 trials CatBoost)", + {"Librería": "Optuna 4.9.0", "Rol": "HPO histórico (no se relanza para el paper activo)", "Alternativas": "Hyperopt, Ray Tune, Optuna", "Justificación": "TPE sampler es state-of-the-art para HPO con presupuesto limitado, pruning vía MedianPruner, integración nativa con CatBoost"}, - {"Librería": "OptBinning 0.19+", "Rol": "WOE/IV binning", + {"Librería": "OptBinning 0.21.0", "Rol": "Controles WOE/IV IJDS aislados", "Alternativas": "scorecardpy, binning manual", "Justificación": "Binning óptimo basado en programación matemática, soporta constraints regulatorios (monotonía, máximo de bins)"}, {"Librería": "SHAP 0.48", "Rol": "Explicabilidad (TreeExplainer)", @@ -586,96 +586,47 @@ Todas las métricas reportadas en este libro --- AUC, Brier, cobertura conformal ## WOE, IV y OptBinning -### Motivación del Encoding WOE - -Las variables categóricas en credit scoring --- como el grado de riesgo, el propósito del préstamo o el tipo de vivienda --- no pueden usarse directamente en modelos lineales ni en algunos modelos de ensemble. El **Weight of Evidence** (WOE) es la transformación estándar en la industria bancaria porque produce una codificación numérica con propiedades deseables: - -1. **Monotonicidad**: La relación entre la variable WOE y el log-odds de default es lineal por construcción, lo que facilita la interpretación y la validación regulatoria. -2. **Poder predictivo cuantificable**: El **Information Value** (IV) asociado mide directamente cuánta información aporta cada variable para discriminar defaults de no-defaults (ver `sec-ml-foundations` para las fórmulas). -3. **Tratamiento natural de categorías raras**: El binning agrupa categorías con pocos casos, evitando estimaciones inestables. - -::: {.callout-note} -## WOE vs. One-Hot Encoding vs. Target Encoding -- **One-Hot**: Crea $k-1$ columnas binarias para $k$ categorías. Explota la dimensionalidad para variables con muchas categorías (Lending Club tiene 35 sub-grados). -- **Target Encoding**: Reemplaza categorías por la media del target. Sufre de **data leakage** si no se implementa con regularización cuidadosa. -- **WOE**: Reemplaza categorías por el log-odds ratio, calculado sobre bins óptimos. Es el estándar regulatorio porque produce una transformación interpretable, monótona y sin leakage cuando se calcula solo sobre el set de entrenamiento. - -CatBoost maneja categorías nativamente sin necesidad de WOE, pero las variables WOE se incluyen igualmente porque: (a) son necesarias para el baseline de regresión logística, (b) sirven como features adicionales para CatBoost, y (c) se usan como confusores en la estimación causal. -::: - -### OptBinning: Binning Óptimo por Programación Matemática - -El cálculo de WOE requiere primero agrupar los valores de la variable en **bins** (intervalos para numéricas, grupos para categóricas). La calidad de los bins determina la calidad del WOE resultante. Los enfoques tradicionales (equal-width, equal-frequency, chi-squared merging) son heurísticos y no garantizan optimalidad. - -**OptBinning** resuelve el problema de binning como un programa de optimización con restricciones (constraint programming, solver `cp`), garantizando bins que maximizan el IV sujeto a restricciones regulatorias: - -- **Monotonía**: El WOE debe ser monótono creciente o decreciente a través de los bins. -- **Número máximo de bins**: Típicamente 5--10 para interpretabilidad. -- **Tamaño mínimo de bin**: Cada bin debe contener al menos un porcentaje mínimo de observaciones. - -La implementación en `src/features/feature_engineering.py` utiliza OptBinning con caching: - -```python -from optbinning import OptimalBinning - -optb = OptimalBinning( - name=feature, - dtype="numerical" | "categorical", - solver="cp", # constraint programming -) -optb.fit(X_train[feature], y_train) -woe_values = optb.transform(X[feature], metric="woe") -``` - -Los objetos OptBinning ajustados se persisten en `models/woe_optbinning_{feature}.pkl` para reutilización en inferencia. Adicionalmente, el pipeline exporta las **tablas de binning** como CSV a `reports/binning_tables/{feature}_bins.csv`, proporcionando una traza de auditoría completa que documenta los límites de cada bin, el WOE asignado, la tasa de eventos y el IV por bin --- un requisito de gobernanza para validación de modelos (SR 11-7). - -### Variables WOE del Proyecto - -No todas las variables categóricas merecen una transformación WOE. En el proyecto priorizamos las que reaparecen en más de una capa del pipeline: modelado de PD, particiones conformales, lectura regulatoria e incluso control causal. La tabla siguiente resume justamente ese criterio de reutilización, no solo su conveniencia estadística. - -El proyecto calcula WOE para tres variables categóricas clave: - -```python - -import pandas as pd - -woe_vars = [ - {"Variable original": "grade", "Variable WOE": "grade_woe", - "Categorías": "A, B, C, D, E, F, G (7 grados)", - "Uso downstream": "Confusor causal, partición Mondrian, IFRS9 staging"}, - {"Variable original": "purpose", "Variable WOE": "purpose_woe", - "Categorías": "debt_consolidation, credit_card, home_improvement, etc. (14 categorías)", - "Uso downstream": "Confusor causal, feature PD"}, - {"Variable original": "home_ownership", "Variable WOE": "home_ownership_woe", - "Categorías": "MORTGAGE, RENT, OWN, OTHER (4 categorías)", - "Uso downstream": "Confusor causal, feature PD"}, -] - -pd.DataFrame(woe_vars) -``` - -### Ranking por Information Value - -El IV de cada variable se calcula como subproducto del binning óptimo y sirve como criterio de selección de features. La interpretación estándar (ver `tbl-iv-interpretation`) guía la decisión de qué variables retener: - -::: {.callout-tip} -## Regla práctica de IV -Variables con IV > 0.50 son sospechosas de contener leakage --- su poder predictivo es "demasiado bueno para ser verdad". En el pipeline de limpieza del capítulo de ingesta y linaje, las 35 columnas de leakage fueron removidas *antes* del cálculo de WOE, por lo que las variables restantes tienen IV en rangos plausibles. Las variables WOE de grade, purpose y home_ownership tienen IV típicamente en el rango 0.10--0.40, indicando poder predictivo medio a fuerte. -::: - -El IV también se usa como criterio de ordenamiento en los notebooks de análisis exploratorio (NB02), donde las variables se rankean por IV descendente para identificar las más informativas antes de la selección final de features. - -::: {.figure-stack} - -![Ranking IV de variables candidatas para el pipeline de feature engineering.](../assets/figures/notebooks/iv_ranking.png){fig-alt="Gráfico de barras que ordena variables candidatas por Information Value para priorizar señales del pipeline de features."} - -La figura `fig-iv-ranking` permite aterrizar el criterio de selección en el problema real de Lending Club: no todas las columnas aportan lo mismo, y varias de las señales fuertes coinciden con intuiciones clásicas de riesgo, como grado, tasa y proxies de carga financiera. - -![Binning WOE supervisado para las variables más importantes del bloque de feature engineering.](../assets/figures/notebooks/woe_binning_top6.png){fig-alt="Panel de binning WOE supervisado para las seis variables principales, con bins ordenados y evidencia de separación de riesgo."} - -::: - -La figura `fig-woe-top6` muestra algo que suele perderse cuando solo vemos tablas: el binning no se usa por estética, sino para convertir relaciones ruidosas en una señal más ordenada y más fácil de explicar ante negocio o validación de modelo. +WOE convierte cada nivel o intervalo en una razón de evidencia entre buenos y +malos; IV resume la separación aportada por la variable. La transformación es +supervisada, por lo que sus bins, conteos y valores deben ajustarse únicamente +con el bloque de entrenamiento y persistirse para los bloques posteriores. +Monotonía no viene garantizada por cualquier WOE: depende del algoritmo de +binning y de las restricciones declaradas. + +### Dos implementaciones con roles distintos + +El repositorio conserva dos carriles que no deben confundirse: + +1. **Pipeline canónico histórico.** `src/features/feature_engineering.py` ajusta + encoders WOE train-only. Para variables numéricas usa hasta seis bins por + cuantiles con `pandas.qcut`; para categóricas conserva niveles observados; + aplica suavizado de 0.5 y persiste `woe_encoders.pkl` junto con los IV. Este + carril no usa OptBinning y no promete bins monotónicos óptimos. +2. **Controles IJDS aislados.** `src/ijds_audit/credit_controls.py` usa + `optbinning.BinningProcess` con tendencia monotónica automática, dos a ocho + bins y participación mínima de 5%. Ajusta un scorecard de 26 señales de + prestatario/plataforma y otro de 19 señales borrower-only. Sus modelos, + tablas de bins, coeficientes, IV y PSI viven en rutas experimentales DVC y + no reemplazan el champion histórico. + +Esta separación es intencional. Refactorizar el carril histórico para usar +OptBinning cambiaría features y artefactos protegidos sin mejorar el estimando +del paper activo. El control IJDS permite estudiar auditabilidad y dependencia +de señales de pricing con una ruta nueva, predeclarada y reproducible. + +### Evidencia activa + +Los 45 problemas de binning IJDS alcanzan estado `OPTIMAL`. En el scorecard de +plataforma lideran la interacción tasa-grado (IV 0.337569), `sub_grade` +(0.319325), `grade` (0.299544) e `int_rate` (0.278429). En borrower-only lideran +FICO (0.213574), consultas recientes (0.170864), propósito (0.088878), uso +revolving (0.073903) y recencia de mora (0.049288). + +Estas cifras son diagnóstico, no un selector. El scorecard borrower-only tiene +AUC OOT 0.612712 y el de plataforma 0.633023; ambos fallan el objetivo de +cobertura conformal en las ocho ventanas. Por eso WOE/IV fortalece la auditoría +de dependencia de especificación, pero no se presenta como novedad central ni +como modelo promovido. ::: {.source-note} **Procedencia:** `book/chapters/05-feature-engineering/05b-derived-features-ratios.qmd` diff --git a/book/chapters/glosario.qmd b/book/chapters/glosario.qmd index 69a7221..6eb23dd 100644 --- a/book/chapters/glosario.qmd +++ b/book/chapters/glosario.qmd @@ -198,4 +198,4 @@ Términos técnicos usados en el manuscrito y el dossier extendido. Se ordenan a ## W **WoE (Weight of Evidence)** -: Transformación supervisada de variables categóricas: `WoE(x) = log(P(x|y=1) / P(x|y=0))`. CRPTO usa OptBinning para WoE de `grade`, `purpose`, `home_ownership`. +: Transformación supervisada por bins que contrasta la distribución de buenos y malos. El pipeline histórico usa encoders train-only con bins por cuantiles; los controles IJDS aislados usan OptBinning para scorecards de plataforma y borrower-only. Ninguno de esos controles entra al optimizador de portafolio. diff --git a/book/includes/_build-info.qmd b/book/includes/_build-info.qmd deleted file mode 100644 index c4dbb67..0000000 --- a/book/includes/_build-info.qmd +++ /dev/null @@ -1,3 +0,0 @@ -::: {.build-info} -Build: `2a9b5e9` | Rama: `codex/ijds-refactor-lab-2026-07-08` | Actualizado: `2026-07-09` -::: diff --git a/book/includes/_chapter-landing.qmd b/book/includes/_chapter-landing.qmd deleted file mode 100644 index 24988fb..0000000 --- a/book/includes/_chapter-landing.qmd +++ /dev/null @@ -1,62 +0,0 @@ -```{=html} -
- -

Introducción

-

Problema, gap aplicado y framing CRPTO.

- - -

Marco teórico

-

Alpha-Gamma, Markov, tightening condicional y uncertainty sets.

- - -

Metodología

-

Pipeline CP → robustez → evidencia CRPTO, SPO+ y baselines.

- - -

Resultados

-

Pareto, región robusta, stress tests, funded set y comparadores.

- - -

Guía editorial

-

Claims, preguntas de reviewer, audiencias y referencias locales.

- - -

Apéndice journal

-

A12-A27, figuras CRPTO y checklist reproducible.

- - -

Trazabilidad

-

Mapa claim → artefacto → script → test y política de freeze.

- - -

Puentes regulatorios

-

SICR, temporal uncertainty y future work fuera del champion.

- -
-``` - - diff --git a/book/includes/_paper-autocontenido.qmd b/book/includes/_paper-autocontenido.qmd deleted file mode 100644 index e9abb90..0000000 --- a/book/includes/_paper-autocontenido.qmd +++ /dev/null @@ -1,4 +0,0 @@ -::: {.callout-important} -## Paper autocontenido -Esta landing funciona como la portada editorial del dossier: explica el claim central, orienta la lectura y deja explícito qué parte del argumento vive en el cuerpo IJDS, el supplement, la tesis y la capa de reproducibilidad. -::: diff --git a/book/includes/_whole-game-close.qmd b/book/includes/_whole-game-close.qmd deleted file mode 100644 index 55bfca5..0000000 --- a/book/includes/_whole-game-close.qmd +++ /dev/null @@ -1,10 +0,0 @@ -::: {.callout-tip .chapter-close} -## Qué debes quedarte -{{< meta close_takeaway >}} - -## Esto conecta con… -{{< meta close_connects >}} - -## Dónde se reutiliza esta señal o artefacto -{{< meta close_reuse >}} -::: diff --git a/book/includes/apa-figures.html b/book/includes/apa-figures.html deleted file mode 100644 index feff0c2..0000000 --- a/book/includes/apa-figures.html +++ /dev/null @@ -1,73 +0,0 @@ - diff --git a/book/index.qmd b/book/index.qmd index 1fe0819..c0d3644 100644 --- a/book/index.qmd +++ b/book/index.qmd @@ -7,88 +7,113 @@ ::: {.whole-game-intro} Pregunta central -¿Qué ganamos cuando la incertidumbre predictiva deja de ser un diagnóstico y se vuelve una entrada explícita del optimizador? Este libro independiente abre el dossier CRPTO alrededor de esa tesis y conserva tanto el material paper-ready como la evidencia extendida para journal, apéndices y respuestas a reviewers. +¿Qué puede identificarse cuando una puntuación de riesgo, un intervalo conformal +binario y un optimizador mensual se conectan bajo transporte temporal, +resultados censurados y comparadores que no comparten una escala natural? ::: ::: {.callout-note .mini-abstract} ## Mini-abstract -**Problema**: la mayoría de pipelines de crédito optimizan con PD puntual y tratan la incertidumbre como diagnóstico tardío. -**Método**: PD calibrada, bandas conformales Mondrian y optimización robusta sobre el conjunto resultante. -**Hallazgo**: el resultado clave no es un AUC aislado, sino la posibilidad de cuantificar explícitamente el trade-off entre retorno, cobertura, robustez y auditabilidad. +**Problema**: una garantía predictiva no se hereda automáticamente por la +decisión que selecciona un portafolio, y copiar el mismo umbral entre dos scores +no fija la misma oportunidad de inversión. + +**Método**: CRPTO audita un universo exhaustivo y status-independent de 640,543 +préstamos a 36 meses, cinco especificaciones predictivas, ocho ventanas +residuales completas, dos reglas outcome-free de comparación, resultados no +resueltos mediante bounds sharp, soporte exacto de caps y una sensibilidad +completa de 36 escenarios de estructura de portafolio. + +**Hallazgo**: la cobertura candidata no transporta en las 40 celdas declaradas; +la geometría residual binaria cambia cerca del umbral de prevalencia; y la +dirección downstream depende del ruler, la coordenada y el soporte del +comparador. Cada escenario estructural conserva celdas adversas de default y +miscoverage, pero ninguno es uniformemente favorable o adverso. No se promueve +una policy, escenario ni modelo ganador. ::: -![Las seis etapas del pipeline CRPTO --- datos y split temporal, PD calibrada, conformal Mondrian, conjunto de incertidumbre, LP robusto y política promovida con bound exacto --- anotadas con métricas del cierre IJDS pool93: retorno $184,832.48, $V=0.035350$, $\Gamma_{\text{CP}}=0.162616$ y frontera finita A35/A36.](assets/figures/editorial/diagrama-crpto.png){#fig-crpto-overview fig-alt="Diagrama maestro del pipeline CRPTO en seis etapas desde datos hasta política promovida auditable."} - -La superficie IJDS activa promueve el punto pool93 body/default: retorno -`$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, -umbral Markov `0.345084`, `alpha_grid_pass = 8/8` y frontera finita A35. A40 -cuantifica el trade-off contra point-PD. Los cierres anteriores quedan -como procedencia histórica, no como baseline operativo vigente. - -::: {.callout-warning} -Algunos capítulos largos de tesis conservan análisis históricos del champion -anterior para trazabilidad. La fuente paper-facing actual es -`paper/CRPTO_ijds.qmd`, su suplemento y las tablas A35--A40. +```{mermaid} +%%| label: fig-crpto-overview +%%| fig-cap: "Arquitectura activa de la auditoría CRPTO." +flowchart LR + A["Archivo crudo: 2,925,493 filas"] --> B["Diseño elegible: 640,543 préstamos"] + B --> C["Cinco scores de cobertura"] + C --> D["Ocho recetas conformales"] + D --> E["LP mensual con CatBoost primario"] + E --> F["Dos rulers y tres coordenadas"] + F --> G["Endpoint reconstruido y bounds sharp"] + G --> H["36 escenarios estructurales"] + H --> I["Soporte exacto y límites de claim"] +``` + +::: {.callout-important} +## Contrato activo +La única narrativa paper-facing está en `paper/CRPTO_ijds.qmd` y +`paper/supplement_ijds.qmd`. La autoridad numérica es +`reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`, cuyos insumos +están hash-locked por `configs/ijds_active_evidence_sources.yaml`; el alcance y +los límites cualitativos se verifican con `configs/ijds_claim_ledger.yaml`. + +Los materiales `pool93`, `compact-v7`, selected-policy y A1--A40 se conservan +solo como procedencia histórica. No prueban los claims del manuscrito activo. ::: ## Mapa editorial -Este libro es el companion vivo del paper y de la tesis. No compite con el -cuerpo IJDS: lo respalda. Conserva explicaciones largas, decisiones -metodológicas, vínculos con artefactos, preguntas de reviewer, tablas y figuras -que el manuscrito comprime en una historia corta y que la tesis desarrolla como -superconjunto académico. - -| Superficie | Qué defiende | Evidencia visible | +| Superficie | Función actual | Evidencia visible | |---|---|---| -| Paper IJDS | El claim estrecho: una PD calibrada congelada puede convertirse en una decisión robusta y auditable mediante conformal prediction. | Teoría mínima, Figura 1, frontera policy-aware A35, baseline A40, regret-auditability y réplica externa compacta. | -| Online supplement | Que el resultado no es frágil ni cherry-picked. | A3--A40, ablations, stress tests, MRM/fairness, funded-set audit, SPO+, tail risk, diagnósticos multi-distribución/online y Prosper/Freddie. | -| Tesis | El superconjunto académico y operativo del paper. | Fundamentos, arquitectura reproducible, gobernanza, trazabilidad, réplica multidataset, bibliografía extendida y future work controlado. | -| Reproducibilidad | Que las cifras vienen de artefactos versionados, no de edición manual. | DVC, manifests, tablas/figuras regenerables, tests de sincronía y guardrails de champion congelado. | +| Paper IJDS | Auditoría de identificación en la interfaz ML, conformal y optimización. | Universo status-independent, cobertura completa, mecanismo de geometría binaria, dos rulers y exact support. | +| Online supplement | Pruebas, censos completos, sensibilidades temporal y estructural, trazabilidad y fronteras de interpretación. | Apéndices A--I, endpoint reconstruido, WOE/IV, monotonicidad, PSI, label lag, 36 escenarios y solver audit. | +| Libro | Companion pedagógico del manuscrito activo. | Blueprint, claims, teoría, límites y defensa ante reviewers. | +| Reproducibilidad | Une cada afirmación activa con una fuente verificable. | Registry, DVC, evidence manifest, tests de publicación y TeX generado. | : Mapa de superficies del dossier CRPTO. -## Contrato de lectura +## Cómo leer el resultado -La lectura correcta es de claim a evidencia. El paper hereda las afirmaciones y -las tablas esenciales; el libro preserva el razonamiento, las cautelas y la -trazabilidad. Cuando una explicación es demasiado larga para IJDS, no se borra: -se mantiene aquí como soporte de tesis, supplement o respuesta a reviewer. +1. La PD y los intervalos conformales siguen siendo componentes del sistema, + pero su validez debe examinarse antes de atribuir propiedades al funded set. +2. El optimizador sigue siendo central: revela que score, ruler y coordenada + definen conjuntamente el contrafactual. +3. Un resultado negativo de transporte o de identificación no elimina el + método. Delimita qué puede defenderse y qué requeriría otro diseño. ## Rutas de lectura ::: {.route-grid} ::: {.route-card} -### En 20 minutos -[CRPTO en una página](chapters/00-crpto-en-una-pagina.qmd) → [Resultados](chapters/04-resultados.qmd) → [Discusión](chapters/05-discusion.qmd). -Ruta para captar problema, método, resultado y límite sin entrar al dossier completo. -::: - -::: {.route-card} -### Reviewer IJDS -[Mapa reviewer](chapters/25-reviewer-map.qmd) → [Introducción](chapters/01-introduccion.qmd) → [Marco teórico](chapters/02-marco-teorico.qmd) → [Metodología](chapters/03-metodologia.qmd) → [Resultados](chapters/04-resultados.qmd) → [Apéndice de robustez](chapters/07-apendice-robustez.qmd). -Ruta para verificar novelty, bound, evidencia central, artefactos y controles de robustez. +### Manuscrito activo +[Blueprint](chapters/06-blueprint-manuscrito.qmd) → +[Guía de claims](chapters/06b-guia-editorial-claims.qmd) → +`paper/CRPTO_ijds.qmd` → `paper/supplement_ijds.qmd`. ::: ::: {.route-card} -### Tesis -[Fundamentos de riesgo](chapters/15-fundamentos-riesgo-ml.qmd) → [Conformal y optimización](chapters/16-fundamentos-conformal-optimizacion.qmd) → [Pipeline](chapters/17-pipeline-datos-features.qmd) → [Gobernanza](chapters/21-gobernanza-explicabilidad-dataset.qmd) → [Bibliografía](chapters/24-bibliografia-crpto-actualizada.qmd). -Ruta para leer CRPTO como superconjunto académico del paper. +### Arquitectura activa +[Blueprint](chapters/06-blueprint-manuscrito.qmd) reúne el estimand, la +secuencia teórica, los objetos empíricos y el contrato reproducible. ::: ::: {.route-card} -### Reproducibilidad -[Trazabilidad](chapters/13-trazabilidad.qmd) → [Release](chapters/14-release.qmd) → [Pipeline](chapters/17-pipeline-datos-features.qmd) → [Literatura y entorno](chapters/22-literatura-trazabilidad-entorno.qmd). -Ruta para auditar artefactos, builds, guardrails y evidencia regenerable. +### Defensa y límites +[Guía de claims](chapters/06b-guia-editorial-claims.qmd) reúne vocabulario, +closest work, objeciones previsibles y fronteras de interpretación. ::: ::: +Los demás capítulos del antiguo libro de tesis permanecen en Git como +procedencia, pero no se renderizan: describen lineages retirados y no deben +leerse como evidencia del manuscrito activo. + ## Frontera del claim -| El libro puede explicar... | El paper debe afirmar... | +| Sí puede afirmarse | No puede afirmarse | |---|---| -| Fundamentos amplios de crédito, conformal prediction, robust optimization, MRM y fairness. | Un patrón post-hoc auditable para llevar intervalos conformales de una PD congelada a un LP robusto de portafolio. | -| Robustez extendida, stress tests, tail risk, satisficing, SPO+, diagnósticos online/multi-distribución y réplica externa. | Que esos análisis fortalecen el champion congelado sin reabrir la búsqueda ni vender un protocolo live o universal. | -| Bibliografía de tesis y future work. | Una frontera de novelty estrecha frente a P2P/Lending Club OR, conformal credit scoring, CP+RO, DFL/SPO+ y conformal finance. | - -: Regla editorial para mantener separado el dossier amplio del claim IJDS. +| Las 640,543 filas agotan la población elegible bajo el contrato declarado. | Que los 2.9 millones de registros comparten horizonte, esquema o estimand. | +| Los 40 upper bounds de cobertura candidata están por debajo de 0.90. | Invalidez conformal universal o validez del selected set. | +| La geometría residual binaria es sensible a prevalencia y saturación. | Que el cruce observado sea una explicación causal completa. | +| La dirección del contraste cambia con ruler, coordenada, ventana y soporte. | Un ganador universal, una policy promovida o dominancia económica. | +| Los bounds sharp exponen incertidumbre por outcomes no resueltos. | Que sean intervalos de confianza o identificación missing-at-random. | +| El exact support es válido sobre el soporte finito declarado. | Unicidad de una frontera continua o un comparador universal. | + +: Regla editorial del objeto IJDS activo. diff --git a/book/latex/preamble.tex b/book/latex/preamble.tex deleted file mode 100644 index ff669b5..0000000 --- a/book/latex/preamble.tex +++ /dev/null @@ -1,15 +0,0 @@ -% Custom LaTeX preamble for PDF output -\usepackage{booktabs} -\usepackage{amsmath} -\usepackage{amssymb} -\usepackage{mathtools} -\usepackage{float} - -% Quarto already loads babel via `lang: es`; only customize captions here. -\AtBeginDocument{% - \providecommand{\tablename}{Tabla}% - \providecommand{\listtablename}{Lista de tablas}% -} - -% Better table formatting -\renewcommand{\arraystretch}{1.2} diff --git a/book/references.bib b/book/references.bib index ac527f5..304c06e 100644 --- a/book/references.bib +++ b/book/references.bib @@ -625,11 +625,15 @@ @incollection{papadopoulos2002 } @inproceedings{patel2024, - author = {Patel, Neel and Rayan, Omar and Tewari, Ambuj}, + author = {Patel, Yash P. and Rayan, Sahana and Tewari, Ambuj}, title = {Conformal Contextual Robust Optimization}, - booktitle = {AISTATS (PMLR v238)}, + booktitle = {Proceedings of the 27th International Conference on Artificial Intelligence and Statistics}, + series = {Proceedings of Machine Learning Research}, + volume = {238}, + pages = {2485--2493}, + publisher = {PMLR}, year = {2024}, - url = {https://arxiv.org/abs/2310.10003} + url = {https://proceedings.mlr.press/v238/patel24a.html} } @article{plassier2024, @@ -1348,6 +1352,14 @@ @article{wang2026optimal_decision_prediction_sets url = {https://arxiv.org/abs/2602.00989} } +@article{zheng2026counterfactual_decision_sets, + author = {Zheng, Yurui and Jin, Ying}, + title = {Prediction Sets for Counterfactual Decisions: Coverage, Optimality, and Conformal Prediction}, + journal = {arXiv preprint arXiv:2607.02206}, + year = {2026}, + url = {https://arxiv.org/abs/2607.02206} +} + @article{huang2026oce_rcps, author = {Huang, Jiayi and Farzaneh, Amirmohammad and Simeone, Osvaldo}, title = {Optimized Certainty Equivalent Risk-Controlling Prediction Sets}, @@ -1408,6 +1420,17 @@ @article{morucci2022robust_matching_uncertainty doi = {10.1287/ijds.2022.0020} } +@article{navaspalencia2020, + author = {Navas-Palencia, Guillermo}, + title = {Optimal Binning: Mathematical Programming Formulation}, + journal = {Expert Systems with Applications}, + volume = {158}, + pages = {113508}, + year = {2020}, + doi = {10.1016/j.eswa.2020.113508}, + url = {https://arxiv.org/abs/2001.08025} +} + @article{chen2025cost_sensitive_adversarial, author = {Chen, Qiyuan and Al Kontar, Raed and Nouiehed, Maher and Yang, X. Jessie and Lester, Corey}, title = {Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial Data Augmentation}, @@ -1418,3 +1441,36 @@ @article{chen2025cost_sensitive_adversarial year = {2025}, doi = {10.1287/ijds.2022.0033} } + +@article{ausset2022censoring, + author = {Ausset, Guillaume and Cl{\'e}men{\c{c}}on, Stephan and Portier, Fran{\c{c}}ois}, + title = {Empirical Risk Minimization under Random Censorship}, + journal = {Journal of Machine Learning Research}, + volume = {23}, + number = {5}, + pages = {1--59}, + year = {2022}, + url = {https://jmlr.org/papers/v23/19-450.html} +} + +@article{li2023online_loans, + author = {Li, Zhiyong and Li, Aimin and Bellotti, Anthony and Yao, Xiao}, + title = {The Profitability of Online Loans: A Competing Risks Analysis on Default and Prepayment}, + journal = {European Journal of Operational Research}, + volume = {306}, + number = {2}, + pages = {968--985}, + year = {2023}, + doi = {10.1016/j.ejor.2022.08.013} +} + +@article{lyocsa2022profit, + author = {Ly{\'o}csa, {\v{S}}tefan and Va{\v{s}}ani{\v{c}}ov{\'a}, Petra and Hadji Misheva, Branka and Vateha, Marko D{\'a}vid}, + title = {Default or Profit Scoring Credit Systems? Evidence from European and {US} Peer-to-Peer Lending Markets}, + journal = {Financial Innovation}, + volume = {8}, + number = {1}, + pages = {32}, + year = {2022}, + doi = {10.1186/s40854-022-00338-5} +} diff --git a/book/references.qmd b/book/references.qmd deleted file mode 100644 index 7a4aba3..0000000 --- a/book/references.qmd +++ /dev/null @@ -1,4 +0,0 @@ -# Referencias - -::: {#refs} -::: diff --git a/configs/crpto_conformal_policy_sensitivity.yaml b/configs/crpto_conformal_policy_sensitivity.yaml deleted file mode 100644 index df9eb4d..0000000 --- a/configs/crpto_conformal_policy_sensitivity.yaml +++ /dev/null @@ -1,29 +0,0 @@ -policy: - target_coverage_90_min: 0.90 - target_coverage_95_min: 0.95 - min_group_coverage_90_min: 0.88 - max_avg_width_90: 0.80 - max_critical_alerts: 0 - max_total_alerts: 5 - max_warning_alerts: 5 - max_winkler_90: 1.20 - enable_compensated_winkler_90: true - compensated_winkler_90_max: 1.22 - compensated_min_coverage_90: 0.92 - compensated_min_group_coverage_90: 0.885 - compensated_max_avg_width_90: 0.80 - max_winkler_95: 1.50 - -policy_sensitivity: - max_winkler_90_values: [1.20, 1.22, 1.25] - -artifacts: - conformal_results_path: models/conformal_results_mondrian.pkl - group_metrics_path: data/processed/conformal_group_metrics_mondrian.parquet - backtest_monthly_path: data/processed/conformal_backtest_monthly.parquet - backtest_alerts_path: data/processed/conformal_backtest_alerts.parquet - intervals_path: data/processed/conformal_intervals_mondrian.parquet - -output: - policy_status_json: models/conformal_policy_sensitivity_status.json - policy_checks_parquet: data/processed/conformal_policy_sensitivity_checks.parquet diff --git a/configs/crpto_fairness_policy.yaml b/configs/crpto_fairness_policy.yaml deleted file mode 100644 index 0ce38dc..0000000 --- a/configs/crpto_fairness_policy.yaml +++ /dev/null @@ -1,51 +0,0 @@ -policy: - dpd_threshold: 0.10 # demographic parity difference < 0.10 - eo_gap_threshold: 0.11 # equalized odds gap < 0.11 (operational tolerance) - dir_threshold: 0.80 # disparate impact ratio > 0.80 (4/5ths rule) - prediction_threshold: 0.35 # fallback operational cutoff; artifact-driven policy remains authoritative - outcome_mode: approval # evaluate fairness on favorable decision (approval) - -threshold_policy: - use_artifact: true - artifact_path: models/decision_threshold.json - selected_threshold_key: selected_threshold - -decision_policy: - auto_select: true - artifact_path: models/fairness_decision_policy.json - -intersectional: - enabled: true - max_order: 2 - min_group_size: 500 - -attributes: - - name: home_ownership - column: home_ownership - - name: annual_inc_quartile - column: annual_inc - binning: quartile - - name: verification_status - column: verification_status - -artifacts: - test_predictions_path: data/processed/test_predictions.parquet - test_data_path: data/processed/test_fe.parquet - -output: - audit_parquet: data/processed/fairness_audit.parquet - frontier_parquet: data/processed/fairness_threshold_frontier.parquet - status_json: models/fairness_audit_status.json - threshold_semantics_json: models/threshold_semantics.json - -fairlearn_sidecar: - enabled: true - status_json: models/fairlearn_fairness_status.json - group_metrics_parquet: data/processed/fairlearn_group_metrics.parquet - bootstrap_samples: 10 - bootstrap_max_rows: 10000 - bootstrap_random_state: 42 - -threshold_frontier: - enabled: true - thresholds: [0.35, 0.40, 0.45, 0.50] diff --git a/configs/crpto_mrm_policy.yaml b/configs/crpto_mrm_policy.yaml deleted file mode 100644 index ded24b8..0000000 --- a/configs/crpto_mrm_policy.yaml +++ /dev/null @@ -1,81 +0,0 @@ -model: - name: CorePDCanonical - version: "1.0" - owner: Carlos Vergara - champion_artifact: models/pd_canonical.cbm - -governance: - validation_frequency_days: 90 - review_cadence: quarterly - -governance_checks: - ks_pvalue_min: 0.01 - cvm_pvalue_min: 0.01 - c2st_auc_max: 0.60 - max_feature_breach_ratio: 0.15 - c2st_max_rows_per_split: 50000 - performance_max_rows_per_split: 100000 - psi_bins: 10 - random_state: 42 - score_psi_max: 0.15 - auc_delta_max: 0.05 - brier_increase_max: 0.02 - calibration_gap_delta_max: 0.02 - explanation_rank_overlap_top10_min: 0.60 - explanation_shap_psi_max: 0.25 - reason_code_stability_min: 0.55 - explanation_min_rows_per_slice: 80 - -retraining_triggers: - psi_threshold: 0.25 - auc_degradation_threshold: 0.03 - coverage_degradation_threshold: 0.02 - -challenger: - criteria: - - metric: auc_roc - improvement_min: 0.005 - - metric: ece - improvement_min: 0.002 - promotion_requires: - - conformal_policy_pass: true - - fairness_audit_pass: true - - governance_checks_pass: true - modeling_constraints: - no_smote: true - monotonic_constraints_required: true - -approval_workflow: - steps: - - name: developer_self_validation - owner: developer - - name: independent_validation - owner: model_risk_team - - name: sign_off - owner: model_owner - -artifacts: - pipeline_summary: data/processed/pipeline_summary.json - conformal_status: models/conformal_policy_status.json - governance_status: models/governance_status.json - fairness_status: models/fairness_audit_status.json - pd_backtesting_status: models/pd_backtesting_status.json - bootstrap_validation_status: models/bootstrap_validation_status.json - monotonicity_status: models/monotonicity_audit_status.json - encoding_stability_status: models/encoding_stability_status.json - pd_validation_interpretation_status: models/pd_validation_interpretation_status.json - calibration_mapping_status: models/calibration_mapping_status.json - model_shift_status: models/model_shift_status.json - -governance_output: - drift_monitoring_path: data/processed/drift_monitoring.parquet - explanation_drift_path: data/processed/explanation_drift.parquet - fairness_status_path: models/fairness_audit_status.json - fairness_frontier_path: data/processed/fairness_threshold_frontier.parquet - challenger_promotion_report_path: models/challenger_promotion_report.json - governance_status_path: models/governance_status.json - model_shift_status_path: models/model_shift_status.json - -output: - mrm_report_json: reports/mrm/mrm_validation_report.json - mrm_status_json: models/mrm_report_status.json diff --git a/configs/crpto_pd_model.yaml b/configs/crpto_pd_model.yaml index 2a88ab6..00c241d 100644 --- a/configs/crpto_pd_model.yaml +++ b/configs/crpto_pd_model.yaml @@ -44,7 +44,6 @@ hpo: n_startup_trials: 50 multivariate_tpe: true group_tpe: true - warn_independent_sampling: false pruner_n_startup_trials: 25 pruner_n_warmup_steps: 120 use_pruning_callback: true diff --git a/configs/crpto_publication_targets.yaml b/configs/crpto_publication_targets.yaml index 0a9632d..621af17 100644 --- a/configs/crpto_publication_targets.yaml +++ b/configs/crpto_publication_targets.yaml @@ -1,6 +1,6 @@ -version: "2026-07-09" +version: "2026-07-15" project: paper-crpto -decision_status: active +decision_status: prefreeze_active primary_target: id: informs_ijds @@ -9,232 +9,168 @@ primary_target: publisher: INFORMS manuscript_source: paper/CRPTO_ijds.qmd supplement_source: paper/supplement_ijds.qmd + official_tex_source: paper/submission/CRPTO_ijds_submission.tex template_family: "INFORMS IJDS LaTeX" - template_note: "Use the official INFORMS-IJDS-Template.tex when producing the submission PDF." official_urls: submission_guidelines: "https://pubsonline.informs.org/page/ijds/submission-guidelines" data_code_policy: "https://pubsonline.informs.org/page/ijds/data-and-code-disclosure-policy" + reviewer_guidelines: "https://pubsonline.informs.org/page/ijds/reviewer-guidelines" + latex_style_files: "https://pubsonline.informs.org/authorportal/latex-style-files" constraints: initial_submission_pages: 25 - page_limit_excludes: - - references - - appendices + page_limit_excludes: [references, appendices] online_supplement: separate_file review_model: double_anonymous - anonymity_start_date: "2025-01-01" submission_file: pdf_from_ijds_latex_template - data_code_disclosure: required_at_submission_acknowledgment - accepted_paper_reproducibility: data_code_required_unless_exempt - crpto_fit: - - "Decision-focused data science with auditable reproducibility." - - "DVC/DagsHub/MLflow companion already matches data/code disclosure expectations." - - "25-page body encourages compression of the current Quarto dossier." - required_before_submission: - - "Create anonymized IJDS PDF from paper/CRPTO_ijds.qmd." - - "Create anonymized online supplement from paper/supplement_ijds.qmd." - - "Remove author-identifying GitHub/DagsHub URLs from the anonymous PDF unless policy allows non-anonymous companion disclosure." - - "Freeze a release tag and reproducibility bundle after anonymity decision." - current_paper_scope: - include: - - "Exact 90% conformal replay plus a November-selected 3x3 linear policy grid under deterministic endpoint cap B_u<=0.28." - - "Outcome-free December selector replay and independent decision audit showing that policy stability is not selected-set coverage validity." - - "One interpretable policy, q=(p+u)/2 with tau=0.17, and a matched point-PD comparator." - - "Frozen upstream PD/calibration/conformal chain and historical pool93 frontier retained only as provenance." - - "A3--A34 as supporting diagnostics and A35--A40 as the active exact-alpha/selector/evaluation bundle." - - "Regret-auditability frontier as the body-level SPO+/CRPTO trade-off." - - "OCE/CVaR, robust satisficing, external replications, and SPO+ as diagnostics or comparators, not additional CRPTO methods." - - "Cluster-aware dependence caveat/proposition in the theory supplement." - - "Prosper/Freddie external economic replication without reopening the Lending Club champion." - - "Quarto/DVC/DagsHub/MLflow companion after anonymity handling." - out_of_scope_method_extensions: - - "OCE/CVaR as an optimized portfolio objective." - - "Multi-distribution or online conformal prediction as the promoted conformal layer." - - "Online DFL, SPO+ + conformal hybrid training, or causal CRPTO." - - "Multi-period portfolio, production dashboard, field trial, or package extraction." + data_code_disclosure: required -journal_strengthening_pack: - stance: "Selected former P2/P3 ideas are included as current paper/journal diagnostics or framing, while method-changing variants remain backlog." - include_in_current_submission: - regret_auditability_frontier: - status: include_body - role: "Central trade-off: SPO+ wins regret, CRPTO buys coverage, exact funded-set control and auditability." - artifacts: - - reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv - - reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png - requires_new_run: false - tail_risk_oce_cvar_diagnostic: - status: include_supplement - role: "Legacy OCE/CVaR sensitivity retained in the supplement; it does not select or redefine the active linear policy." - artifacts: - - reports/crpto/tables/crpto_tableA12_tail_risk_oce_cvar.csv - - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - requires_new_run: false - exact_alpha_calibration_selected_policy: - status: include_body_and_supplement - role: "Active IJDS evidence: exact alpha replay, split nine-policy selector/audit, temporal/funded-set audits, month-cluster bootstrap, and matched comparisons." - artifacts: - - reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv - - reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv - - reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv - - reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv - - reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv - - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json - requires_new_run: false - matched_point_pd_baseline: - status: include_body_and_supplement - role: "A40 matched Lending Club baseline: same candidates, tau=0.17 and operating constraints; point PD replaces the conformal guardrail." - artifacts: - - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv - - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json - requires_new_run: false - robust_satisficing_margins: - status: include_supplement_or_short_body - role: "OR/committee margins for return, V, Gamma_CP, realized risk-tolerance excess and robust-region pass." - artifacts: - - reports/crpto/tables/crpto_tableA13_satisficing_margins.csv - requires_new_run: false - dependence_aware_bound: - status: include_theory_appendix_or_caveat - role: "Cluster-aware conditional tightening statement; Markov remains the main distribution-free bound." - artifacts: - - reports/crpto/tables/crpto_tableA14_dependency_cluster_diagnostics.csv - - reports/crpto/tables/crpto_tableA21_cluster_bound_tightening.csv - - docs/research/crpto_conditional_tightening_appendix_2026-05-04.md - requires_new_run: false - tail_satisficing_challenger_audit: - status: include_supplement - role: "Journal-only legacy audit; it is not a selector for the active 50/50 linear policy." - artifacts: - - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - - models/crpto_tail_satisficing_audit_status.json - requires_new_run: false - tail_constrained_reoptimization: - status: include_supplement - role: "CVaR/OCE tail-constrained re-optimization over the frozen robust-region policies; challenger evidence only." - artifacts: - - reports/crpto/tables/crpto_tableA22_tail_constrained_reoptimization.csv - - models/crpto_tail_constrained_reopt_status.json - requires_new_run: false - distribution_online_diagnostics: - status: include_supplement - role: "Multi-distribution and online ACI diagnostics over frozen Lending Club conformal intervals." - artifacts: - - reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv - - reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv - - models/crpto_distribution_robustness_status.json - requires_new_run: false - multidataset_external_replication: - status: include_supplement_or_short_body - role: "External economic replication on Prosper and Freddie/Mendeley; it supports transfer of the recipe without replacing the Lending Club certificate." - artifacts: - - reports/crpto/tables/crpto_tableA25_external_replication_gate.csv - - reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.csv - - reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.csv - - reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv - - reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.csv - - reports/crpto/tables/crpto_tableA30_external_metric_intervals.csv - - reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.csv - - reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.csv - - reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv - - reports/crpto/figures/crpto_fig22_external_replication.png - - reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png - - reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png - - models/crpto_multidataset_external_status.json - requires_new_run: false - backlog_not_blocking: - tail_satisficing_objective_scaffold: - status: research_scaffold_not_champion - role: "Reusable OCE/CVaR/satisficing scoring layer for a future isolated search or sensitivity experiment." - config: configs/crpto_tail_satisficing_objective.yaml - module: src/optimization/tail_satisficing_objective.py - requires_new_dataset: false - requires_new_run: true - prospective_multidataset_validation: - status: future_protocol_not_blocker - role: "Prospective/live external validation beyond the static Prosper/Freddie replications." - requires_new_dataset: true - requires_new_run: true - still_out_of_scope: - - "OCE/CVaR as a promoted optimized portfolio objective or hard search constraint for the current champion." - - "Multi-distribution conformal prediction as the promoted conformal layer." - - "Online conformal recalibration, online DFL, SPO+ + conformal hybrid training or causal CRPTO." - - "Production monitoring, package extraction, field trial or multi-period portfolio." +active_scientific_contract: + claim_registry: docs/research/active_claims_2026-07-14.md + claim_ledger: configs/ijds_claim_ledger.yaml + source_registry: configs/ijds_active_evidence_sources.yaml + evidence_manifest: reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json + lineage_and_dvc_authority: configs/ijds_active_evidence_sources.yaml + + active_code_surface: + source_roots: + - src + paper_pipeline: + - scripts/build_ijds_binary_geometry_frontier_v4_evidence.py + - scripts/check_publication_integrity.py + - scripts/build_ijds_submission_tex.py + - scripts/compile_ijds_submission.py + - scripts/inspect_ijds_pdfs.py + - scripts/render_submission_pdf_previews.py + protocol_entrypoints: + - scripts/experiments/run_ijds_binary_geometry_frontier_v4.py + - scripts/experiments/run_ijds_normalized_objective_frontier.py + - scripts/experiments/run_ijds_normalized_objective_frontier_v2.py + - scripts/experiments/run_ijds_raw_data_audit.py + - scripts/experiments/run_ijds_credit_risk_controls.py + - scripts/experiments/run_ijds_label_lag_sensitivity.py + - scripts/experiments/run_ijds_fit_label_completion_sensitivity.py + - scripts/experiments/run_ijds_endpoint_availability_sensitivity.py + - scripts/experiments/run_ijds_missingness_sensitivity.py + - scripts/experiments/run_ijds_portfolio_structure_sensitivity.py + - scripts/experiments/run_ijds_allocation_granularity_sensitivity.py + - scripts/experiments/run_ijds_policy_support_tie_audit.py + support_tools: + - scripts/build_ijds_policy_support_tie_evidence.py + - scripts/manage_ijds_dvc_capsule.py + - scripts/run_complexity_report.py + - scripts/run_ty_advisory.py + historical_execution_in_active_capsule: false + + sealed_compatibility_capsule: + purpose: >- + Preserve path-bound hashes and the historical DVC graph without exposing + either as current evidence, an executable workflow, or manuscript history. + authorities: + - EXTRACTION_MANIFEST.json + - dvc.yaml + - dvc.lock + execution_allowed: false + paper_evidence_allowed: false + + previously_inspected_retrospective_archive: true + archive_is_verified_point_in_time_snapshot: false + endpoint_cutoff: "2020-09-30" + confirmatory: false + prospective: false + causal: false + policy_winner_allowed: false + + method: + prediction: >- + CatBoost/Platt is the only portfolio score. Numeric logistic, monotonic + CatBoost, platform-signal WOE/IV, and pricing-excluded application WOE/IV + are coverage-only controls and cannot be selected from OOT outcomes. + conformal: >- + Five score strata are fixed from 2011 scores. Exact binary-outcome + residual ranks are reported for all eight eligible six-month windows; + an observed-only fit and three declared completion rules stress labels + unavailable at fit time. + decision: >- + Gamma {0,.25,.50,.75,1} is evaluated under objective-matched and + normalized-score rulers at {.25,.50,.75}; gamma 1 minus gamma 0 is the + frozen endpoint contrast, with no selector. + comparator: >- + C0, C1, C2, exact development-admissible support, and broad [0.05,0.12] + support are outcome-free diagnostics. HiGHS basis endpoints give exact + sharp envelopes on each declared point-cap interval. + endpoint: >- + Terminal status is conservatively reconstructed as observable by the + cutoff; later terminal statuses and nonterminal rows remain unresolved. + payoff: >- + Plug-in (1-p)r-p*LGD and status-indexed standardized proxy + (1-Y)r-Y*LGD; neither is a cash-flow return, IRR, NPV, or welfare measure. + allocation_granularity: >- + The portfolio is a continuous linear-program relaxation. A deterministic + USD 25 floor-with-cash diagnostic measures implementation granularity but + does not define or optimize an integer policy. + + required_artifacts: + - reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json + - reports/crpto/ijds_policy_support_tie_evidence.json + - reports/crpto/tables/crpto_ijds_comparator_support_domain.csv + - reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv + - reports/crpto/tables/crpto_ijds_policy_family_domain.csv + - reports/crpto/tables/crpto_ijds_v4_table1_coverage_windows.csv + - reports/crpto/tables/crpto_ijds_v4_table2_phase_transition.csv + - reports/crpto/tables/crpto_ijds_v4_table3_development_envelopes.csv + - reports/crpto/tables/crpto_ijds_v4_table4_direction_summary.csv + - reports/crpto/tables/crpto_ijds_v4_table5_two_ruler_tracks.csv + - reports/crpto/tables/crpto_ijds_v4_table6_credit_controls.csv + - reports/crpto/tables/crpto_ijds_v4_tableS1_named_comparators.csv + - reports/crpto/tables/crpto_ijds_v4_tableS2_credit_prediction_metrics.csv + - reports/crpto/tables/crpto_ijds_v4_tableS3_woe_iv_psi.csv + - reports/crpto/tables/crpto_ijds_v4_tableS4_score_psi.csv + - reports/crpto/tables/crpto_ijds_v4_tableS5_label_lag_sensitivity.csv + - reports/crpto/tables/crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv + - reports/crpto/tables/crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv + - reports/crpto/tables/crpto_ijds_v4_tableS8_endpoint_resolution.csv + - reports/crpto/tables/crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv + - reports/crpto/tables/crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv + - reports/crpto/tables/crpto_ijds_v4_tableS11_fit_label_completion.csv + - reports/crpto/tables/crpto_ijds_v4_tableS12_allocation_granularity.csv + - reports/crpto/figures/crpto_ijds_v4_fig1_coverage.pdf + - reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png + - reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.pdf + - reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png + - reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.pdf + - reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png + +claim_boundaries: + required: + - "Observed binary-endpoint residual interval, not a latent-PD confidence interval." + - "Five coverage controls, one portfolio learner, and no OOT model winner." + - "Overlapping windows and repeated allocations are not independent replications." + - "Sharp bounds are partial identification, not confidence intervals." + - "Retrospective contrasts are not causal, prospective, or confirmatory." + - "The archive is not a verified point-in-time snapshot." + - "The status-indexed payoff proxy is not cash-flow return, NPV, IRR, or welfare." + - "Fit-label scenarios are declared stresses, not sharp bounds over all assignments." + - "USD 25 floor rounding is a diagnostic, not an optimized integer policy." + forbidden: + - "Selected model, gamma, ruler, coordinate, cap, comparator, or policy." + - "Universal portfolio direction or selected-set conformal guarantee." + - "Markov, deployment, causal, or fair-lending certificate." secondary_targets: - id: ejor role: secondary name: "European Journal of Operational Research" publisher: Elsevier - template_family: "Elsevier elsarticle" - official_urls: - journal: "https://www.sciencedirect.com/journal/european-journal-of-operational-research" - crpto_fit: - - "Operational research methodology plus decision-making practice." - - "Strong fit if the manuscript emphasizes robust optimization and applied decision value." - pivot_trigger: - - "IJDS desk-rejects as too OR/methodological." - - "Internal draft reads more like robust optimization than data science." - - id: informs_ijoo - role: stretch_theory - name: "INFORMS Journal on Optimization" - publisher: INFORMS - template_family: "INFORMS optimization journal style" - official_urls: - journal: "https://pubsonline.informs.org/journal/ijoo" - crpto_fit: - - "Good stretch only if the dependence-aware theory or optimization contribution becomes central." - pivot_trigger: - - "A stronger theorem, algorithm, or satisficing/robust optimization formulation is added." - - id: management_science - role: stretch_managerial - name: "Management Science" - publisher: INFORMS - template_family: "Management Science LaTeX" - official_urls: - submission_guidelines: "https://pubsonline.informs.org/page/mnsc/submission-guidelines" - crpto_fit: - - "Possible stretch if managerial decision value and governance become the dominant story." - pivot_trigger: - - "The paper gains broader managerial insight beyond the Lending Club experiment." - - id: operations_research - role: stretch_or - name: "Operations Research" - publisher: INFORMS - template_family: "INFORMS Operations Research style" - official_urls: - submission_guidelines: "https://pubsonline.informs.org/page/opre/submission-guidelines" - crpto_fit: - - "Possible stretch if the OR contribution is stronger than the empirical pipeline." - pivot_trigger: - - "The paper can be written as a broad OR method with concise proofs and code/data disclosure." - id: decision_support_systems role: applied_fallback name: "Decision Support Systems" publisher: Elsevier - template_family: "Elsevier article template" - official_urls: - journal: "https://www.sciencedirect.com/journal/decision-support-systems" - crpto_fit: - - "Applied fallback if the decision-support framing dominates the methodological framing." - - id: expert_systems_with_applications - role: applied_fallback - name: "Expert Systems with Applications" - publisher: Elsevier - template_family: "Elsevier article template" - official_urls: - journal: "https://www.sciencedirect.com/journal/expert-systems-with-applications" - crpto_fit: - - "Applied fallback if the manuscript is judged primarily as an intelligent risk system." current_decision: write_first_for: informs_ijds - target_submission_date: "2026-08-10" keep_second_ready_for: ejor - do_not_write_first_for: - - management_science - - operations_research - - informs_ijoo - reason: "CRPTO currently has the strongest fit as reproducible decision-focused data science with OR substance." - p2_p3_boundary: "P2/P3 are no longer a blanket exclusion: the journal strengthening pack enters as diagnostics, framing and theory caveat; method-changing extensions remain outside the submitted claim and are not acceptance criteria." + target_submission_date: not_fixed + freeze_rule: >- + Continue pre-freeze scientific, editorial, and reproducibility work. Do not + select on 2016--2017 outcomes. Submission freeze requires a separate + explicit decision. diff --git a/configs/experiments/champion_reopen.yaml b/configs/experiments/champion_reopen.yaml deleted file mode 100644 index 79af0e6..0000000 --- a/configs/experiments/champion_reopen.yaml +++ /dev/null @@ -1,169 +0,0 @@ -run_tag: champion-reopen-2026-06-19 -seeds: [42, 52, 62, 72, 82] -sample_rows: 0 - -data: - train_path: data/processed/train_fe.parquet - calibration_path: data/processed/calibration_fe.parquet - test_path: data/processed/test_fe.parquet - feature_config_path: data/processed/feature_config.yml - -validation: - scheme: temporal_train_val_cal_test - val_fraction: 0.15 - date_col: issue_d - -tabprep: - variants: [balanced_1500] - persist_transformed: false - suspicious_auc_threshold: 0.90 - extra_blacklist: - - id - - url - - loan_status - - emp_title - - title - - zip_code - - addr_state - - next_pymnt_d - - pymnt_plan - - application_type - - verification_status_joint - - settlement_status - - hardship_status - - debt_settlement_flag - monotonic_constraints: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - installment_burden: 1 - fico_score: -1 - credit_age_years: -1 - bc_util: 1 - percent_bc_gt_75: 1 - -model: - type: catboost - params: - iterations: 4200 - learning_rate: 0.057321202729872456 - depth: 8 - l2_leaf_reg: 119.37272987133554 - min_data_in_leaf: 135 - rsm: 0.5716653769355704 - random_strength: 1.3208942645900998e-07 - border_count: 254 - bootstrap_type: MVS - subsample: 0.678878683066026 - loss_function: Logloss - auto_class_weights: Balanced - eval_metric: AUC - has_time: true - allow_writing_files: false - random_seed: 42 - early_stopping_rounds: 150 - grow_policy: SymmetricTree - leaf_estimation_iterations: 5 - monotone_constraints: installment:1,annual_inc:-1,dti:1,loan_to_income:1,installment_burden:1,fico_score:-1,credit_age_years:-1,bc_util:1,percent_bc_gt_75:1 - feature_weights: - loan_to_income: 1.3 - annual_inc: 1.2 - dti: 1.2 - installment_burden: 1.1 - fico_score: 1.2 - penalties_coefficient: 0.75 - -calibration: - method: auto - candidates: - - platt - - isotonic - - beta - - venn_abers - - temperature - - quadratic_logit - -evaluation: - fairness_threshold: 0.50 - fairness_columns: - - grade - - sub_grade - - term - - purpose - - home_ownership - -output: - data_dir: data/processed/experiments/champion_reopen - model_dir: models/experiments/champion_reopen - report_dir: reports/crpto/experiments/champion_reopen - -champion_reopen: - selector_model: models/experiments/tabprep/tabprep-catboost-full-2026-06-17/balanced_1500/seed_42/pd_tabprep_challenger.cbm - ranking_method: pvc - shap_rows: 30000 - smoke_sample_rows: 50000 - smoke_cases: - - core42 - - pool93 - - pool93_woe - - pooltop42_tab20 - - pooltop55_business40 - feature_search_cases: - - core42 - - catboost44 - - pool93 - - pool93_woe - - pool93_top90 - - pool93_business80 - - pool93_business100 - - pooltop42_tab20 - - pooltop42_tab40 - - pooltop55_tab40 - - pooltop55_tab60 - - pooltop64_tab60 - - pooltop64_tab90 - - pooltop72_tab60 - - pooltop72_tab90 - - pooltop80_tab90 - - pooltop93_tab120 - - pooltop42_business20 - - pooltop55_business40 - - pooltop64_business60 - - pooltop72_business80 - - pooltop80_business80 - - pooltop93_business100 - seed_replay_cases: - - pool93_woe - - pooltop93_tab120 - - pooltop80_business80 - - pooltop80_tab90 - - pooltop72_business80 - - pool93_business80 - - pool93 - - pool93_top90 - - pooltop72_tab60 - - pooltop64_business60 - - pooltop55_business40 - - catboost44 - seed_replay_seeds: [52, 62, 72, 82] - pd_promotion_gates: - champion_auc: 0.7138518124963467 - champion_brier: 0.15439302183275685 - champion_ece: 0.006998009158194006 - min_auc_delta_mean: 0.005 - max_auc_seed_std: 0.0015 - max_brier_increase: 0.0005 - max_ece_increase: 0.0025 - portfolio_gates: - champion_return: 170464.5429 - champion_v_alpha01: 0.028875 - champion_c_cp: 0.187987 - min_robust_region_passers: 45 - require_exact_alpha01_pass: true - -downstream: - enabled: false - requires_explicit_promotion: true - conformal_config_path: configs/experiments/tabprep_conformal_policy.yaml - optimization_config_path: configs/experiments/tabprep_optimization.yaml diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml deleted file mode 100644 index d1fe691..0000000 --- a/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml +++ /dev/null @@ -1,52 +0,0 @@ -schema_version: "2026-07-09.7" -run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" - -source: - upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" - conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" - candidate_path: "data/processed/test_fe.parquet" - exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" - -design: - alpha: 0.10 - budget: 1000000.0 - max_concentration: 0.25 - lgd: 0.45 - period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] - combine_years_from: 2020 - selection_period: "2017-11" - audit_period: "2017-12" - endpoint_budget_cap: 0.28 - selection_min_budget_utilization: 0.999 - selection_rule: >- - maximize expected point-PD objective on November 2017 under a deterministic - endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; - reserve December 2017 for an outcome-free selector stability replay and a - post-selection decision audit - -policy_grid: - family: "simple_linear_conformal_guardrail" - risk_tolerances: [0.15, 0.17, 0.19] - gammas: [0.25, 0.50, 0.75] - uncertainty_aversions: [0.0] - -incumbent_policy: - risk_tolerance: 0.17 - gamma: 0.75 - uncertainty_aversion: 0.0 - -execution: - solver_backend: "highspy" - time_limit: 300 - threads: 1 - random_seed: 42 - -claim_boundary: >- - The final tagged rule selects among nine round-number policies on November - 2017 without exposing the selector to outcomes or assumption-conditional - statistics. December 2017 independently replays the outcome-free selector - and audits the already-fixed decision. The audit is diagnostic rather than a - selected-set coverage theorem. Earlier development inspected the static OOT - corpus, so the January 2018--September 2020 evaluation remains a transparent - retrospective lockbox replay, not a pristine prospective trial, causal - estimate, or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml deleted file mode 100644 index 6e98e4a..0000000 --- a/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml +++ /dev/null @@ -1,45 +0,0 @@ -schema_version: "2026-07-09.6" -run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" - -source: - upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" - conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" - conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" - exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" - -design: - alpha: 0.10 - budget: 1000000.0 - max_concentration: 0.25 - lgd: 0.45 - period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] - combine_years_from: 2020 - markov_threshold_cap: 0.60 - selection_min_budget_utilization: 0.999 - selection_rule: "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" - -policy_grid: - family: "simple_linear_conformal_guardrail" - risk_tolerances: [0.15, 0.17, 0.19] - gammas: [0.25, 0.50, 0.75] - uncertainty_aversions: [0.0] - -incumbent_policy: - risk_tolerance: 0.17 - gamma: 0.75 - uncertainty_aversion: 0.0 - -execution: - solver_backend: "highspy" - time_limit: 300 - threads: 1 - random_seed: 42 - -claim_boundary: >- - The final tagged policy rule ranks nine round-number candidates on the - calibration development block without reading default, realized-return, or - other outcome-derived selector columns, and then freezes the selected policy - before OOT evaluation. Conformal endpoints themselves use calibration labels, - as required. Earlier project development inspected this static OOT corpus, so - the evaluation is a transparent retrospective lockbox replay rather than a - pristine prospective trial, causal estimate, or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml b/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml deleted file mode 100644 index 8f447b4..0000000 --- a/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml +++ /dev/null @@ -1,19 +0,0 @@ -schema_version: "2026-07-09.1" -run_tag: "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" - -source: - upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" - conformal_namespace: "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1" - conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" - conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" - -design: - alpha_grid: [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] - alpha_mapping: "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe" - replay_tolerance: 1.0e-12 - -claim_boundary: >- - Recomputes finite-sample Mondrian quantiles for every declared alpha while - freezing the selected partition, calibration fit/holdout split, score scale, - and holdout-learned widening factors. It is a retrospective OOT audit, not a - post-selection or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_requirements.txt b/configs/experiments/champion_reopen_requirements.txt deleted file mode 100644 index 72eb9f8..0000000 --- a/configs/experiments/champion_reopen_requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Dedicated exploratory environment for the champion reopen search. -# Keep canonical CRPTO dependencies in pyproject/uv.lock frozen until a candidate -# passes promotion gates. --e .[search] -pytest>=9.0 -ruff>=0.10 -probmetrics>=1.3 -crepes>=0.9 -netcal>=1.3 -# Optional if a C compiler is available in the local environment: -# splinecalib>=0.0.13,<0.1 -cir-model>=0.1 -optuna>=4.0 -mapie>=1.4 -venn-abers>=1.5 -betacal>=1.1 diff --git a/configs/experiments/ijds_allocation_granularity_sensitivity_2026-07-16.yaml b/configs/experiments/ijds_allocation_granularity_sensitivity_2026-07-16.yaml new file mode 100644 index 0000000..4b12234 --- /dev/null +++ b/configs/experiments/ijds_allocation_granularity_sensitivity_2026-07-16.yaml @@ -0,0 +1,49 @@ +schema_version: "2026-07-16.3" +protocol_tag: "protocol/ijds-allocation-granularity-sensitivity-2026-07-16-v3" +run_tag: "ijds-allocation-granularity-sensitivity-2026-07-16-v3" + +parent: + v4_config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" + allocations: + path: "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/scenarios/b1000k_p025_l045/frontier/allocations.parquet" + bytes: 3180848 + sha256: "83870a9e0234289a46641f53f8857eeba5937ebdb8cd19b25eb56f7e8a8dfa81" + solve_records: + path: "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/scenarios/b1000k_p025_l045/frontier/solve_records.parquet" + bytes: 98930 + sha256: "2fd5bf83dcfce215a44521edf2475c8e4fe2a1df03e6061a59ce619563c6ef51" + +allocation_granularity: + lot_size_usd: 25.0 + committed_budget_usd: 1000000.0 + lgd: 0.45 + rounding_rule: "floor_each_exposure_hold_residual_as_cash" + periods: + - "2016-04" + - "2016-05" + - "2016-06" + - "2016-07" + - "2016-08" + - "2016-09" + - "2016-10" + - "2016-11" + - "2016-12" + - "2017-01" + - "2017-02" + - "2017-03" + - "2017-04" + - "2017-05" + - "2017-06" + expected_portfolios: 1440 + expected_tracks: 96 + numerical_tolerance: 1.0e-8 + committed_capital_comparison_rtol: 1.0e-10 + all_tracks_reported: true + historical_archive_previously_inspected: true + outcome_based_selection: false + paper_role: "continuous_relaxation_granularity_diagnostic_not_policy_selection" + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml new file mode 100644 index 0000000..0b2ff41 --- /dev/null +++ b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml @@ -0,0 +1,117 @@ +extends: "ijds_fixed_taxonomy_c2_2026-07-11.yaml" + +schema_version: "2026-07-12.2" +protocol_status: "locked_retrospective_complete_specification_audit" +protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" +run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + +hypothesis: >- + Absolute-residual conformalization for a binary outcome has a prevalence- + sensitive set geometry, candidate coverage need not transport over time, + and portfolio direction is not identified until the score-cap comparator + family is declared. Every eligible six-month residual window and every + declared comparator must be reported without result-based selection. + +design: + conformal_fit_start: "2012-01-01" + conformal_fit_end: "2013-01-31" + policy_development_start: "2013-02-01" + policy_development_end: "2013-12-31" + +residual_specification: + window_months: 6 + minimum_monthly_label_retention: 0.99 + windows: + - {id: "w01_2012m01_m06", start: "2012-01-01", end: "2012-06-30"} + - {id: "w02_2012m02_m07", start: "2012-02-01", end: "2012-07-31"} + - {id: "w03_2012m03_m08", start: "2012-03-01", end: "2012-08-31"} + - {id: "w04_2012m04_m09", start: "2012-04-01", end: "2012-09-30"} + - {id: "w05_2012m05_m10", start: "2012-05-01", end: "2012-10-31"} + - {id: "w06_2012m06_m11", start: "2012-06-01", end: "2012-11-30"} + - {id: "w07_2012m07_m12", start: "2012-07-01", end: "2012-12-31"} + - {id: "w08_2012m08_2013m01", start: "2012-08-01", end: "2013-01-31"} + +conformal: + method: "fixed_taxonomy_split_mondrian_absolute_residual" + estimand: "clipped_residual_interval_for_observed_binary_outcome" + residual_source: "all_eight_declared_six_month_availability_safe_windows" + discrete_intersection: "interval_intersect_binary_outcome_space" + canonical_groups: 5 + diagnostic_group_counts: [1, 2, 5, 10] + report_set_cardinality: true + report_width_quantiles: [0.0, 0.10, 0.25, 0.50, 0.75, 0.90, 1.0] + +learner_control: + enabled: true + role: "coverage_only_negative_control" + model: "numeric_logistic_regression" + numeric_features: "inherit_primary_numeric_features" + imputer: "median" + scaler: "standard" + logistic_regression: + C: 1.0 + class_weight: "balanced" + solver: "lbfgs" + max_iter: 3000 + random_state: 42 + platt_calibration: "independent_2011_raw_margin" + portfolio_optimization: false + +policy: + max_concentration_by_purpose: 0.25 + +comparators: + primary: "complete_development_supported_point_cap_frontier" + same_numeric_cap: true + development_fixed_funded_pd_cap: true + contemporaneous_funded_pd_match: true + development_support: + definition: "window_policy_monthly_guardrail_point_moment_hull" + rounding: false + exact_point_cap_frontier: + start: 0.05 + stop: 0.12 + method: "highs_rhs_basis_ranging" + include_c0_c1_c2_caps: true + cap_tolerance: 1.0e-10 + objective_dominance_tolerance: 1.0e-5 + +analysis: + all_eight_windows_primary: true + all_nine_policies_primary: true + canonical_seed_only_for_v4_portfolio: true + nested_scope_counts_are_not_independent: true + report_discrete_set_geometry: true + report_exact_frontier_breakpoints: true + confirmatory_language_forbidden: true + no_result_based_promotion: true + +simulation: + enabled: true + random_seed: 20260712 + repetitions: 100 + sample_size: 2000 + mechanism_policy: + risk_tolerance: 0.40 + gamma: 0.25 + budget_units: 50.0 + purpose_cap: 1.0 + lgd: 0.45 + score_shift_grid: [0.0, 0.05, 0.10, 0.15] + prevalence_shift_grid: [0.0, 0.025, 0.05, 0.10] + taxonomy_groups_grid: [1, 2, 5, 10] + censoring_rate_grid: [0.0, 0.05, 0.15] + +resume_outcome_free: null + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + deterministic_summary: "binary_geometry_frontier_v4_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12_v2.yaml b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12_v2.yaml new file mode 100644 index 0000000..08d7442 --- /dev/null +++ b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12_v2.yaml @@ -0,0 +1,18 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + +schema_version: "2026-07-12.3" +protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v2" +run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v2" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md" + +resume_outcome_free: + source_run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + source_protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" + source_protocol_commit: "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd" + source_freeze_sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" diff --git a/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml new file mode 100644 index 0000000..3fd2d45 --- /dev/null +++ b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml @@ -0,0 +1,29 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-12_v2.yaml" + +schema_version: "2026-07-14.1" +protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-14-v3" +run_tag: "ijds-binary-geometry-frontier-v4-2026-07-14-v3" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + +source: + snapshot_date: "2020-09-30" + snapshot_date_role: "reconstructed_endpoint_cutoff_not_verified_archive_snapshot" + +target: + name: "terminal_default_reconstructed_by_cutoff" + evaluation_outcome_contract: + mode: "conservative_terminal_status_reconstruction" + cutoff: "2020-09-30" + terminal_status_source: "distributed_archive_final_status" + archive_is_verified_point_in_time_snapshot: false + charged_off_reporting_lag_months: 6 + +design: + endpoint: "terminal_default_reconstructed_as_observable_by_2020-09-30" diff --git a/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v4.yaml b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v4.yaml new file mode 100644 index 0000000..c2c94c9 --- /dev/null +++ b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v4.yaml @@ -0,0 +1,23 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + +schema_version: "2026-07-15.3" +protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v4" +run_tag: "ijds-binary-geometry-frontier-v4-2026-07-15-v4" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_no_scientific_metric_change" + require_exact_reference_column_equivalence: true + artifact_section: "artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3/binary_geometry_frontier_v4_summary.json" + bytes: 17794 + sha256: "c05a6048fd6e259d53b5901b975a5970256bdd20901fdf236112e85930bafa5f" diff --git a/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml new file mode 100644 index 0000000..d484feb --- /dev/null +++ b/configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml @@ -0,0 +1,27 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-15_v4.yaml" + +schema_version: "2026-07-15.4" +protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5" +run_tag: "ijds-binary-geometry-frontier-v4-2026-07-15-v5" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + - "docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_machine_tolerance_recovery" + equivalence_mode: "exact_non_float_machine_tolerant_float" + require_exact_non_float_reference_equivalence: true + float_atol: 5.0e-14 + float_rtol: 5.0e-14 + artifact_section: "artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3/binary_geometry_frontier_v4_summary.json" + bytes: 17794 + sha256: "c05a6048fd6e259d53b5901b975a5970256bdd20901fdf236112e85930bafa5f" diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1.yaml new file mode 100644 index 0000000..3e83cc7 --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1.yaml @@ -0,0 +1,138 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + +schema_version: "2026-07-13.1" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v1" +run_tag: "ijds-credit-risk-controls-2026-07-13-v1" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + +credit_risk_controls: + question: >- + Does the V4 temporal coverage result survive an auditable credit-scorecard, + removal of incumbent platform pricing/grade signals, and domain-safe + monotonic CatBoost constraints when every eligible row is used? + inference_role: "coverage_and_model_generalization_control_only" + co_primary_models: + - "catboost_platt" + - "numeric_logistic_platt" + - "catboost_monotonic_platt" + - "woe_scorecard_platform_platt" + - "woe_scorecard_borrower_platt" + all_models_reported: true + selection_from_oot: false + portfolio_optimization: false + sampling: "none_all_eligible_rows" + active_score_reference: + path: "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/prediction/scores.parquet" + bytes: 14212211 + sha256: "4053efdbf13066355bb772233231e5fe4ccb436ca75aaf873fb14e213c1e319c" + numerical_tolerance: 1.0e-12 + score_psi_bins: 10 + platform_signal_features: + - "int_rate" + - "installment" + - "installment_burden" + - "grade" + - "sub_grade" + - "int_rate_bucket" + - "int_rate_bucket__grade" + monotonic_catboost: + constraints: + int_rate: 1 + annual_inc: -1 + dti: 1 + loan_to_income: 1 + loan_to_income_sq: 1 + installment_burden: 1 + rev_utilization: 1 + revol_bal_to_income: 1 + fico_score: -1 + credit_age_years: -1 + pub_rec: 1 + inq_last_6mths: 1 + delinq_severity: 1 + delinq_recency: -1 + log_annual_inc: -1 + has_delinq_2yrs: 1 + has_pub_rec: 1 + has_bankruptcy: 1 + has_recent_inq: 1 + recent_chargeoff: 1 + optbinning: + max_n_prebins: 20 + min_prebin_size: 0.05 + min_n_bins: 2 + max_n_bins: 8 + min_bin_size: 0.05 + n_jobs: -1 + scorecard_logistic: + C: 1.0 + class_weight: "balanced" + solver: "lbfgs" + max_iter: 3000 + random_state: 42 + scorecards: + platform: + name: "woe_scorecard_platform_platt" + features: + - "loan_amnt" + - "int_rate" + - "installment" + - "annual_inc" + - "dti" + - "loan_to_income" + - "installment_burden" + - "rev_utilization" + - "revol_bal_to_income" + - "fico_score" + - "credit_age_years" + - "emp_length_num" + - "open_acc" + - "total_acc" + - "revol_bal" + - "pub_rec" + - "inq_last_6mths" + - "delinq_severity" + - "delinq_recency" + - "grade" + - "sub_grade" + - "home_ownership" + - "purpose" + - "verification_status" + - "int_rate_bucket" + - "int_rate_bucket__grade" + borrower: + name: "woe_scorecard_borrower_platt" + features: + - "loan_amnt" + - "annual_inc" + - "dti" + - "loan_to_income" + - "rev_utilization" + - "revol_bal_to_income" + - "fico_score" + - "credit_age_years" + - "emp_length_num" + - "open_acc" + - "total_acc" + - "revol_bal" + - "pub_rec" + - "inq_last_6mths" + - "delinq_severity" + - "delinq_recency" + - "home_ownership" + - "purpose" + - "verification_status" + +resume_credit_control_freeze: null + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + deterministic_summary: "credit_risk_controls_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml new file mode 100644 index 0000000..d36a78c --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml @@ -0,0 +1,14 @@ +extends: "ijds_credit_risk_controls_2026-07-13_v1.yaml" + +schema_version: "2026-07-13.2" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v1b" +run_tag: "ijds-credit-risk-controls-2026-07-13-v1b" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + +resume_credit_control_freeze: null diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml new file mode 100644 index 0000000..dc1b28d --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml @@ -0,0 +1,20 @@ +extends: "ijds_credit_risk_controls_2026-07-13_v1b.yaml" + +schema_version: "2026-07-13.3" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v2" +run_tag: "ijds-credit-risk-controls-2026-07-13-v2" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml" + - "docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md" + +resume_credit_control_freeze: + source_run_tag: "ijds-credit-risk-controls-2026-07-13-v1b" + source_protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v1b" + source_protocol_commit: "1776cbf8b201ae5b92756e5ea397a403d6cc7c9f" + source_freeze_sha256: "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35" diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2b.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2b.yaml new file mode 100644 index 0000000..e4ab941 --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-13_v2b.yaml @@ -0,0 +1,25 @@ +extends: "ijds_credit_risk_controls_2026-07-13_v2.yaml" + +schema_version: "2026-07-13.4" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v2b" +run_tag: "ijds-credit-risk-controls-2026-07-13-v2b" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml" + - "docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml" + - "docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md" + +evaluation_recovery: + status: "numerical_calibration_recovery_only" + calibration_solver: "sklearn_unpenalized_lbfgs" + require_exact_coverage_equivalence: true + coverage_reference: + path: "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v2/evaluation/temporal_coverage.parquet" + bytes: 501571 + sha256: "956bdc9880c80cebc1f48fc2cdf57688dfff7dddd3939b4fd972413d83039767" diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-14_v3.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-14_v3.yaml new file mode 100644 index 0000000..277e988 --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-14_v3.yaml @@ -0,0 +1,35 @@ +extends: "ijds_credit_risk_controls_2026-07-13_v2b.yaml" + +schema_version: "2026-07-14.1" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-14-v3" +run_tag: "ijds-credit-risk-controls-2026-07-14-v3" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml" + - "docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml" + - "docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + +source: + snapshot_date: "2020-09-30" + snapshot_date_role: "reconstructed_endpoint_cutoff_not_verified_archive_snapshot" + +target: + name: "terminal_default_reconstructed_by_cutoff" + evaluation_outcome_contract: + mode: "conservative_terminal_status_reconstruction" + cutoff: "2020-09-30" + terminal_status_source: "distributed_archive_final_status" + archive_is_verified_point_in_time_snapshot: false + charged_off_reporting_lag_months: 6 + +design: + endpoint: "terminal_default_reconstructed_as_observable_by_2020-09-30" + +evaluation_recovery: null diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-15_v4.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-15_v4.yaml new file mode 100644 index 0000000..0b7a101 --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-15_v4.yaml @@ -0,0 +1,27 @@ +extends: "ijds_credit_risk_controls_2026-07-14_v3.yaml" + +schema_version: "2026-07-15.3" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-15-v4" +run_tag: "ijds-credit-risk-controls-2026-07-15-v4" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml" + - "docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml" + - "docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_no_scientific_metric_change" + require_exact_reference_column_equivalence: true + artifact_section: "evaluation_artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3/credit_risk_controls_summary.json" + bytes: 12473 + sha256: "c64bb3088771d7f3c9a2fb550bbd8ee9a789a5e340054e09f3126656b6a89ca7" diff --git a/configs/experiments/ijds_credit_risk_controls_2026-07-15_v5.yaml b/configs/experiments/ijds_credit_risk_controls_2026-07-15_v5.yaml new file mode 100644 index 0000000..17a2459 --- /dev/null +++ b/configs/experiments/ijds_credit_risk_controls_2026-07-15_v5.yaml @@ -0,0 +1,31 @@ +extends: "ijds_credit_risk_controls_2026-07-15_v4.yaml" + +schema_version: "2026-07-15.4" +protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-15-v5" +run_tag: "ijds-credit-risk-controls-2026-07-15-v5" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_raw_data_contract_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md" + - "docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1b.yaml" + - "docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md" + - "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml" + - "docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + - "docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_machine_tolerance_recovery" + equivalence_mode: "exact_non_float_machine_tolerant_float" + require_exact_non_float_reference_equivalence: true + float_atol: 5.0e-14 + float_rtol: 5.0e-14 + artifact_section: "evaluation_artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3/credit_risk_controls_summary.json" + bytes: 12473 + sha256: "c64bb3088771d7f3c9a2fb550bbd8ee9a789a5e340054e09f3126656b6a89ca7" diff --git a/configs/experiments/ijds_endpoint_availability_sensitivity_2026-07-14.yaml b/configs/experiments/ijds_endpoint_availability_sensitivity_2026-07-14.yaml new file mode 100644 index 0000000..249065e --- /dev/null +++ b/configs/experiments/ijds_endpoint_availability_sensitivity_2026-07-14.yaml @@ -0,0 +1,52 @@ +schema_version: "2026-07-14.1" +protocol_status: "locked_retrospective_endpoint_assumption_sensitivity" +protocol_tag: "protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1" +run_tag: "ijds-endpoint-availability-sensitivity-2026-07-14-v1" + +parent: + v4_config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + two_ruler_config: "configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml" + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + v4_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + credit_freeze: + path: "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/protocol_freeze.json" + bytes: 25908 + sha256: "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35" + +endpoint: + evaluation_cutoff: "2020-09-30" + charged_off_lag_months: [0, 3, 6, 8, 12] + fully_paid_lag_months: 0 + roles: ["policy_development", "primary_oot", "censored_extension"] + canonical_taxonomy_groups: 5 + all_lags_models_windows_and_rulers_reported: true + +claim_boundary: + historical_archive_previously_inspected: true + exploratory_values_seen_before_protocol: true + preregistered: false + confirmatory: false + prospective: false + outcome_based_selection: false + allocation_refit: false + policy_selection: false + model_selection: false + endpoint_selection: false + no_submission_freeze: true + +stop_rules: + stop_on_source_hash_mismatch: true + stop_on_candidate_membership_change: true + stop_on_incomplete_coverage_census: true + stop_on_incomplete_two_ruler_census: true + stop_on_incomplete_exact_support_census: true + retain_every_declared_lag: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_fit_label_completion_sensitivity_2026-07-16.yaml b/configs/experiments/ijds_fit_label_completion_sensitivity_2026-07-16.yaml new file mode 100644 index 0000000..3a18020 --- /dev/null +++ b/configs/experiments/ijds_fit_label_completion_sensitivity_2026-07-16.yaml @@ -0,0 +1,42 @@ +schema_version: "2026-07-16.2" +protocol_tag: "protocol/ijds-fit-label-completion-sensitivity-2026-07-16-v2" +run_tag: "ijds-fit-label-completion-sensitivity-2026-07-16-v2" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" + outcome_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + active_coverage: + path: "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/temporal_coverage.parquet" + bytes: 215583 + sha256: "25be1d2e065a6eab632618127cb21ccfb99be6f582ef557e80122c184f28dfe2" + +fit_label_completion: + scenarios: + - "observed_only" + - "all_unavailable_nondefault" + - "all_unavailable_default" + - "hindsight_terminal" + taxonomy_groups: 5 + evaluation_strata: [-1, 2] + baseline_score_replay_tolerance: 1.0e-4 + baseline_recipe_replay_tolerance: 1.0e-4 + baseline_coverage_replay_tolerance: 1.0e-5 + baseline_geometry_replay_tolerance: 2.0e-5 + v1_stopped_score_drift: 7.389153786618863e-6 + numerical_recovery_only: true + all_scenarios_reported: true + historical_archive_previously_inspected: true + outcome_based_selection: false + nonlinear_corner_interpretation_only: true + paper_role: "fit_label_availability_stress_not_model_or_scenario_selection" + +execution: + threads: 10 + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml b/configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml new file mode 100644 index 0000000..f0c239c --- /dev/null +++ b/configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml @@ -0,0 +1,230 @@ +schema_version: "2026-07-11.2" +protocol_status: "locked_retrospective_prefreeze_audit" +protocol_tag: "protocol/ijds-fixed-taxonomy-c2-2026-07-11-v2" +run_tag: "ijds-fixed-taxonomy-c2-2026-07-11-v2" + +hypothesis: >- + A conformal-derived portfolio score can attain its target in a near + post-calibration cohort yet lose both coverage and decision meaning after + temporal transport; conclusions about the score must therefore be robust + to an outcome-free comparator multiverse rather than one copied risk cap. + +source: + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + snapshot_date: "2020-09-30" + information_cutoff: "2016-03-31" + charged_off_reporting_lag_months: 6 + minimum_label_retention: 0.99 + csv_chunksize: 100000 + required_raw_columns: + - id + - loan_amnt + - funded_amnt + - total_pymnt + - term + - int_rate + - installment + - grade + - sub_grade + - emp_length + - home_ownership + - annual_inc + - verification_status + - issue_d + - loan_status + - last_pymnt_d + - purpose + - dti + - delinq_2yrs + - earliest_cr_line + - fico_range_low + - fico_range_high + - inq_last_6mths + - mths_since_last_delinq + - open_acc + - pub_rec + - revol_bal + - revol_util + - total_acc + - collections_12_mths_ex_med + - chargeoff_within_12_mths + - tax_liens + - pub_rec_bankruptcies + - mort_acc + - num_tl_op_past_12m + - acc_open_past_24mths + +target: + name: "terminal_default_by_snapshot" + positive_definition: "Normalized status contains Charged Off." + negative_definition: "Normalized status contains Fully Paid." + right_censored_definition: >- + Exact Default and every other nonterminal snapshot status remain + unresolved; no unresolved row is coerced to a binary outcome. + +design: + term_months: 36 + development_end: "2010-12-31" + validation_tail_fraction: 0.20 + probability_calibration_start: "2011-01-01" + probability_calibration_end: "2011-12-31" + conformal_fit_start: "2012-01-01" + conformal_fit_end: "2012-06-30" + policy_development_start: "2012-07-01" + policy_development_end: "2012-12-31" + primary_oot_start_month: "2016-04" + primary_oot_end_month: "2017-06" + censored_extension_start_month: "2017-07" + censored_extension_end_month: "2017-09" + endpoint: "terminal_default_observed_by_2020-09-30_with_right_censoring" + historical_archive_previously_inspected: true + +model: + type: "CatBoostClassifier" + canonical_seed: 42 + sensitivity_seeds: [40, 41, 42, 43, 44] + fixed_params: + iterations: 500 + learning_rate: 0.04 + depth: 6 + l2_leaf_reg: 10.0 + loss_function: "Logloss" + eval_metric: "AUC" + auto_class_weights: "Balanced" + bootstrap_type: "Bernoulli" + subsample: 0.80 + has_time: true + allow_writing_files: false + verbose: false + numeric_features: + - loan_amnt + - int_rate + - installment + - annual_inc + - dti + - loan_to_income + - installment_burden + - rev_utilization + - revol_bal_to_income + - open_acc_ratio + - fico_score + - credit_age_years + - emp_length_num + - open_acc + - total_acc + - revol_bal + - pub_rec + - inq_last_6mths + - delinq_severity + - delinq_recency + - log_annual_inc + - log_revol_bal + - loan_to_income_sq + - fico_x_dti + - has_delinq_2yrs + - has_pub_rec + - has_bankruptcy + - has_recent_inq + - recent_chargeoff + categorical_features: + - grade + - sub_grade + - home_ownership + - purpose + - verification_status + - int_rate_bucket + - dti_bucket + - fico_bucket + - int_rate_bucket__grade + +probability_calibration: + method: "platt_on_catboost_raw_margin" + logistic_regression: + C: 1000000.0 + solver: "lbfgs" + max_iter: 2000 + random_state: 42 + +conformal: + method: "fixed_taxonomy_split_mondrian_absolute_residual" + estimand: "binary_terminal_outcome_prediction_set_convex_hull" + alpha: 0.10 + canonical_groups: 5 + taxonomy_source: "2011_all_status_independent_calibrated_scores" + residual_source: "2012H1_availability_safe_labels" + minimum_rows_per_group: 1000 + diagnostic_group_counts: [1, 2, 5, 10] + finite_sample_rank: "ceil((n_group + 1) * (1 - alpha))" + exactness_boundary: "order_statistic_rank_only_under_exchangeability" + +payoff: + id: "coherent_standardized_binary_payoff_v1" + expected_formula: "(1 - p) * r - p * LGD" + realized_formula: "(1 - y) * r - y * LGD" + lgd: 0.45 + reoptimization_lgd_grid: [0.25, 0.45, 0.65] + secondary_cash_metric: "undiscounted_snapshot_cumulative_net_cash_yield" + +policy: + family: "closed_linear_conformal_score_family" + risk_tolerances: [0.15, 0.17, 0.19] + gammas: [0.25, 0.50, 0.75] + uncertainty_aversions: [0.0] + outcome_based_selection: false + budget: 1000000.0 + min_budget_utilization_solver: 1.0 + canonical_purpose_cap: 0.25 + purpose_cap_sensitivity: [0.20, 0.25, 0.30, 1.00] + score_ablations: + - "clipped_conformal_upper_blend" + - "unclipped_group_residual_penalty" + - "pooled_residual_penalty" + +comparators: + primary: "contemporaneous_outcome_free_funded_pd_match" + match_tolerance: 1.0e-10 + same_numeric_cap: true + development_fixed_funded_pd_cap: true + point_cap_frontier: + start: 0.05 + stop: 0.12 + step: 0.0025 + selection_from_outcomes: false + +analysis: + all_nine_policies_primary: true + report_every_cell: true + unresolved_outcomes: "sharp_common_outcome_binary_bounds" + finite_population_estimand: true + confirmatory_language_forbidden: true + universal_direction_requires: "9_of_9_canonical_pairs_and_no_opposite_seed_purpose_cell" + comparator_reversal_requires: "same_threshold_and_c2_opposite_for_9_of_9" + miscoverage_direction_requires: "9_of_9" + +simulation: + enabled: true + random_seed: 20260711 + repetitions: 200 + sample_size: 2000 + temporal_shift_grid: [0.0, 0.05, 0.10, 0.15] + +resume_outcome_free: + enabled: true + source_run_tag: "ijds-fixed-taxonomy-c2-2026-07-11-v1" + source_protocol_tag: "protocol/ijds-fixed-taxonomy-c2-2026-07-11-v1" + source_protocol_commit: "4835cc18a0117a695f89f9da70a4e3af97663a27" + source_freeze_sha256: "93690082880ef4ff1375dcd5b26d2df79f80e6ebe09a6d83b7fd99a9abb4cfae" + +execution: + solver_backend: "highspy" + strict_solver_backend: true + solver_time_limit_seconds: 300 + threads: 4 + allocation_tolerance: 1.0e-10 + +output: + data_root: "data/processed/experiments/ijds_prefreeze" + model_root: "models/experiments/ijds_prefreeze" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + deterministic_summary: "fixed_taxonomy_c2_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml b/configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml new file mode 100644 index 0000000..d694cbb --- /dev/null +++ b/configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml @@ -0,0 +1,23 @@ +schema_version: "2026-07-14.1" +protocol_tag: "protocol/ijds-label-lag-sensitivity-2026-07-14-v1" +run_tag: "ijds-label-lag-sensitivity-2026-07-14-v1" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + protocol_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + +lag_sensitivity: + charged_off_lag_months: [0, 3, 6, 8, 12] + phase_stratum: 2 + minimum_monthly_label_retention: 0.99 + all_lags_reported: true + historical_archive_previously_inspected: true + outcome_based_selection: false + paper_role: "assumption_sensitivity_not_model_or_window_selection" + +output: + root: "reports/crpto/sensitivity" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_missingness_sensitivity_2026-07-15_v3.yaml b/configs/experiments/ijds_missingness_sensitivity_2026-07-15_v3.yaml new file mode 100644 index 0000000..a77d99b --- /dev/null +++ b/configs/experiments/ijds_missingness_sensitivity_2026-07-15_v3.yaml @@ -0,0 +1,50 @@ +schema_version: "2026-07-15.3" +protocol_status: "locked_outcome_free_then_evaluate_missingness_sensitivity" +protocol_tag: "protocol/ijds-missingness-sensitivity-2026-07-15-v3" +run_tag: "ijds-missingness-sensitivity-2026-07-15-v3" + +base_config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" +protocol_document: "docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md" +implementation_lineage_files: + - "docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md" + - "docs/research/ijds_missingness_sensitivity_v3_erratum_2026-07-15.md" + +source_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_commit: "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd" + +specifications: + - id: "catboost_platt" + encoding: "active_sentinel_convention" + imported: true + - id: "catboost_missing_indicators_platt" + encoding: "active_mappings_plus_explicit_missing_indicators" + imported: false + added_numeric_features: ["delinq_recency_missing", "bankruptcy_count_missing"] + - id: "catboost_native_missing_platt" + encoding: "native_numeric_nan_with_nullable_binary_semantics" + imported: false + dropped_numeric_features: ["delinq_recency", "has_bankruptcy"] + added_numeric_features: ["delinq_recency_native", "has_bankruptcy_native"] + +evaluation: + role: "primary_oot" + expected_candidates: 376890 + expected_windows: 8 + taxonomy_groups: 5 + aggregate_stratum: -1 + nominal_coverage: 0.90 + no_model_selection: true + no_window_selection: true + no_portfolio_optimization: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + deterministic_summary: "missingness_sensitivity_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml b/configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml new file mode 100644 index 0000000..158aba6 --- /dev/null +++ b/configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml @@ -0,0 +1,110 @@ +schema_version: "2026-07-13.2" +protocol_status: "locked_outcome_free_frontier_before_execution" +protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-13-v1c" +run_tag: "ijds-normalized-objective-frontier-2026-07-13-v1c" + +hypothesis: >- + Fixed numeric caps confound score scale with decision stringency. A complete + gamma path evaluated under both an affine-invariant normalized-score ruler + and a common plug-in-objective ruler can determine whether portfolio + directions survive the comparator coordinate or remain unidentified. + +lineage: + original_failed_protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-12-v1" + objective_tie_erratum_tag: "protocol/ijds-normalized-objective-frontier-2026-07-13-v1b" + objective_tie_correction: "replace_slack_floor_score_span_with_basis_reduced_cost_and_id_reversal" + v1b_stop: "maximum_budget_residual_6.366e-6_exceeded_1e-6" + budget_correction: "align_final_budget_tolerance_with_solver_wrappers_at_1e-4_dollars" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + protocol: "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_commit: "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd" + protocol_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + +source_ingest: + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + allowed_raw_columns: ["id", "loan_amnt", "int_rate", "purpose"] + forbidden_tokens: ["status", "outcome", "default", "pymnt", "realized", "miscoverage"] + chunksize: 100000 + +frontier: + roles: ["policy_development", "primary_oot"] + expected_development_months: 11 + expected_primary_months: 15 + expected_windows: 8 + gamma_grid: [0.0, 0.25, 0.50, 0.75, 1.0] + endpoint_contrast: [1.0, 0.0] + rulers: + primary: "objective_matched" + secondary: "normalized_score" + coordinate_grid: [0.25, 0.50, 0.75] + normalized_score: + definition: "score_min_plus_coordinate_times_score_at_objective_minus_score_min" + minimum_score_range: 1.0e-4 + cap_residual_tolerance: 1.0e-8 + objective_matched: + definition: "score_minimizer_subject_to_common_absolute_plugin_objective_floor" + common_lower_endpoint: "maximum_minimum_score_portfolio_objective_over_gamma" + minimum_objective_range_dollars: 1.0e-4 + floor_residual_tolerance_dollars: 1.0e-5 + objective_optimum: + diagnostic: "nonbasic_reduced_costs_plus_reversed_id_order" + dual_tolerance: 1.0e-7 + primal_tolerance: 1.0e-8 + +solver: + primary: "highspy_exact_budget_simplex" + threads: 1 + time_limit_seconds: 300 + allocation_tolerance: 1.0e-8 + budget_residual_tolerance_dollars: 1.0e-4 + order_rule: "ascending_id_vs_descending_id" + order_audit_scope: "objective_optimum_once_per_menu_and_primary_oot_gamma_endpoints" + order_exposure_distance_tolerance: 1.0e-10 + order_objective_tolerance_dollars: 1.0e-5 + endpoint_pair_degeneracy_tolerance: 1.0e-6 + independent_validation: + solver: "ortools_glop" + periods: ["2016-04", "2016-11", "2017-06"] + scope: "gamma_endpoints_all_windows_rulers_and_coordinates" + objective_rate_tolerance: 1.0e-7 + weighted_score_tolerance: 1.0e-7 + +claim_boundary: + outcome_columns_passed: [] + no_policy_selection: true + no_policy_winner: true + no_conformal_guarantee_repair: true + no_equal_true_risk_claim: true + no_equal_shadow_price_claim: true + no_causal_claim: true + no_submission_freeze: true + +stop_rules: + stop_on_incomplete_cell: true + stop_on_score_range_failure: true + stop_on_objective_range_failure: true + stop_on_objective_optimum_tie: true + stop_on_order_sensitivity: true + stop_on_independent_solver_mismatch: true + stop_before_outcomes_if_endpoint_allocations_all_identical: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + solve_records: "frontier_solve_records.parquet" + allocations: "frontier_funded_allocations.parquet" + endpoint_diagnostics: "frontier_endpoint_diagnostics.parquet" + objective_optimum_diagnostics: "objective_optimum_diagnostics.parquet" + order_sensitivity: "frontier_order_sensitivity.parquet" + independent_validation: "frontier_independent_solver_validation.parquet" + deterministic_summary: "normalized_objective_frontier_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml b/configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml new file mode 100644 index 0000000..0f9ed4e --- /dev/null +++ b/configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml @@ -0,0 +1,100 @@ +schema_version: "2026-07-14.1" +protocol_status: "locked_post_freeze_outcome_evaluation" +protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-14-v3" +run_tag: "ijds-normalized-objective-frontier-2026-07-14-v3" + +protocol_lineage_files: + - "docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md" + - "docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + +source_frontier: + run_tag: "ijds-normalized-objective-frontier-2026-07-13-v1c" + protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-13-v1c" + protocol_commit: "46f4df915d38eb5a6cc144484c6e6fe56d8ed397" + status: "outcome_free_frontiers_frozen_before_archive_outcome_join" + freeze: + path: "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/protocol_freeze.json" + bytes: 15192 + sha256: "7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + +evaluation: + evaluated_roles: ["policy_development", "primary_oot"] + primary_contrast_role: "primary_oot" + endpoint_contrast: [1.0, 0.0] + rulers: ["objective_matched", "normalized_score"] + coordinates: [0.25, 0.50, 0.75] + expected_solve_records: 6240 + expected_funded_rows: 622455 + expected_window_contrasts: 48 + expected_monthly_contrasts: 720 + expected_metric_directions: 144 + expected_windows: 8 + expected_primary_months: 15 + expected_candidate_counts: + policy_development: 94885 + primary_oot: 376890 + outcome_join: "single_validated_join_after_hash_verified_v1c_freeze" + unresolved_outcomes: "sharp_common_loanwise_assignment_on_funded_union" + +metrics: + standardized_payoff: + lower: "realized_payoff_difference_lower" + upper: "realized_payoff_difference_upper" + direction_tolerance: 1.0e-4 + funded_default: + lower: "weighted_default_difference_lower" + upper: "weighted_default_difference_upper" + direction_tolerance: 1.0e-10 + funded_binary_miscoverage: + lower: "weighted_miscoverage_difference_lower" + upper: "weighted_miscoverage_difference_upper" + direction_tolerance: 1.0e-10 + +outcomes: + source_columns: ["loan_status", "last_pymnt_d"] + joined_columns: ["snapshot_default", "snapshot_resolution"] + endpoint_contract: "conservative_terminal_status_reconstruction_by_2020-09-30" + no_outcome_refit: true + no_outcome_resolve: true + no_outcome_selection: true + +claim_boundary: + no_policy_selection: true + no_window_selection: true + no_ruler_selection: true + no_coordinate_selection: true + no_gamma_selection: true + no_policy_winner: true + no_causal_claim: true + no_conformal_guarantee_repair: true + no_selected_set_validity: true + no_submission_freeze: true + +stop_rules: + stop_on_v1c_freeze_mismatch: true + stop_on_artifact_descriptor_mismatch: true + stop_on_outcome_alignment_failure: true + stop_on_incomplete_portfolio_evaluation: true + stop_on_incomplete_contrast_census: true + stop_on_nonbinary_observed_outcome: true + retain_all_predeclared_results: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + evaluated_portfolios: "evaluated_portfolios.parquet" + joined_funded_allocations: "joined_funded_allocations.parquet" + window_endpoint_contrasts: "window_endpoint_contrasts.parquet" + monthly_endpoint_contrasts: "monthly_endpoint_contrasts.parquet" + metric_direction_census: "metric_direction_census.parquet" + outcome_join_audit: "outcome_join_audit.parquet" + deterministic_summary: "normalized_objective_frontier_v3_summary.json" + execution_receipt: "execution_receipt.json" + evaluation_manifest: "verified_evaluation_manifest.json" diff --git a/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v4.yaml b/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v4.yaml new file mode 100644 index 0000000..b79ce13 --- /dev/null +++ b/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v4.yaml @@ -0,0 +1,23 @@ +extends: "ijds_normalized_objective_frontier_2026-07-14_v3.yaml" + +schema_version: "2026-07-15.3" +protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-15-v4" +run_tag: "ijds-normalized-objective-frontier-2026-07-15-v4" + +protocol_lineage_files: + - "docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md" + - "docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v4.yaml" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_no_scientific_metric_change" + require_exact_reference_column_equivalence: true + artifact_section: "evaluation_artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3/verified_evaluation_manifest.json" + bytes: 16944 + sha256: "76e3643b92a537c9e0cea4d131f8085d4c843ead0cb95362f19a3442207af96e" diff --git a/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v5.yaml b/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v5.yaml new file mode 100644 index 0000000..ac4d9fe --- /dev/null +++ b/configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v5.yaml @@ -0,0 +1,27 @@ +extends: "ijds_normalized_objective_frontier_2026-07-15_v4.yaml" + +schema_version: "2026-07-15.4" +protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-15-v5" +run_tag: "ijds-normalized-objective-frontier-2026-07-15-v5" + +protocol_lineage_files: + - "docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md" + - "docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + - "docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md" + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" + +endpoint_reason_recovery: + status: "reason_taxonomy_only_machine_tolerance_recovery" + equivalence_mode: "exact_non_float_machine_tolerant_float" + require_exact_non_float_reference_equivalence: true + float_atol: 5.0e-14 + float_rtol: 5.0e-14 + artifact_section: "evaluation_artifacts" + reference_json: + path: "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3/verified_evaluation_manifest.json" + bytes: 16944 + sha256: "76e3643b92a537c9e0cea4d131f8085d4c843ead0cb95362f19a3442207af96e" diff --git a/configs/experiments/ijds_policy_support_tie_audit_2026-07-12.yaml b/configs/experiments/ijds_policy_support_tie_audit_2026-07-12.yaml new file mode 100644 index 0000000..bf5487b --- /dev/null +++ b/configs/experiments/ijds_policy_support_tie_audit_2026-07-12.yaml @@ -0,0 +1,85 @@ +schema_version: "2026-07-12.1" +protocol_status: "locked_outcome_free_structural_audit_before_execution" +protocol_tag: "protocol/ijds-policy-support-tie-audit-2026-07-12-v1" +run_tag: "ijds-policy-support-tie-audit-2026-07-12-v1" + +hypothesis: >- + The inherited interior policy grid can be interpreted only after its feasible + and decision-active domain is compared with the point and full-endpoint score + limits, while comparator conclusions require a declared outcome-free cap + support and evidence that HiGHS tie resolution does not choose the result. + +parent: + config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + protocol: "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_commit: "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd" + protocol_freeze: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json" + bytes: 20362 + sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + +source_ingest: + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + allowed_raw_columns: ["id", "loan_amnt", "int_rate", "purpose"] + forbidden_tokens: ["status", "outcome", "default", "pymnt", "realized", "miscoverage"] + chunksize: 100000 + +family_audit: + roles: ["policy_development", "primary_oot"] + expected_development_months: 11 + expected_primary_months: 15 + expected_windows: 8 + risk_tolerances: [0.15, 0.17, 0.19] + gamma_grid: [0.0, 0.25, 0.50, 0.75, 1.0] + inherited_interior_gammas: [0.25, 0.50, 0.75] + gamma_zero_role: "point_score_endpoint_already_represented_by_c0" + gamma_one_role: "full_conformal_upper_endpoint_diagnostic_not_automatic_promotion" + cap_classification_tolerance: 1.0e-8 + reconstruction_objective_tolerance: 1.0e-4 + reconstruction_score_tolerance: 1.0e-8 + minimum_score_range: 1.0e-8 + +comparator_support: + named_rules: + - "c0_same_numeric_cap" + - "c1_development_mean" + - "c2_contemporaneous" + development_definition: "window_policy_monthly_guardrail_point_moment_hull" + broad_stress: [0.05, 0.12] + broad_stress_role: "sensitivity_only_not_normative_admissibility" + cap_deduplication_tolerance: 1.0e-10 + cap_census_cardinality: "derived_from_tolerance_union_not_preselected" + +tie_audit: + cap_family: + - "named_c0_c1_c2" + - "development_support_endpoints" + - "period_specific_basis_breakpoints" + - "broad_stress_endpoints" + dual_tolerance: 1.0e-7 + primal_tolerance: 1.0e-9 + objective_reconciliation_tolerance: 1.0e-5 + order_sensitivity_tolerance: 1.0e-10 + order_rule: "original_source_order_vs_descending_loan_id" + rerun_trigger: "near_zero_nonbasic_reduced_cost_or_primal_degenerate_basis" + threads: 1 + +claim_boundary: + outcome_columns_passed: [] + no_empirical_metric_or_direction: true + no_policy_promotion: true + no_universal_support_claim: true + no_unique_optimum_claim_without_diagnostics: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" + family_feasibility: "policy_family_feasibility.parquet" + point_cap_diagnostics: "point_cap_basis_diagnostics.parquet" + order_sensitivity: "point_cap_order_sensitivity.parquet" + deterministic_result: "policy_support_tie_audit_summary.json" + execution_receipt: "execution_receipt.json" diff --git a/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v5.yaml b/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v5.yaml new file mode 100644 index 0000000..cc316d9 --- /dev/null +++ b/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v5.yaml @@ -0,0 +1,112 @@ +schema_version: "2026-07-15.4" +protocol_status: "locked_retrospective_outcome_free_structural_sensitivity_v5_retry_ladder" +protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5" +run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v5" +protocol_documents: + - "docs/research/ijds_portfolio_structure_sensitivity_v2_protocol_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v3_execution_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v4_recovery_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v5_protocol_2026-07-15.md" + +lineage: + inherits_scientific_protocol: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v2" + inherits_parallel_execution: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v3" + inherits_recovery_contract: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v4" + scientific_grid_changed: false + endpoint_changed: false + retry_ladder_amendment_only: true + outcomes_inspected_for_amendment: false + +parent: + frontier_config: "configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml" + evaluation_config: "configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml" + v4_config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + +structural_grid: + budgets: [500000.0, 1000000.0, 2000000.0] + purpose_caps: [0.20, 0.25, 0.30, 1.00] + lgds: [0.25, 0.45, 0.65] + design: "complete_cartesian_product" + scenarios: 36 + roles: ["primary_oot"] + state_gamma_grid: [0.0, 0.25, 0.50, 0.75, 1.0] + reported_gamma_grid: [0.0, 1.0] + rulers: ["objective_matched", "normalized_score"] + coordinates: [0.25, 0.50, 0.75] + all_scenarios_windows_rulers_coordinates_and_metrics_reported: true + +numerics: + minimum_endpoint_retry_slacks: [1.0e-12, 1.0e-10] + maximum_minimum_endpoint_retry_slack: 1.0e-10 + retry_scope: "closed_known_boundary_status_ladder" + known_boundary_failures: + - "Point LP is not optimal: Infeasible." + - "Point LP is not optimal: Unknown." + - "Point LP did not fill its budget:" + cap_residual_tolerance: 1.0e-8 + maximum_retry_is_below_cap_residual_tolerance: true + retry_count_and_applied_slack_persisted: true + +execution: + freeze_workers: 3 + worker_start_method: "spawn" + solver_threads_per_worker: 1 + worker_input_load: "hash_recorded_outcome_free_parquet_once_per_worker" + scenario_write_scope: "disjoint_directory_per_scenario" + consolidation_order: "scenario_id_sorted" + recovery: + source_run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v4" + source_protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v4" + source_protocol_commit: "dc7d99a746310c090fa8efe80170a7287d9c2f64" + expected_recovered_scenarios: 33 + expected_missing_scenario_ids: + - "b0500k_p020_l025" + - "b0500k_p020_l045" + - "b0500k_p020_l065" + require_complete_artifact_inventory: true + require_expected_row_counts: true + require_outcome_free_columns: true + require_scenario_identity: true + require_numerical_contract: true + copy_method: "ntfs_hardlink_after_validation" + +endpoint: + evaluation_cutoff: "2020-09-30" + charged_off_lag_months: 6 + +claim_boundary: + historical_archive_previously_inspected: true + baseline_binding_diagnostics_previously_inspected: true + v1_numerical_failure_inspected: true + v3_interruption_state_inspected: true + v4_unknown_solver_state_inspected: true + preregistered: false + confirmatory: false + prospective: false + outcome_based_scenario_selection: false + model_refit: false + conformal_refit: false + ruler_selection: false + scenario_selection: false + policy_winner: false + no_submission_freeze: true + +stop_rules: + stop_on_source_hash_mismatch: true + stop_on_outcome_column_in_freeze: true + stop_on_incomplete_scenario_grid: true + stop_on_incomplete_frontier_cell: true + stop_on_solver_or_order_mismatch: true + stop_on_incomplete_evaluation: true + stop_on_unrecognized_solver_failure: true + stop_after_closed_retry_ladder: true + stop_on_retry_above_declared_ladder: true + stop_on_invalid_recovered_shard: true + retain_every_declared_scenario: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml b/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml new file mode 100644 index 0000000..9ba20f1 --- /dev/null +++ b/configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml @@ -0,0 +1,120 @@ +schema_version: "2026-07-15.5" +protocol_status: "locked_retrospective_outcome_free_structural_sensitivity_v6_order_tolerance" +protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6" +run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v6" +protocol_documents: + - "docs/research/ijds_portfolio_structure_sensitivity_v2_protocol_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v3_execution_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v4_recovery_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v5_protocol_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v5_stop_2026-07-15.md" + - "docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md" + +lineage: + inherits_scientific_protocol: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v2" + inherits_parallel_execution: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v3" + inherits_recovery_contract: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v4" + inherits_retry_ladder: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5" + scientific_grid_changed: false + endpoint_changed: false + retry_ladder_changed: false + order_exposure_tolerance_amendment_only: true + outcomes_inspected_for_amendment: false + +parent: + frontier_config: "configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml" + evaluation_config: "configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml" + v4_config: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + raw_path: "data/raw/Loan_status_2007-2020Q3.csv" + raw_sha256: "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f" + +structural_grid: + budgets: [500000.0, 1000000.0, 2000000.0] + purpose_caps: [0.20, 0.25, 0.30, 1.00] + lgds: [0.25, 0.45, 0.65] + design: "complete_cartesian_product" + scenarios: 36 + roles: ["primary_oot"] + state_gamma_grid: [0.0, 0.25, 0.50, 0.75, 1.0] + reported_gamma_grid: [0.0, 1.0] + rulers: ["objective_matched", "normalized_score"] + coordinates: [0.25, 0.50, 0.75] + all_scenarios_windows_rulers_coordinates_and_metrics_reported: true + +numerics: + minimum_endpoint_retry_slacks: [1.0e-12, 1.0e-10] + maximum_minimum_endpoint_retry_slack: 1.0e-10 + retry_scope: "closed_known_boundary_status_ladder" + known_boundary_failures: + - "Point LP is not optimal: Infeasible." + - "Point LP is not optimal: Unknown." + - "Point LP did not fill its budget:" + cap_residual_tolerance: 1.0e-8 + order_exposure_distance_tolerance: 1.0e-8 + order_objective_tolerance_dollars: 1.0e-5 + maximum_retry_is_below_cap_residual_tolerance: true + retry_count_and_applied_slack_persisted: true + order_tolerance_diagnosed_without_outcomes: true + +execution: + freeze_workers: 1 + worker_start_method: "spawn" + solver_threads_per_worker: 1 + worker_input_load: "hash_recorded_outcome_free_parquet_once_per_worker" + scenario_write_scope: "disjoint_directory_per_scenario" + consolidation_order: "scenario_id_sorted" + recovery: + source_run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v5" + source_protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5" + source_protocol_commit: "481528a6973eded94189435957f9b8064bc4bb06" + expected_recovered_scenarios: 35 + expected_missing_scenario_ids: + - "b0500k_p020_l025" + require_complete_artifact_inventory: true + require_expected_row_counts: true + require_outcome_free_columns: true + require_scenario_identity: true + require_numerical_contract: true + copy_method: "ntfs_hardlink_after_validation" + +endpoint: + evaluation_cutoff: "2020-09-30" + charged_off_lag_months: 6 + +claim_boundary: + historical_archive_previously_inspected: true + baseline_binding_diagnostics_previously_inspected: true + v1_numerical_failure_inspected: true + v3_interruption_state_inspected: true + v4_unknown_solver_state_inspected: true + v5_order_failure_inspected: true + v5_outcome_free_numerical_diagnosis_inspected: true + preregistered: false + confirmatory: false + prospective: false + outcome_based_scenario_selection: false + model_refit: false + conformal_refit: false + ruler_selection: false + scenario_selection: false + policy_winner: false + no_submission_freeze: true + +stop_rules: + stop_on_source_hash_mismatch: true + stop_on_outcome_column_in_freeze: true + stop_on_incomplete_scenario_grid: true + stop_on_incomplete_frontier_cell: true + stop_on_order_exposure_distance_above_1e_8: true + stop_on_order_objective_difference_above_1e_5_dollars: true + stop_on_incomplete_evaluation: true + stop_on_unrecognized_solver_failure: true + stop_after_closed_retry_ladder: true + stop_on_retry_above_declared_ladder: true + stop_on_invalid_recovered_shard: true + retain_every_declared_scenario: true + +output: + data_root: "data/processed/experiments/ijds_audit" + model_root: "models/experiments/ijds_audit" + immutability: "hard_no_overwrite_choose_fresh_run_tag" diff --git a/configs/experiments/ijds_raw_data_contract_2026-07-14_v2.yaml b/configs/experiments/ijds_raw_data_contract_2026-07-14_v2.yaml new file mode 100644 index 0000000..243267f --- /dev/null +++ b/configs/experiments/ijds_raw_data_contract_2026-07-14_v2.yaml @@ -0,0 +1,30 @@ +schema_version: "2026-07-14.1" +status: "retrospective_full_archive_data_contract_audit" +run_tag: "ijds-raw-data-contract-2026-07-14-v2" +base_protocol: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + +questions: + - "Does the distributed archive establish the claimed endpoint date?" + - "Which raw columns meet the ordinary temporal coverage threshold?" + - "Which active inputs require explicit structural-null or legacy-schema exceptions?" + - "Do those exceptions require a model sensitivity before paper-facing use?" + +rules: + term_months: 36 + minimum_fitting_feature_coverage: 0.95 + late_feature_primary_coverage: 0.80 + late_feature_fitting_coverage: 0.50 + active_feature_coverage_exceptions: + mths_since_last_delinq: + type: "structurally_nullable_credit_history_field" + missingness_semantics: "frozen feature engineering maps missing recency to 999" + requires_sensitivity: true + pub_rec_bankruptcies: + type: "legacy_schema_partial_coverage" + missingness_semantics: "frozen feature engineering maps missing bankruptcy count to no recorded bankruptcy" + requires_sensitivity: true + no_model_or_policy_selection: true + no_protected_stage_execution: true + +output: + root: "reports/crpto/data_audit" diff --git a/configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml b/configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml new file mode 100644 index 0000000..d3041c2 --- /dev/null +++ b/configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml @@ -0,0 +1,61 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + +schema_version: "2026-07-12.4" +protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-12-v1" +run_tag: "ijds-rolling-origin-2015-2026-07-12-v1" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md" + +hypothesis: >- + The V4 binary-geometry and comparator-identification findings may depend on + the calendar origin. A symmetric three-month rolling-origin audit must + therefore report 2015, 2016, and 2017 without selecting an origin, residual + window, learner, policy, comparator, or metric from outcomes. + +source: + information_cutoff: "2015-03-31" + +design: + development_end: "2009-12-31" + probability_calibration_start: "2010-01-01" + probability_calibration_end: "2010-12-31" + conformal_fit_start: "2011-01-01" + conformal_fit_end: "2012-01-31" + policy_development_start: "2012-02-01" + policy_development_end: "2012-12-31" + primary_oot_start_month: "2015-04" + primary_oot_end_month: "2015-06" + censored_extension_start_month: "2015-07" + censored_extension_end_month: "2015-09" + +conformal: + taxonomy_source: "2010_all_status_independent_calibrated_scores" + +learner_control: + platt_calibration: "independent_2010_raw_margin" + +residual_specification: + windows: + - {id: "w01_2011m01_m06", start: "2011-01-01", end: "2011-06-30"} + - {id: "w02_2011m02_m07", start: "2011-02-01", end: "2011-07-31"} + - {id: "w03_2011m03_m08", start: "2011-03-01", end: "2011-08-31"} + - {id: "w04_2011m04_m09", start: "2011-04-01", end: "2011-09-30"} + - {id: "w05_2011m05_m10", start: "2011-05-01", end: "2011-10-31"} + - {id: "w06_2011m06_m11", start: "2011-06-01", end: "2011-11-30"} + - {id: "w07_2011m07_m12", start: "2011-07-01", end: "2011-12-31"} + - {id: "w08_2011m08_2012m01", start: "2011-08-01", end: "2012-01-31"} + +rolling_origin: + audit_id: "symmetric_calendar_origin_stability_v1" + origin_year: 2015 + common_primary_months: 3 + reference_origin_year: 2016 + outcome_based_origin_selection: false + pooled_origin_claims: false + +resume_outcome_free: null diff --git a/configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml b/configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml new file mode 100644 index 0000000..b48e6c8 --- /dev/null +++ b/configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml @@ -0,0 +1,61 @@ +extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + +schema_version: "2026-07-12.4" +protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-12-v1" +run_tag: "ijds-rolling-origin-2017-2026-07-12-v1" + +protocol_lineage_files: + - "configs/experiments/ijds_fixed_taxonomy_c2_2026-07-11.yaml" + - "docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md" + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md" + +hypothesis: >- + The V4 binary-geometry and comparator-identification findings may depend on + the calendar origin. A symmetric three-month rolling-origin audit must + therefore report 2015, 2016, and 2017 without selecting an origin, residual + window, learner, policy, comparator, or metric from outcomes. + +source: + information_cutoff: "2017-03-31" + +design: + development_end: "2011-12-31" + probability_calibration_start: "2012-01-01" + probability_calibration_end: "2012-12-31" + conformal_fit_start: "2013-01-01" + conformal_fit_end: "2014-01-31" + policy_development_start: "2014-02-01" + policy_development_end: "2014-12-31" + primary_oot_start_month: "2017-04" + primary_oot_end_month: "2017-06" + censored_extension_start_month: "2017-07" + censored_extension_end_month: "2017-09" + +conformal: + taxonomy_source: "2012_all_status_independent_calibrated_scores" + +learner_control: + platt_calibration: "independent_2012_raw_margin" + +residual_specification: + windows: + - {id: "w01_2013m01_m06", start: "2013-01-01", end: "2013-06-30"} + - {id: "w02_2013m02_m07", start: "2013-02-01", end: "2013-07-31"} + - {id: "w03_2013m03_m08", start: "2013-03-01", end: "2013-08-31"} + - {id: "w04_2013m04_m09", start: "2013-04-01", end: "2013-09-30"} + - {id: "w05_2013m05_m10", start: "2013-05-01", end: "2013-10-31"} + - {id: "w06_2013m06_m11", start: "2013-06-01", end: "2013-11-30"} + - {id: "w07_2013m07_m12", start: "2013-07-01", end: "2013-12-31"} + - {id: "w08_2013m08_2014m01", start: "2013-08-01", end: "2014-01-31"} + +rolling_origin: + audit_id: "symmetric_calendar_origin_stability_v1" + origin_year: 2017 + common_primary_months: 3 + reference_origin_year: 2016 + outcome_based_origin_selection: false + pooled_origin_claims: false + +resume_outcome_free: null diff --git a/configs/experiments/ijds_rolling_origin_2017_2026-07-12_v2.yaml b/configs/experiments/ijds_rolling_origin_2017_2026-07-12_v2.yaml new file mode 100644 index 0000000..7a34f1f --- /dev/null +++ b/configs/experiments/ijds_rolling_origin_2017_2026-07-12_v2.yaml @@ -0,0 +1,13 @@ +extends: "ijds_rolling_origin_2017_2026-07-12.yaml" + +schema_version: "2026-07-12.5" +protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-12-v2" +run_tag: "ijds-rolling-origin-2017-2026-07-12-v2" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml" + - "configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml" + - "docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md" + - "docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md" diff --git a/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v3.yaml b/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v3.yaml new file mode 100644 index 0000000..744b24b --- /dev/null +++ b/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v3.yaml @@ -0,0 +1,37 @@ +extends: "ijds_rolling_origin_2017_2026-07-12_v2.yaml" + +schema_version: "2026-07-15.3" +protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-15-v3" +run_tag: "ijds-rolling-origin-2017-2026-07-15-v3" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml" + - "configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml" + - "docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md" + - "docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + - "docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md" + +source: + snapshot_date: "2020-09-30" + snapshot_date_role: "reconstructed_endpoint_cutoff_not_verified_archive_snapshot" + +target: + name: "terminal_default_reconstructed_by_cutoff" + evaluation_outcome_contract: + mode: "conservative_terminal_status_reconstruction" + cutoff: "2020-09-30" + terminal_status_source: "distributed_archive_final_status" + archive_is_verified_point_in_time_snapshot: false + charged_off_reporting_lag_months: 6 + +design: + endpoint: "terminal_default_reconstructed_as_observable_by_2020-09-30" + +resume_outcome_free: + source_run_tag: "ijds-rolling-origin-2017-2026-07-12-v2" + source_protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-12-v2" + source_protocol_commit: "9e689b2e3ca18aae5a2a967cc186da5dcd140891" + source_freeze_sha256: "e224e1ae534435d1b166a07c50fb1ce907b07d36257f37e826ee41a0cb086759" diff --git a/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v4.yaml b/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v4.yaml new file mode 100644 index 0000000..149dd15 --- /dev/null +++ b/configs/experiments/ijds_rolling_origin_2017_2026-07-15_v4.yaml @@ -0,0 +1,16 @@ +extends: "ijds_rolling_origin_2017_2026-07-15_v3.yaml" + +schema_version: "2026-07-15.4" +protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-15-v4" +run_tag: "ijds-rolling-origin-2017-2026-07-15-v4" + +protocol_lineage_files: + - "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + - "docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md" + - "configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml" + - "configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml" + - "docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md" + - "docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md" + - "docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md" + - "docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md" + - "docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md" diff --git a/configs/experiments/tabpfn_tabprep_full.yaml b/configs/experiments/tabpfn_tabprep_full.yaml deleted file mode 100644 index 61d9d53..0000000 --- a/configs/experiments/tabpfn_tabprep_full.yaml +++ /dev/null @@ -1,49 +0,0 @@ -run_tag: tabpfn-tabprep-full-2026-06-16 - -data: - train_path: data/processed/train_fe.parquet - calibration_path: data/processed/calibration_fe.parquet - test_path: data/processed/test_fe.parquet - feature_config_path: data/processed/feature_config.yml - -tabprep: - variant: full_3000 - max_tabpfn_features: 2000 - extra_blacklist: - - id - - url - - loan_status - - emp_title - - title - - zip_code - - addr_state - - next_pymnt_d - - pymnt_plan - - application_type - - verification_status_joint - -tabpfn: - n_estimators: 8 - device: cuda - use_native_categorical_features: false - ignore_pretraining_limits: true - inference_precision: auto - fit_mode: low_memory - memory_saving_mode: true - keep_cache_on_device: false - n_preprocessing_jobs: 1 - show_progress_bar: true - prediction_chunk_rows: 1000 - inference_config: - ENABLE_GPU_PREPROCESSING: false - FEATURE_SUBSAMPLING_METHOD: gini_feature_importance - FEATURE_SUBSAMPLING_IMPORTANCE_TOP_K_COUNT: 150 - -calibration: - method: platt - -output: - data_dir: data/processed/experiments/tabpfn_tabprep - model_dir: models/experiments/tabpfn_tabprep - report_dir: reports/crpto/experiments/tabpfn_tabprep - save_classifier_pickle: false diff --git a/configs/experiments/tabprep_catboost_full.yaml b/configs/experiments/tabprep_catboost_full.yaml deleted file mode 100644 index 7987092..0000000 --- a/configs/experiments/tabprep_catboost_full.yaml +++ /dev/null @@ -1,90 +0,0 @@ -run_tag: tabprep-catboost-full-2026-06-17 -seeds: [42] -sample_rows: 0 - -data: - train_path: data/processed/train_fe.parquet - calibration_path: data/processed/calibration_fe.parquet - test_path: data/processed/test_fe.parquet - feature_config_path: data/processed/feature_config.yml - -validation: - scheme: temporal_train_val_cal_test - val_fraction: 0.15 - date_col: issue_d - -tabprep: - variants: [full_3000] - persist_transformed: false - suspicious_auc_threshold: 0.90 - extra_blacklist: - - id - - url - - loan_status - - emp_title - - title - - zip_code - - addr_state - - next_pymnt_d - - pymnt_plan - - application_type - - verification_status_joint - monotonic_constraints: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - -model: - type: catboost - params: - iterations: 4200 - learning_rate: 0.057321202729872456 - depth: 8 - l2_leaf_reg: 119.37272987133554 - min_data_in_leaf: 135 - rsm: 0.5716653769355704 - random_strength: 1.3208942645900998e-07 - border_count: 254 - bootstrap_type: MVS - subsample: 0.678878683066026 - loss_function: Logloss - auto_class_weights: Balanced - eval_metric: AUC - has_time: true - allow_writing_files: false - random_seed: 42 - early_stopping_rounds: 150 - grow_policy: SymmetricTree - leaf_estimation_iterations: 5 - monotone_constraints: installment:1,annual_inc:-1,dti:1,loan_to_income:1 - feature_weights: - loan_to_income: 1.3 - annual_inc: 1.2 - dti: 1.2 - installment: 1.0 - penalties_coefficient: 0.75 - -calibration: - method: auto - candidates: [platt, isotonic, venn_abers] - -evaluation: - fairness_threshold: 0.50 - fairness_columns: - - grade - - sub_grade - - term - - purpose - - home_ownership - -output: - data_dir: data/processed/experiments/tabprep - model_dir: models/experiments/tabprep - report_dir: reports/crpto/experiments/tabprep - -downstream: - enabled: false - requires_explicit_approval: true - conformal_config_path: configs/experiments/tabprep_conformal_policy.yaml - optimization_config_path: configs/experiments/tabprep_optimization.yaml diff --git a/configs/experiments/tabprep_challenger.yaml b/configs/experiments/tabprep_challenger.yaml deleted file mode 100644 index c8777e2..0000000 --- a/configs/experiments/tabprep_challenger.yaml +++ /dev/null @@ -1,90 +0,0 @@ -run_tag: tabprep-challenger-2026-06-16 -seeds: [42, 52, 62] -sample_rows: 0 - -data: - train_path: data/processed/train_fe.parquet - calibration_path: data/processed/calibration_fe.parquet - test_path: data/processed/test_fe.parquet - feature_config_path: data/processed/feature_config.yml - -validation: - scheme: temporal_train_val_cal_test - val_fraction: 0.15 - date_col: issue_d - -tabprep: - variants: [safe_500, balanced_1500, full_3000] - persist_transformed: false - suspicious_auc_threshold: 0.90 - extra_blacklist: - - id - - url - - loan_status - - emp_title - - title - - zip_code - - addr_state - - next_pymnt_d - - pymnt_plan - - application_type - - verification_status_joint - monotonic_constraints: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - -model: - type: catboost - params: - iterations: 4200 - learning_rate: 0.057321202729872456 - depth: 8 - l2_leaf_reg: 119.37272987133554 - min_data_in_leaf: 135 - rsm: 0.5716653769355704 - random_strength: 1.3208942645900998e-07 - border_count: 254 - bootstrap_type: MVS - subsample: 0.678878683066026 - loss_function: Logloss - auto_class_weights: Balanced - eval_metric: AUC - has_time: true - allow_writing_files: false - random_seed: 42 - early_stopping_rounds: 150 - grow_policy: SymmetricTree - leaf_estimation_iterations: 5 - monotone_constraints: installment:1,annual_inc:-1,dti:1,loan_to_income:1 - feature_weights: - loan_to_income: 1.3 - annual_inc: 1.2 - dti: 1.2 - installment: 1.0 - penalties_coefficient: 0.75 - -calibration: - method: auto - candidates: [platt, isotonic, venn_abers] - -evaluation: - fairness_threshold: 0.50 - fairness_columns: - - grade - - sub_grade - - term - - purpose - - home_ownership - -output: - data_dir: data/processed/experiments/tabprep - model_dir: models/experiments/tabprep - report_dir: reports/crpto/experiments/tabprep - -downstream: - enabled: false - requires_explicit_approval: true - conformal_config_path: configs/experiments/tabprep_conformal_policy.yaml - optimization_config_path: configs/experiments/tabprep_optimization.yaml diff --git a/configs/experiments/tabprep_conformal_policy.yaml b/configs/experiments/tabprep_conformal_policy.yaml deleted file mode 100644 index cc3fe99..0000000 --- a/configs/experiments/tabprep_conformal_policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Isolated downstream placeholder for an approved TabPrep conformal experiment. -# Do not run protected canonical DVC stages with this config. -run_tag: tabprep-challenger-2026-06-16 -enabled: false -requires_explicit_approval: true - -input: - predictions_path: data/processed/experiments/tabprep/tabprep-challenger-2026-06-16/safe_500/seed_42/test_predictions.parquet - calibrator_path: models/experiments/tabprep/tabprep-challenger-2026-06-16/safe_500/seed_42/pd_tabprep_calibrator.pkl - -output: - intervals_path: data/processed/experiments/tabprep/tabprep-challenger-2026-06-16/conformal/conformal_intervals.parquet - validation_path: models/experiments/tabprep/tabprep-challenger-2026-06-16/conformal/conformal_validation_status.json - report_dir: reports/crpto/experiments/tabprep/tabprep-challenger-2026-06-16/conformal - -guardrails: - forbidden_outputs: - - data/processed/conformal_intervals_mondrian.parquet - - models/conformal_policy_status.json - - EXTRACTION_MANIFEST.json diff --git a/configs/experiments/tabprep_optimization.yaml b/configs/experiments/tabprep_optimization.yaml deleted file mode 100644 index 216497e..0000000 --- a/configs/experiments/tabprep_optimization.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Isolated downstream placeholder for an approved TabPrep portfolio experiment. -# This must not run crpto.portfolio.bound_exact_eval. -run_tag: tabprep-challenger-2026-06-16 -enabled: false -requires_explicit_approval: true -run_bound_exact_eval: false - -input: - predictions_path: data/processed/experiments/tabprep/tabprep-challenger-2026-06-16/safe_500/seed_42/test_predictions.parquet - intervals_path: data/processed/experiments/tabprep/tabprep-challenger-2026-06-16/conformal/conformal_intervals.parquet - -output: - allocation_path: data/processed/experiments/tabprep/tabprep-challenger-2026-06-16/portfolio/portfolio_allocations.parquet - status_path: models/experiments/tabprep/tabprep-challenger-2026-06-16/portfolio/portfolio_status.json - report_dir: reports/crpto/experiments/tabprep/tabprep-challenger-2026-06-16/portfolio - -guardrails: - forbidden_outputs: - - data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734 - - EXTRACTION_MANIFEST.json diff --git a/configs/ijds_active_evidence_sources.yaml b/configs/ijds_active_evidence_sources.yaml new file mode 100644 index 0000000..cfd021d --- /dev/null +++ b/configs/ijds_active_evidence_sources.yaml @@ -0,0 +1,266 @@ +schema_version: "2026-07-15.4" +status: "active_ijds_paper_evidence_source_registry" + +lineages: + binary_geometry: + outcome_free: + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1" + protocol_commit: "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + freeze_sha256: "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + paper_role: "outcome_free_freeze" + dvc_tracked: true + evaluation: + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-15-v5" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5" + protocol_commit: "e2bba580a0b07c145bd64ff61440973d6e31349b" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "verified_evaluation" + dvc_tracked: true + two_ruler: + outcome_free: + run_tag: "ijds-normalized-objective-frontier-2026-07-13-v1c" + protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-13-v1c" + protocol_commit: "46f4df915d38eb5a6cc144484c6e6fe56d8ed397" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + freeze_sha256: "7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185" + paper_role: "outcome_free_freeze" + dvc_tracked: true + evaluation: + run_tag: "ijds-normalized-objective-frontier-2026-07-15-v5" + protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-15-v5" + protocol_commit: "e2bba580a0b07c145bd64ff61440973d6e31349b" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "verified_evaluation" + dvc_tracked: true + credit_controls: + outcome_free: + run_tag: "ijds-credit-risk-controls-2026-07-13-v1b" + protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-13-v1b" + protocol_commit: "1776cbf8b201ae5b92756e5ea397a403d6cc7c9f" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + freeze_sha256: "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35" + paper_role: "outcome_free_freeze" + dvc_tracked: true + evaluation: + run_tag: "ijds-credit-risk-controls-2026-07-15-v5" + protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-15-v5" + protocol_commit: "e2bba580a0b07c145bd64ff61440973d6e31349b" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "verified_evaluation" + dvc_tracked: true + diagnostics: + raw_data_audit: + run_tag: "ijds-raw-data-contract-2026-07-14-v2" + status: "complete_full_archive_data_contract_audit" + paper_role: "data_contract_diagnostic" + dvc_tracked: false + label_lag_sensitivity: + run_tag: "ijds-label-lag-sensitivity-2026-07-14-v1" + protocol_tag: "protocol/ijds-label-lag-sensitivity-2026-07-14-v1" + protocol_commit: "688f75dc4f285c75bc499c9e041dd30fb3acd70d" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "fit_label_timing_sensitivity" + dvc_tracked: false + solver_tie_audit: + run_tag: "ijds-policy-support-tie-audit-2026-07-12-v1" + protocol_tag: "protocol/ijds-policy-support-tie-audit-2026-07-12-v1" + protocol_commit: "115eaf1b81ed5f05ffe601e9c20079059c40c200" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "evaluated_cap_numerical_diagnostic" + dvc_tracked: false + +sensitivities: + endpoint_availability: + run_tag: "ijds-endpoint-availability-sensitivity-2026-07-14-v1" + protocol_tag: "protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1" + protocol_commit: "8865f1cfbd387576bdf805f3e52f030261e4b717" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "evaluation_endpoint_timing_sensitivity" + dvc_tracked: true + portfolio_structure: + run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v6" + protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6" + protocol_commit: "490c653a43e2003d83184f47e1277bd2d4390c43" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "complete_budget_concentration_lgd_sensitivity" + dvc_tracked: true + rolling_origin: + run_tag: "ijds-rolling-origin-2017-2026-07-15-v4" + protocol_tag: "protocol/ijds-rolling-origin-stability-2026-07-15-v4" + protocol_commit: "e2bba580a0b07c145bd64ff61440973d6e31349b" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "two_origin_retrospective_recurrence" + dvc_tracked: true + missingness_encoding: + run_tag: "ijds-missingness-sensitivity-2026-07-15-v3" + protocol_tag: "protocol/ijds-missingness-sensitivity-2026-07-15-v3" + protocol_commit: "199afb083da37af6a51d5ba9e3c4d6280b952fe9" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "feature_semantics_preserving_encoding_sensitivity" + dvc_tracked: true + fit_label_completion: + run_tag: "ijds-fit-label-completion-sensitivity-2026-07-16-v2" + protocol_tag: "protocol/ijds-fit-label-completion-sensitivity-2026-07-16-v2" + protocol_commit: "fbcafcf84645024b9753aba2f04a4263b8e76236" + scientific_uv_lock_sha256: "25cefb168506538c22b86a348c42869ea7fda64338815f2adea3fe7e07608f93" + paper_role: "declared_fit_label_scenario_sensitivity" + dvc_tracked: true + allocation_granularity: + run_tag: "ijds-allocation-granularity-sensitivity-2026-07-16-v3" + protocol_tag: "protocol/ijds-allocation-granularity-sensitivity-2026-07-16-v3" + protocol_commit: "fb1a7b1837d1f8ab2b81239533f51c996f41671c" + scientific_uv_lock_sha256: "25cefb168506538c22b86a348c42869ea7fda64338815f2adea3fe7e07608f93" + paper_role: "continuous_allocation_granularity_diagnostic" + dvc_tracked: true + +replay_dependencies: + binary_geometry_v3_reason_reference: + run_tag: "ijds-binary-geometry-frontier-v4-2026-07-14-v3" + protocol_tag: "protocol/ijds-binary-geometry-frontier-v4-2026-07-14-v3" + protocol_commit: "688f75dc4f285c75bc499c9e041dd30fb3acd70d" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "non_evidence_v5_reason_recovery_reference" + dvc_tracked: true + normalized_frontier_v3_reason_reference: + run_tag: "ijds-normalized-objective-frontier-2026-07-14-v3" + protocol_tag: "protocol/ijds-normalized-objective-frontier-2026-07-14-v3" + protocol_commit: "a1ae516a6c9674686dba245cb275475073b298a0" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "non_evidence_v5_reason_recovery_reference" + dvc_tracked: true + credit_controls_v3_reason_reference: + run_tag: "ijds-credit-risk-controls-2026-07-14-v3" + protocol_tag: "protocol/ijds-credit-risk-controls-2026-07-14-v3" + protocol_commit: "688f75dc4f285c75bc499c9e041dd30fb3acd70d" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "non_evidence_v5_reason_recovery_reference" + dvc_tracked: true + portfolio_structure_v5_shards: + run_tag: "ijds-portfolio-structure-sensitivity-2026-07-15-v5" + protocol_tag: "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5" + protocol_commit: "481528a6973eded94189435957f9b8064bc4bb06" + scientific_uv_lock_sha256: "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2" + paper_role: "non_evidence_structural_v6_replay_dependency" + dvc_tracked: true + dvc_roots: ["data/processed"] + +dvc_pointers: + - "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc" + - "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc" + - "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc" + - "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc" + - "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc" + - "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc" + - "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc" + - "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc" + - "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc" + - "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc" + - "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc" + - "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc" + - "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc" + - "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc" + - "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc" + - "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc" + - "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc" + - "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc" + - "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc" + - "models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc" + - "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc" + - "models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc" + - "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc" + - "models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc" + - "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc" + - "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc" + - "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc" + - "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc" + - "data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc" + - "models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc" + - "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v5.dvc" + +sources: + v4_config: + path: "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" + bytes: 1371 + sha256: "c749befbd0ab7e0f8d6fcded7e7c730cae998032f897f65c7e7673d2a12c3715" + v4_summary: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/binary_geometry_frontier_v4_summary.json" + bytes: 58584 + sha256: "07e793df7e457c6afa946e04d3e95d30307faa48424249659ae8d921ea060a3f" + v4_receipt: + path: "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/execution_receipt.json" + bytes: 1142 + sha256: "b67cf57d200e71417f4a0b51ac2a2978b8b5105396ebfe3b7647cc1554d2aced" + two_ruler_manifest: + path: "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/verified_evaluation_manifest.json" + bytes: 18231 + sha256: "9ee55a2522349c8520f308bc69273774dd48964847dfd340b78a7be46474cd7f" + credit_summary: + path: "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/credit_risk_controls_summary.json" + bytes: 23359 + sha256: "37c0a5c224b62ee3f47259aea4231df26ea184d3666e7fab80431ad26d611715" + credit_receipt: + path: "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/execution_receipt.json" + bytes: 826 + sha256: "179016a1c064318502cc11a7f197db0656eff8cf3f896e0e1905ddc1a9f994db" + raw_data_audit: + path: "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json" + bytes: 3654 + sha256: "ac3cda119d919006210435eb0ff2234ffd6281d46edf8d29f04157b5f64ba6a5" + label_lag_sensitivity: + path: "reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/evidence.json" + bytes: 2476 + sha256: "6078892647c0f041631ebdb47c63b5613619fd61a82c2d13d9fb8c9c84e0adc7" + solver_tie_audit: + path: "reports/crpto/ijds_policy_support_tie_evidence.json" + bytes: 12845 + sha256: "2903694ee502e6144008bbbc5891e5932f88f8527e12686363c85a28fa78684e" + endpoint_sensitivity_summary: + path: "models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/endpoint_sensitivity_summary.json" + bytes: 10692 + sha256: "f8192d9b97289ea49480b92975258801c2f9e6640acdee7606307d0368575ff6" + structural_sensitivity_config: + path: "configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml" + bytes: 5016 + sha256: "600d3332da280f4329603b4c5e72df2077711137de403b0236ab92e7285122e4" + structural_sensitivity_freeze: + path: "models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/protocol_freeze.json" + bytes: 176170 + sha256: "2fb1ac688cc702c47b26552c2af9d953ad6103eb1df17190f388dfeeb04de951" + structural_sensitivity_summary: + path: "models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/structural_sensitivity_summary.json" + bytes: 8091 + sha256: "240aef92e9eb204d6a9dd3eda388544face71118c63509476087c6baa461b853" + rolling_origin_summary: + path: "models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/binary_geometry_frontier_v4_summary.json" + bytes: 21830 + sha256: "ab1bf51d5a6f54592276ec33de4affda6ff5327e6c2c362327f08089b09ff4a8" + rolling_origin_receipt: + path: "models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/execution_receipt.json" + bytes: 1134 + sha256: "d64ef87ce6e1a2f7f0493ed17e009bf8349f68546ed137d0cafd74a6292edb0a" + missingness_summary: + path: "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/missingness_sensitivity_summary.json" + bytes: 10170 + sha256: "33d4c760746b8c860d441fc5a461fd94affbe06c0bcfa47a69aa5a7aef077ebe" + missingness_receipt: + path: "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/execution_receipt.json" + bytes: 574 + sha256: "85f1da84b2961d1fcbb2bc3c45f0e4b970d1ed011e8023e248db410230b1186d" + fit_label_completion_freeze: + path: "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/protocol_freeze.json" + bytes: 7348 + sha256: "f4b887f3a08e453e39be6b461887449e89c7882bff8b0918b9849cd29a8426db" + fit_label_completion_summary: + path: "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/fit_label_completion_summary.json" + bytes: 4644 + sha256: "8b9f7fde7f8776809350d4921b4221a9df0325f957ffbbfc94686bf1463b9c38" + allocation_granularity_freeze: + path: "models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/protocol_freeze.json" + bytes: 4677 + sha256: "cb66f9eedcd130d5d8f57cd182bcb83603df3fb2bdd8292244ec6a5f77d78c1d" + allocation_granularity_summary: + path: "models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/allocation_granularity_summary.json" + bytes: 2226 + sha256: "7e8d9609367ae2cb07e9d0d56e676d7af5486258f7f36a2680d66d2f90d4b51a" diff --git a/configs/ijds_claim_ledger.yaml b/configs/ijds_claim_ledger.yaml new file mode 100644 index 0000000..75e6331 --- /dev/null +++ b/configs/ijds_claim_ledger.yaml @@ -0,0 +1,259 @@ +schema_version: "2026-07-15.3" +status: active_ijds_claim_contract + +surfaces: + body: paper/CRPTO_ijds.qmd + supplement: paper/supplement_ijds.qmd + registry: docs/research/active_claims_2026-07-14.md + claim_matrix: paper/submission/CLAIM_AUDIT_MATRIX.md + +claims: + - id: data.exhaustive_status_independent_population + status: active + kind: empirical + rule: equals + result_pointer: /data_contract/sampling + expected: none_all_eligible_rows_within_each_declared_temporal_role + lineages: [raw_data_audit, binary_geometry] + scope: declared_36_month_temporal_design + forbidden_inference: [raw_archive_census_is_one_common_estimand, external_validity] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: endpoint.not_verified_snapshot + status: active + kind: boundary + rule: equals + result_pointer: /design/archive_is_verified_point_in_time_snapshot + expected: false + lineages: [raw_data_audit, binary_geometry.evaluation] + scope: reconstructed_status_availability_at_2020_09_30 + forbidden_inference: [verified_point_in_time_snapshot, observed_operational_chargeoff_date] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: endpoint.reason_census_partitions_candidates + status: active + kind: empirical + rule: equals + result_pointer: /evaluation_endpoint/reason_census_partitions_primary_candidates + expected: true + lineages: [binary_geometry.evaluation] + scope: primary_oot_reconstructed_endpoint_reason_taxonomy + forbidden_inference: [identified_missingness_mechanism, observed_operational_event_dates] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: coverage.five_models_all_windows_below_nominal + status: active + kind: empirical + rule: equals + result_pointer: /credit_risk_controls/all_five_all_eight_upper_below_nominal + expected: true + lineages: [binary_geometry.evaluation, credit_controls.evaluation] + scope: all_candidate_sharp_bounds_under_six_month_endpoint + forbidden_inference: [universal_conformal_invalidity, selected_model_winner, funded_set_validity] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: geometry.prevalence_sensitive_mechanism + status: active + kind: theorem + rule: documented + lineages: [binary_geometry.outcome_free, label_lag_sensitivity] + scope: constant_score_binary_threshold_and_observed_stratum_path + forbidden_inference: [finite_sample_causal_explanation, universal_threshold_transition] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: timing.fit_label_crossing_retained + status: active + kind: empirical + rule: equals + result_pointer: /binary_phase_transition/label_lag_sensitivity/w7_to_w8_threshold_crossing_at_all_admissible_lags + expected: true + lineages: [label_lag_sensitivity] + scope: lags_passing_locked_monthly_retention_rule + forbidden_inference: [arbitrary_label_lag_invariance, causal_attribution] + surfaces: + required: [body, supplement, registry] + allowed: [body, supplement, registry, claim_matrix] + + - id: timing.endpoint_six_month_reconciles_active + status: active + kind: empirical + rule: equals + result_pointer: /sensitivity/evaluation_endpoint_availability/six_month_endpoint_reconciles_to_active_evaluation + expected: true + lineages: [endpoint_availability, binary_geometry.evaluation, two_ruler.evaluation] + scope: complete_endpoint_availability_lag_grid + forbidden_inference: [endpoint_selection, independent_replications] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: sensitivity.second_origin_coverage_recurrence + status: active + kind: empirical + rule: equals + result_pointer: /sensitivity/rolling_origin/all_sixteen_upper_below_nominal + expected: true + lineages: [binary_geometry.evaluation, rolling_origin] + scope: two_retrospective_origins_eight_windows_each + forbidden_inference: [independent_replication, temporal_invariance, prospective_validation] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: sensitivity.missingness_encoding_recurrence + status: active + kind: empirical + rule: equals + result_pointer: /sensitivity/missingness_encoding/all_three_all_eight_upper_below_nominal + expected: true + lineages: [binary_geometry.evaluation, missingness_encoding] + scope: three_feature_semantics_preserving_catboost_encodings + forbidden_inference: [identified_missingness_mechanism, encoding_winner, portfolio_robustness] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: sensitivity.fit_label_completion_coverage_recurrence + status: active + kind: empirical + rule: equals + result_pointer: /sensitivity/fit_label_completion/all_scenarios_all_windows_upper_below_nominal + expected: true + lineages: [binary_geometry.evaluation, fit_label_completion] + scope: observed_only_plus_three_declared_fit_label_completion_rules + forbidden_inference: [sharp_completion_bounds, arbitrary_label_assignment_robustness, preregistration] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: geometry.fit_label_completion_crossing_not_universal + status: active + kind: boundary + rule: equals + result_pointer: /sensitivity/fit_label_completion/w7_w8_crossing_in_all_scenarios + expected: false + lineages: [binary_geometry.outcome_free, fit_label_completion] + scope: stratum_2_w7_w8_path_over_four_fit_label_scenarios + forbidden_inference: [universal_threshold_transition, causal_explanation, geometry_invariance] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: timing.fit_and_endpoint_lags_not_factorial + status: active + kind: boundary + rule: equals + result_pointer: /sensitivity/evaluation_endpoint_availability/fit_label_lag_crossed_factorially + expected: false + lineages: [label_lag_sensitivity, endpoint_availability] + scope: two_separate_one_factor_sensitivities + forbidden_inference: [joint_lag_robustness, full_factorial_robustness] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: decision.no_selected_policy + status: active + kind: boundary + rule: equals + result_pointer: /decision_challenger/interpretation/policy_winner + expected: null + lineages: [two_ruler.outcome_free, two_ruler.evaluation, binary_geometry.evaluation] + scope: all_reported_rulers_coordinates_windows_and_supports + forbidden_inference: [policy_winner, preferred_ruler, preferred_coordinate, universal_direction] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: comparator.broad_support_all_cross_zero + status: active + kind: empirical + rule: equals + result_pointer: /portfolio/broad_stress_all_envelopes_cross_zero + expected: true + lineages: [binary_geometry.outcome_free, binary_geometry.evaluation] + scope: declared_point_cap_support_0_05_to_0_12 + forbidden_inference: [universal_comparator_quantification, policy_equivalence] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: theory.sharp_common_outcome_bounds + status: active + kind: theorem + rule: documented + lineages: [binary_geometry.evaluation, two_ruler.evaluation] + scope: binary_completion_bounds_for_candidates_and_paired_allocations + forbidden_inference: [sampling_confidence_interval, missing_at_random_identification] + surfaces: + required: [body, supplement, registry] + allowed: [body, supplement, registry, claim_matrix] + + - id: theory.binary_identification_width + status: active + kind: theorem + rule: documented + lineages: [binary_geometry.evaluation, two_ruler.evaluation] + scope: unrestricted_binary_completion_for_common_outcome_policy_contrasts + forbidden_inference: [missing_at_random_identification, sampling_uncertainty, causal_effect] + surfaces: + required: [body, supplement, registry] + allowed: [body, supplement, registry, claim_matrix] + + - id: theory.basis_endpoint_sufficiency + status: active + kind: theorem + rule: documented + lineages: [binary_geometry.outcome_free, binary_geometry.evaluation] + scope: declared_piecewise_linear_point_cap_support + forbidden_inference: [continuous_joint_frontier_uniqueness, arbitrary_comparator_support] + surfaces: + required: [body, supplement, registry] + allowed: [body, supplement, registry, claim_matrix] + + - id: boundary.no_selected_set_validity + status: active + kind: boundary + rule: documented + lineages: [binary_geometry, two_ruler] + scope: candidate_conformal_audit_followed_by_outcome_free_allocation + forbidden_inference: [selected_set_conformal_guarantee, post_selection_coverage] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: sensitivity.structure_no_universal_direction + status: active + kind: empirical + rule: equals + result_pointer: /sensitivity/portfolio_structure/every_scenario_has_adverse_default_and_miscoverage_cells + expected: true + lineages: [portfolio_structure, two_ruler.evaluation] + scope: complete_budget_by_purpose_cap_by_lgd_cartesian_grid + forbidden_inference: [scenario_winner, universal_adversity, structural_invariance, deployment] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] + + - id: optimization.allocation_granularity_is_diagnostic + status: active + kind: boundary + rule: equals + result_pointer: /sensitivity/allocation_granularity/integer_policy_or_reoptimization_claim_authorized + expected: false + lineages: [two_ruler.outcome_free, allocation_granularity] + scope: deterministic_usd25_floor_with_residual_cash_on_active_allocations + forbidden_inference: [integer_optimality, discrete_policy_robustness, reoptimization] + surfaces: + required: [body, supplement, registry, claim_matrix] + allowed: [body, supplement, registry, claim_matrix] diff --git a/configs/pipeline_registry/script_role_registry.yaml b/configs/pipeline_registry/script_role_registry.yaml index b9cef48..bd262af 100644 --- a/configs/pipeline_registry/script_role_registry.yaml +++ b/configs/pipeline_registry/script_role_registry.yaml @@ -1,5 +1,29 @@ -schema_version: "2026-05-10.1" +schema_version: "2026-07-15.3" scripts: + - + script: scripts/experiments/run_ijds_endpoint_availability_sensitivity.py + role: complete frozen endpoint-availability sensitivity + category: diagnostics + pipeline_owner: ijds_active_audit + status: active + - + script: scripts/experiments/run_ijds_portfolio_structure_sensitivity.py + role: complete nonselective budget-purpose-LGD structural sensitivity V6 + category: diagnostics + pipeline_owner: ijds_active_audit + status: active + - + script: scripts/experiments/run_ijds_policy_support_tie_audit.py + role: evaluated-cap basis and ID-order stability audit + category: diagnostics + pipeline_owner: ijds_active_audit + status: active + - + script: scripts/render_submission_pdf_previews.py + role: paper and supplement PDF preview renderer + category: publication + pipeline_owner: ijds_active_audit + status: active - script: scripts/run_crpto_pipeline.py role: "fixed CRPTO end-to-end orchestrator" @@ -108,6 +132,84 @@ scripts: category: publication pipeline_owner: publication_exports status: public + - + script: scripts/experiments/run_ijds_fixed_taxonomy_c2.py + role: historical fixed-taxonomy V1--V3 IJDS experiment + category: provenance + pipeline_owner: ijds_historical_audit + status: historical + - + script: scripts/build_ijds_fixed_taxonomy_c2_evidence.py + role: historical fixed-taxonomy V1--V3 evidence builder + category: provenance + pipeline_owner: ijds_historical_audit + status: historical + - + script: scripts/experiments/run_ijds_binary_geometry_frontier_v4.py + role: two-phase outcome-free freeze and endpoint-corrected V4 evaluation + category: evaluation + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/build_ijds_binary_geometry_frontier_v4_evidence.py + role: sole active endpoint-corrected IJDS evidence builder + category: publication + pipeline_owner: ijds_active_audit + status: public + - + script: scripts/experiments/run_ijds_normalized_objective_frontier.py + role: outcome-free two-ruler frontier freeze + category: evaluation + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/experiments/run_ijds_normalized_objective_frontier_v2.py + role: hash-verified two-ruler endpoint evaluation + category: evaluation + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/experiments/run_ijds_credit_risk_controls.py + role: two-phase five-model coverage-control audit + category: evaluation + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/experiments/run_ijds_raw_data_audit.py + role: full-archive data and endpoint contract audit + category: diagnostics + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/experiments/run_ijds_label_lag_sensitivity.py + role: frozen-score label-availability sensitivity + category: diagnostics + pipeline_owner: ijds_active_audit + status: paper + - + script: scripts/build_ijds_submission_tex.py + role: deterministic QMD-to-INFORMS TeX generator + category: publication + pipeline_owner: ijds_active_audit + status: public + - + script: scripts/compile_ijds_submission.py + role: official IJDS LaTeX compiler and citation-reference gate + category: publication + pipeline_owner: ijds_active_audit + status: public + - + script: scripts/check_publication_integrity.py + role: reviewer-surface claim, anonymity, and evidence integrity gate + category: governance + pipeline_owner: ijds_active_audit + status: public + - + script: scripts/run_ty_advisory.py + role: focused and full-tree ty type-advisory runner + category: governance + pipeline_owner: ijds_active_audit + status: public - script: scripts/build_crpto_journal_package.py role: journal package builder diff --git a/configs/pipelines/crpto_e2e.yaml b/configs/pipelines/crpto_e2e.yaml deleted file mode 100644 index 2730ee4..0000000 --- a/configs/pipelines/crpto_e2e.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: crpto_e2e -description: Standalone CRPTO end-to-end pipeline. -writes_canonical_artifacts: false -promotion_state: paper_grade -include_rapids: false -include_notebooks: false -freeze_if_available: true -explicit_champion_only: true -steps: - - preflight - - core_data_pd - - core_conformal - - core_portfolio - - diagnostics_governance - - publication_exports diff --git a/configs/profiles/crpto_default.yaml b/configs/profiles/crpto_default.yaml deleted file mode 100644 index e94bb91..0000000 --- a/configs/profiles/crpto_default.yaml +++ /dev/null @@ -1,14 +0,0 @@ -profile: crpto_default -writes_canonical_artifacts: false -promotion_state: paper_grade -include_rapids: false -include_notebooks: false -freeze_if_available: true -explicit_champion_only: true -champion: - run_tag: ijds-rebaseline-2026-06-07 - policy: bound_aware_276k_economic_champion -paths: - tables: reports/crpto/tables - figures: reports/crpto/figures - book: book diff --git a/configs/profiles/regret_auditability_sandbox.yaml b/configs/profiles/regret_auditability_sandbox.yaml deleted file mode 100644 index ae2c39a..0000000 --- a/configs/profiles/regret_auditability_sandbox.yaml +++ /dev/null @@ -1,81 +0,0 @@ -schema_version: "2026-05-12.1" -profile_name: regret_auditability_sandbox -description: Orchestrator profile for the isolated CRPTO regret-auditability sandbox. -writes_canonical_artifacts: false -orchestrator: scripts/search/run_regret_auditability_sandbox.py -artifact_root_template: "D:/crpto_experiments/regret_auditability/{run_tag}" -profiles: - pd: configs/profiles/search_pd_extensive.yaml - conformal: configs/profiles/search_conformal_extensive.yaml - portfolio: configs/profiles/search_portfolio_extensive.yaml -resource_policy: - cpu: "AMD Ryzen 9 5900X, 12 cores / 24 threads" - ram_gb: 64 - no_gpu: true - heartbeat_seconds: 60 - reserve_logical_cpus: 2 - min_available_ram_gb: 10 -defaults: - pd_hpo_workers: 4 - catboost_threads_per_worker: 5 - pd_broad_top_lanes_from_smoke: 6 - pd_refine_top_lanes_from_broad: 3 - conformal_workers: 6 - portfolio_workers: 4 - highs_threads_per_worker: 4 -adaptive_budget: - pd_smoke_trials_per_lane: 12 - pd_broad_trials_per_lane: 300 - pd_refine_trials_per_lane: 300 - portfolio_proxy_candidates: 100000 - portfolio_shortlist_top_k: 1000 - portfolio_random_states: [42, 52, 62] - portfolio_risk_step: 0.01 - portfolio_gamma_step: 0.10 - portfolio_aversion_grid: [0, 0.02, 0.05, 0.10, 0.25, 0.50] - portfolio_cap_tail_quantiles: [0.60, 0.75, 0.90, 1.0] -phase_handoff: - pd_smoke_to_broad: "rank all feature-profile x monotonic-policy lanes on OOT AUC, then Brier/ECE" - pd_broad_to_refine: "rerun only top lanes and initialize local-refine from each lane's broad best params" - pd_refine_to_conformal: "copy best refined model/calibrator under artifact_root/pd/best/models" - conformal_to_portfolio: "portfolio consumes only the sandbox conformal finalist artifact" -runtime_notes: - gpu_probe: "CatBoost GPU smoke passes, but CatBoost monotone_constraints fail on GPU; PD remains CPU to preserve monotonic method family." - pd_failure_policy: "A failed PD lane is logged and skipped; the phase continues as long as at least one candidate produces a training record." - per_lane_logs: "Each command writes logs under artifact_root/logs//." -guardrails: - fail_on_outputs: - - EXTRACTION_MANIFEST.json - - models/pd_canonical.cbm - - models/pd_canonical_calibrator.pkl - - models/final_project_promotion.json - - models/conformal_policy_status.json - - data/processed/conformal_intervals_mondrian.parquet - - data/processed/portfolio_bound_aware/** -metrics: - regret_formula: "oracle_realized_return_same_budget_concentration_and_expost_default_cap - policy_realized_return" - auditability_score: - max_score: 100 - weights: - coverage90_pass: 15 - coverage95_pass: 10 - min_group_coverage_pass: 15 - no_critical_alerts: 10 - alpha01_exact_pass: 15 - violation_zero: 10 - v_within_sqrt_alpha: 10 - monotonic_audit_pass: 10 - reproducible_resume_manifest: 5 -final_table_columns: - - regret - - robust_return - - price_of_robustness - - coverage - - min_group_coverage - - width - - winkler - - V - - Gamma_CP - - violation - - funded_count - - auditability_score diff --git a/configs/profiles/search_conformal_claim_max.yaml b/configs/profiles/search_conformal_claim_max.yaml deleted file mode 100644 index 4053c09..0000000 --- a/configs/profiles/search_conformal_claim_max.yaml +++ /dev/null @@ -1,68 +0,0 @@ -schema_version: "2026-06-21.1" -profile_name: search_conformal_claim_max -description: > - Claim-max conformal search for champion reopening. Optimizes coverage, - interval width, temporal/group robustness, and downstream portfolio usefulness - without writing canonical artifacts. -writes_canonical_artifacts: false -search_space: - phase1: - parallel_workers: 3 - alpha_candidates_90: [0.05, 0.075, 0.085, 0.09, 0.095, 0.10, 0.105, 0.11, 0.125, 0.15, 0.20] - alpha_candidates_95: [0.025, 0.035, 0.04, 0.045, 0.05, 0.055, 0.06, 0.075] - partition_candidates: - - grade - - score_decile_mondrian - - grade_x_scoreband_mondrian - partition_probability_sources: [calibrated, raw] - n_score_bins_candidates: [5, 10, 15, 20, 30] - min_group_sizes: [50, 100, 150, 250, 500, 1000, 2000] - fallback_modes: [grade_then_global, global_only] - score_scale_families: - - none - - bernoulli_sqrt - - bernoulli_sqrt_clipped_0.02 - - bernoulli_sqrt_clipped_0.05 - calibration_fractions: [0.50, 0.75, 1.00] - tuning_holdout_ratios: [0.20, 0.30] - inner_random_states: [42, 52, 62] - phase2: - enabled: true - always_evaluate: true - top_k_designs: 5 - calibrators: - - venn_abers - - beta - - isotonic - - platt - - temperature - - quadratic_logit - max_metric_degradation: - ece: 0.0015 - brier_score: 0.0008 - log_loss: 0.0025 -sidecar: - methods: [lac, margin] - partitions: - - global - - grade - - score_decile_mondrian - - grade_x_scoreband_mondrian - partition_probability_sources: [calibrated] - n_score_bins: 10 - min_group_size: 100 - fallback_mode: grade_then_global - calibration_size_fractions: [0.50, 0.75, 1.00] -validation: - top_k_inner: 5 - acceptance: - warning_alerts_max: 4 - total_alerts_max: 5 - coverage_deviation_90_max: 0.025 - min_group_coverage_90_min: 0.88 - avg_width_90_max: 0.80 -claim_targets: - same_region_higher_return: downstream portfolio may prefer this if width/bound stays stable - same_region_better_bound: prioritize lower width, Winkler, Gamma_CP, and V - wider_zero_violation_region: preserve group/temporal coverage while broadening feasible policies - clean_paper_region: prefer grade/score/grade-score partitions over opaque generated groups diff --git a/configs/profiles/search_conformal_extensive.yaml b/configs/profiles/search_conformal_extensive.yaml deleted file mode 100644 index c82fcb2..0000000 --- a/configs/profiles/search_conformal_extensive.yaml +++ /dev/null @@ -1,36 +0,0 @@ -schema_version: "2026-05-12.1" -profile_name: search_conformal_extensive -description: Venn-Abers plus Mondrian conformal grid for the sandbox frontier. -writes_canonical_artifacts: false -artifact_root_template: "D:/crpto_experiments/regret_auditability/{run_tag}" -runtime: - workers: 6 - reserve_logical_cpus: 2 - min_available_ram_gb: 10 -calibration: - allowed_methods: [venn_abers] -selection_protocol: - select_on: "2018 OOT" - confirm_on: - - "2019-2020" - - "stress_slices" -mondrian_families: - - grade - - score_decile_mondrian - - grade_x_scoreband_mondrian -grid: - alpha_90: [0.05, 0.075, 0.09, 0.095, 0.10, 0.105, 0.11, 0.125, 0.15, 0.20] - alpha_95: [0.025, 0.04, 0.045, 0.05, 0.055, 0.06, 0.075] - score_bins: [5, 10, 15, 20, 30] - min_group_sizes: [100, 150, 250, 500, 1000, 2000] - score_scales: - - none - - bernoulli_sqrt - - bernoulli_sqrt_clipped_0.02 - - bernoulli_sqrt_clipped_0.05 -metrics: - - coverage90 - - coverage95 - - min_group_coverage - - width - - winkler diff --git a/configs/profiles/search_pd_extensive.yaml b/configs/profiles/search_pd_extensive.yaml deleted file mode 100644 index b8fefb9..0000000 --- a/configs/profiles/search_pd_extensive.yaml +++ /dev/null @@ -1,129 +0,0 @@ -schema_version: "2026-05-12.1" -profile_name: search_pd_extensive -description: CPU-only CatBoost monotonic Optuna search for regret-auditability sandbox. -writes_canonical_artifacts: false -artifact_root_template: "D:/crpto_experiments/regret_auditability/{run_tag}" -runtime: - no_gpu: true - workers: 4 - catboost_threads_per_worker: 5 - reserve_logical_cpus: 2 - min_available_ram_gb: 10 - notes: - - CatBoost GPU is available on the machine, but monotone_constraints are not usable in the GPU path for this sandbox. - - CPU parallelism is increased to 4 workers x 5 threads with RAM gating. -model: - base_config: configs/crpto_pd_model.yaml - feature_config_source: data/processed/feature_config.yml - feature_profiles: - core_stable: - groups: [CATBOOST_FEATURES] - stable_core_enabled: true - core_wide: - groups: [CATBOOST_FEATURES] - stable_core_enabled: false - core_woe: - groups: [CATBOOST_FEATURES, WOE_FEATURES] - stable_core_enabled: false - bureau_high: - groups: [CATBOOST_FEATURES, HIGH_COVERAGE_BUREAU_FEATURES] - stable_core_enabled: false - full_challenger: - groups: [CHALLENGER_FEATURE_POOL_V2] - stable_core_enabled: false - full_challenger_woe: - groups: [CHALLENGER_FEATURE_POOL_V2, WOE_FEATURES] - stable_core_enabled: false - force_params: - task_type: CPU - allow_writing_files: false -monotonic_policies: - canonical_4: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - affordability_rate_5: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - int_rate: 1 - credit_history_7: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - int_rate: 1 - delinq_severity: 1 - delinq_recency: -1 - bureau_utilization_11: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - int_rate: 1 - rev_utilization: 1 - high_util_pct: 1 - bc_util: 1 - percent_bc_gt_75: 1 - fico_score: -1 - credit_age_years: -1 - bureau_behavior_15: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - int_rate: 1 - delinq_severity: 1 - delinq_recency: -1 - pub_rec: 1 - has_bankruptcy: 1 - num_accts_ever_120_pd: 1 - num_tl_90g_dpd_24m: 1 - num_tl_30dpd: 1 - pct_tl_nvr_dlq: -1 - fico_score: -1 - credit_age_years: -1 - inquiry_velocity_12: - installment: 1 - annual_inc: -1 - dti: 1 - loan_to_income: 1 - int_rate: 1 - inq_last_6mths: 1 - inq_last_12m: 1 - inq_fi: 1 - mths_since_recent_inq: -1 - acc_open_past_24mths: 1 - num_tl_op_past_12m: 1 - fico_score: -1 -optuna: - storage_family: rdb - sampler: tpe - multivariate_tpe: true - group_tpe: true - constant_liar: true - constraints_policy: - max_brier_delta: 0.0025 - max_ece_delta: 0.0025 - min_auc_delta: -0.001 - seed_replay: [42, 52, 62, 72, 82] -phases: - smoke: - trials_per_lane: 12 - feature_profiles: all - monotonic_policies: all - broad: - trials_per_lane: 300 - select_top_lanes_from_smoke: 6 - local_refine: - trials_per_lane: 300 - select_top_lanes_from_broad: 3 - initialize_from_previous_best_params: true - refine_from_top_k: 30 -operational_hardening: - continue_after_failed_pd_lane: true - per_lane_logs: true - optuna_search_space_version: cb_space_v3_monotone_symmetric - forced_monotone_grow_policy: SymmetricTree diff --git a/configs/profiles/search_portfolio_claim_max.yaml b/configs/profiles/search_portfolio_claim_max.yaml deleted file mode 100644 index d337a6f..0000000 --- a/configs/profiles/search_portfolio_claim_max.yaml +++ /dev/null @@ -1,58 +0,0 @@ -schema_version: "2026-06-21.1" -profile_name: search_portfolio_claim_max -description: > - Bound-aware robust portfolio search for CRPTO champion reopening. The search - keeps exact alpha checks, region summaries, and return-bound frontier artifacts. -writes_canonical_artifacts: false -runtime: - workers: 1 - highs_threads_per_worker: 4 - reserve_logical_cpus: 2 - min_available_ram_gb: 10 -candidate_policy_families: - - blended_uncertainty - - capped_blended_uncertainty - - tail_blended_uncertainty - - segment_tail_blended_uncertainty - - segment_relative_tail_blended_uncertainty -frontier: - proxy_candidates_per_conformal_finalist: 100000 - exact_rerank_top_k: 1200 - bucket_return_k: 240 - bucket_proxy_k: 240 - bucket_family_k: 120 - bucket_region_k: 240 -grids: - risk_grid: "0.08,0.09,0.10,0.11,0.12,0.13,0.14,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.19,0.20,0.21,0.22,0.23,0.24,0.25" - gamma_grid: "0,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60,0.65,0.70,0.80,0.90,1.0" - aversion_grid: "0,0.01,0.02,0.05,0.10,0.25,0.50,0.75" - delta_cap_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - tail_focus_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - alpha_grid: "0.005,0.01,0.02,0.03,0.05,0.075,0.10,0.15,0.20" - random_states: "42,52,62,72,82" -incumbent_region: - risk_neighbors: "0.145,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.185" - gamma_neighbors: "0.375,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60" - policy_modes: "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty" -selection_objectives: - - alpha01_exact_pass - - alpha_exact_pass_count - - realized_total_return - - alpha01_weighted_miscoverage_V - - alpha01_gamma_cp - - price_of_robustness - - robust_region_pass_rate -claim_targets: - same_region_higher_return: - metric: realized_total_return - gate: alpha01_exact_pass - same_region_better_bound: - metrics: [alpha01_gamma_cp, alpha01_weighted_miscoverage_V] - direction: lower_is_better - wider_zero_violation_region: - metrics: [n_alpha01_passers, alpha01_pass_rate, n_all_alpha_passers] - direction: higher_is_better - clean_paper_region: - preferred_buckets: [forced_incumbent_neighbors, incumbent_region] - return_bound_frontier: - artifact: portfolio_bound_aware_shortlist_exact.parquet diff --git a/configs/profiles/search_portfolio_cuopt_frontier_26_06.yaml b/configs/profiles/search_portfolio_cuopt_frontier_26_06.yaml deleted file mode 100644 index 195d5d0..0000000 --- a/configs/profiles/search_portfolio_cuopt_frontier_26_06.yaml +++ /dev/null @@ -1,61 +0,0 @@ -schema_version: "2026-06-22.1" -profile_name: search_portfolio_cuopt_frontier_26_06 -description: > - GPU frontier-only bound-aware portfolio search for CRPTO champion reopening. - This profile uses cuOpt 26.06-style LP settings to explore a broad - return-bound frontier, but defers exact bound reranking to the HiGHS - evidence lane before any paper-facing promotion. -writes_canonical_artifacts: false -runtime: - workers: 1 - reserve_logical_cpus: 8 - min_available_ram_gb: 10 -execution: - solver_backend: cuopt - exact_solver_backend: highs - frontier_only: true -candidate_policy_families: - - blended_uncertainty - - capped_blended_uncertainty - - tail_blended_uncertainty - - segment_tail_blended_uncertainty - - segment_relative_tail_blended_uncertainty -frontier: - proxy_candidates_per_conformal_finalist: 100000 - exact_rerank_top_k: 1200 - bucket_return_k: 240 - bucket_proxy_k: 240 - bucket_family_k: 120 - bucket_region_k: 240 -grids: - risk_grid: "0.08,0.09,0.10,0.11,0.12,0.13,0.14,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.19,0.20,0.21,0.22,0.23,0.24,0.25" - gamma_grid: "0,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60,0.65,0.70,0.80,0.90,1.0" - aversion_grid: "0,0.01,0.02,0.05,0.10,0.25,0.50,0.75" - delta_cap_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - tail_focus_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - alpha_grid: "0.005,0.01,0.02,0.03,0.05,0.075,0.10,0.15,0.20" - random_states: "42,52,62,72,82" -incumbent_region: - risk_neighbors: "0.145,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.185" - gamma_neighbors: "0.375,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60" - policy_modes: "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty" -cuopt: - method: Concurrent - presolve: 2 - pdlp_precision: mixed - pdlp_solver_mode: Stable3 - crossover: false - first_primal_feasible: false - infeasibility_detection: false - dual_postsolve: false - num_cpu_threads: 2 - log_to_console: false - log_dir: auto -selection_objectives: - - frontier_return - - frontier_bound_proxy - - incumbent_region_coverage - - exact_context_for_higs_rerank -claim_targets: - exploration_only: true - promotion_gate: "HiGHS exact rerank over frozen cuOpt frontier shortlist" diff --git a/configs/profiles/search_portfolio_cuopt_pdlp_stable3_26_06.yaml b/configs/profiles/search_portfolio_cuopt_pdlp_stable3_26_06.yaml deleted file mode 100644 index 8d03ef2..0000000 --- a/configs/profiles/search_portfolio_cuopt_pdlp_stable3_26_06.yaml +++ /dev/null @@ -1,64 +0,0 @@ -schema_version: "2026-06-24.1" -profile_name: search_portfolio_cuopt_pdlp_stable3_26_06 -description: > - GPU frontier-only bound-aware portfolio search for CRPTO champion reopening. - This profile keeps the broad paper-facing frontier from the 26.06 cuOpt - profile but uses PDLP Stable3 instead of Concurrent, after Concurrent hit a - cuDSS threading-layer abort on the pool93 frontier. HiGHS exact reranking - remains the promotion lane. -writes_canonical_artifacts: false -runtime: - workers: 1 - reserve_logical_cpus: 8 - min_available_ram_gb: 10 -execution: - solver_backend: cuopt - exact_solver_backend: highs - frontier_only: true -candidate_policy_families: - - blended_uncertainty - - capped_blended_uncertainty - - tail_blended_uncertainty - - segment_tail_blended_uncertainty - - segment_relative_tail_blended_uncertainty -frontier: - proxy_candidates_per_conformal_finalist: 100000 - exact_rerank_top_k: 1200 - bucket_return_k: 240 - bucket_proxy_k: 240 - bucket_family_k: 120 - bucket_region_k: 240 -grids: - risk_grid: "0.08,0.09,0.10,0.11,0.12,0.13,0.14,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.19,0.20,0.21,0.22,0.23,0.24,0.25" - gamma_grid: "0,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60,0.65,0.70,0.80,0.90,1.0" - aversion_grid: "0,0.01,0.02,0.05,0.10,0.25,0.50,0.75" - delta_cap_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - tail_focus_grid: "0.50,0.60,0.75,0.85,0.90,0.95,1.0" - alpha_grid: "0.005,0.01,0.02,0.03,0.05,0.075,0.10,0.15,0.20" - random_states: "42,52,62,72,82" -incumbent_region: - risk_neighbors: "0.145,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.185" - gamma_neighbors: "0.375,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60" - policy_modes: "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty" -cuopt: - method: PDLP - presolve: 2 - pdlp_precision: mixed - pdlp_solver_mode: Stable3 - crossover: false - first_primal_feasible: false - save_best_primal_solution: true - infeasibility_detection: false - dual_postsolve: false - num_cpu_threads: 2 - num_gpus: 1 - log_to_console: false - log_dir: auto -selection_objectives: - - frontier_return - - frontier_bound_proxy - - incumbent_region_coverage - - exact_context_for_higs_rerank -claim_targets: - exploration_only: true - promotion_gate: "HiGHS exact rerank over frozen cuOpt frontier shortlist" diff --git a/configs/profiles/search_portfolio_extensive.yaml b/configs/profiles/search_portfolio_extensive.yaml deleted file mode 100644 index 4cd7d25..0000000 --- a/configs/profiles/search_portfolio_extensive.yaml +++ /dev/null @@ -1,39 +0,0 @@ -schema_version: "2026-05-12.1" -profile_name: search_portfolio_extensive -description: Bound-aware robust portfolio frontier search for sandbox finalists. -writes_canonical_artifacts: false -artifact_root_template: "D:/crpto_experiments/regret_auditability/{run_tag}" -runtime: - workers: 4 - highs_threads_per_worker: 4 - reserve_logical_cpus: 2 - min_available_ram_gb: 10 -candidate_policy_families: - - blended_uncertainty - - capped_blended_uncertainty - - tail_blended_uncertainty - - segment_tail_blended_uncertainty - - segment_relative_tail_blended_uncertainty -frontier: - proxy_candidates_per_conformal_finalist: 100000 - exact_rerank_top_k: 1000 -grids: - risk_tolerance: - start: 0.08 - stop: 0.25 - step: 0.01 - gamma: - start: 0.0 - stop: 1.0 - step: 0.10 - aversion: [0, 0.02, 0.05, 0.10, 0.25, 0.50] - cap_tail_quantiles: [0.60, 0.75, 0.90, 1.0] - alpha_exact: [0.01, 0.02, 0.03, 0.05, 0.10, 0.15, 0.20] -metrics: - - regret - - robust_return - - price_of_robustness - - V - - Gamma_CP - - violation - - funded_count diff --git a/configs/profiles/search_portfolio_pool93_stage1_claim_26_06.yaml b/configs/profiles/search_portfolio_pool93_stage1_claim_26_06.yaml deleted file mode 100644 index a23e317..0000000 --- a/configs/profiles/search_portfolio_pool93_stage1_claim_26_06.yaml +++ /dev/null @@ -1,70 +0,0 @@ -schema_version: "2026-06-25.1" -profile_name: search_portfolio_pool93_stage1_claim_26_06 -description: > - Stage-1 paper-facing portfolio search for the pool93 champion candidate. - cuOpt 26.06 PDLP Fast1 is used only as a broad proxy-frontier discovery - engine; HiGHS exact reranking validates the frozen shortlist on the full - universe with five evaluation seeds before any paper-facing claim. -writes_canonical_artifacts: false -runtime: - workers: 1 - reserve_logical_cpus: 8 - min_available_ram_gb: 10 -execution: - solver_backend: cuopt - exact_solver_backend: highs - frontier_only: true -candidate_policy_families: - - blended_uncertainty - - capped_blended_uncertainty - - tail_blended_uncertainty - - segment_tail_blended_uncertainty - - segment_relative_tail_blended_uncertainty -frontier: - proxy_candidates_per_conformal_finalist: 100000 - exact_max_candidates: 0 - exact_random_states: "42,52,62,72,82" - exact_checkpoint_every: 25 - exact_threads: 8 - exact_rerank_top_k: 420 - bucket_return_k: 80 - bucket_proxy_k: 80 - bucket_family_k: 24 - bucket_region_k: 120 -grids: - risk_grid: "0.08,0.10,0.12,0.14,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.19,0.20,0.22,0.24,0.25" - gamma_grid: "0,0.10,0.20,0.30,0.40,0.425,0.45,0.475,0.50,0.55,0.60,0.70,0.80,1.0" - aversion_grid: "0,0.02,0.05,0.10,0.25" - delta_cap_grid: "0.50,0.75,0.90,1.0" - tail_focus_grid: "0.50,0.75,0.90,1.0" - alpha_grid: "0.01,0.03,0.05,0.07,0.10,0.12,0.15,0.20" - random_states: "42" - budget_profiles: "free" -incumbent_region: - risk_neighbors: "0.145,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.185" - gamma_neighbors: "0.40,0.425,0.45,0.475,0.50,0.55,0.60" - policy_modes: "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty" -cuopt: - method: PDLP - presolve: 2 - pdlp_precision: mixed - pdlp_solver_mode: Fast1 - crossover: false - first_primal_feasible: false - save_best_primal_solution: true - infeasibility_detection: false - dual_postsolve: false - num_cpu_threads: 2 - num_gpus: 1 - log_to_console: false - log_dir: auto -selection_objectives: - - same_robust_region_higher_return - - same_robust_region_lower_c_cp_or_bound - - wider_region_zero_violations - - cleaner_interpretable_region - - return_bound_frontier -claim_targets: - exploration_only: true - pool93_mandatory_downstream: true - promotion_gate: "Full-universe HiGHS exact rerank over frozen cuOpt proxy frontier shortlist" diff --git a/crpto/__init__.py b/crpto/__init__.py deleted file mode 100644 index 7b1623c..0000000 --- a/crpto/__init__.py +++ /dev/null @@ -1,90 +0,0 @@ -"""CRPTO — Conformal Robust Predict-Then-Optimize. - -The package is named after the paper's core thesis: combine conformal -prediction with robust portfolio optimization. CRPTO is applied to credit -risk (Lending Club data) but the acronym itself refers to the -methodological pair, not to the domain. - -Public API for the standalone Paper_CRPTO research pipeline. This top-level -``crpto`` package re-exports the most useful entry points from the internal -``src.*`` layout so external notebooks, the Quarto book and downstream -scripts can write:: - - from crpto import PolicyMode, load_pipeline_state, make_study - -instead of dipping into ``src.optimization.policy`` etc. - -The underlying modules under ``src/`` remain the source of truth. This -package is a thin re-export layer — adding new public symbols here is the -recommended way to surface them to API consumers without leaking the -``src.`` prefix. -""" - -from __future__ import annotations - -# Feature config IO -from src.features.feature_config_io import ( - load_feature_config, - pickle_to_yaml, - save_feature_config, -) - -# Conformal diagnostics (cheap, no model state) -from src.models.conformal_diagnostics import ( - summarize_prediction_sets, - validate_coverage, -) - -# Calibration -from src.models.venn_abers import VennAbersScoreCalibrator - -# Optimization -from src.optimization.policy import ( - PolicyMode, - all_policy_modes, - resolve_policy_mode, -) - -# MLflow helpers -from src.utils.mlflow_tracing import ( - PAPER_RUN_TAG, - paper_run, - register_parquet_dataset, - set_paper_tags, - trace, -) - -# Optuna helpers -from src.utils.optuna_storage import make_storage, make_study - -# Pipeline state aggregator -from src.utils.pipeline_state import ( - PipelineState, - load_pipeline_state, - write_pipeline_state, -) - -__version__ = "0.1.0" - -__all__ = [ - "PAPER_RUN_TAG", - "PipelineState", - "PolicyMode", - "VennAbersScoreCalibrator", - "__version__", - "all_policy_modes", - "load_feature_config", - "load_pipeline_state", - "make_storage", - "make_study", - "paper_run", - "pickle_to_yaml", - "register_parquet_dataset", - "resolve_policy_mode", - "save_feature_config", - "set_paper_tags", - "summarize_prediction_sets", - "trace", - "validate_coverage", - "write_pipeline_state", -] diff --git a/data/processed/ab_attribution_by_grade.parquet.dvc b/data/processed/ab_attribution_by_grade.parquet.dvc deleted file mode 100644 index ccfa09d..0000000 --- a/data/processed/ab_attribution_by_grade.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 93c32e9e7e5a387002f0af9fed5a4425 - size: 6083 - hash: md5 - path: ab_attribution_by_grade.parquet diff --git a/data/processed/ab_simulation_summary.parquet.dvc b/data/processed/ab_simulation_summary.parquet.dvc deleted file mode 100644 index cb733ef..0000000 --- a/data/processed/ab_simulation_summary.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: a9f960a5d1da6251be0fdc86b4df0769 - size: 3046 - hash: md5 - path: ab_simulation_summary.parquet diff --git a/data/processed/bma_comparison.parquet.dvc b/data/processed/bma_comparison.parquet.dvc deleted file mode 100644 index 1f198ff..0000000 --- a/data/processed/bma_comparison.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: d425d8afdb5f418a7f01086a837e2805 - size: 6719261 - hash: md5 - path: bma_comparison.parquet diff --git a/data/processed/cif_ecl_impact.parquet.dvc b/data/processed/cif_ecl_impact.parquet.dvc deleted file mode 100644 index 088e6ae..0000000 --- a/data/processed/cif_ecl_impact.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 35eb3294a5cc8bb93437efa441335cfe - size: 11546 - hash: md5 - path: cif_ecl_impact.parquet diff --git a/data/processed/competing_risks_cif.parquet.dvc b/data/processed/competing_risks_cif.parquet.dvc deleted file mode 100644 index a773a01..0000000 --- a/data/processed/competing_risks_cif.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 97c46c8c5a0a4e65168f767b5f2e3328 - size: 58371 - hash: md5 - path: competing_risks_cif.parquet diff --git a/data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01_audit.json b/data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01_audit.json deleted file mode 100644 index 67d97cb..0000000 --- a/data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01_audit.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "generated_at_utc": "2026-07-02T06:04:23.949710+00:00", - "frontier_path": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json", - "role": "body/default balanced return-bound point", - "frontier_row": { - "role": "body/default balanced return-bound point", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 131, - "family": "claim_micro_ext_body_cap345", - "anchor_rank": 219, - "source_reason": "candidate37_205_body_cap345_extension", - "risk_tolerance": 0.1715, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5475, - "delta_cap_quantile": 0.975, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05, - "return": 184832.475845, - "return_floor_surplus": 14367.935845, - "Gamma_CP": 0.162616, - "V": 0.03535, - "endpoint_budget_upper": 0.24508374, - "Markov_cap": 0.34508374, - "alpha_pass": "8/8", - "n_funded_mean": 320.5, - "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" - }, - "manifest_path": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext/portfolio/pool93_ijds_local_refinement_manifest.json", - "conformal_intervals_path": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "alpha": 0.01, - "policy": { - "delta_cap_quantile": 0.975, - "gamma": 0.5475, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "capped_blended_uncertainty", - "risk_tolerance": 0.1715, - "solver_backend": "highspy", - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05 - }, - "solver": { - "status": "Optimal", - "backend": "highspy", - "threads": 1 - }, - "metrics": { - "n_funded": 314, - "total_allocated": 1000000.0, - "realized_return": 184832.475845, - "Gamma_CP": 0.162616, - "V": 0.03535, - "weighted_pd_true": 0.03535, - "endpoint_budget_upper": 0.245083866, - "markov_cap": 0.345083866, - "empirical_coverage_funded": 0.942675 - }, - "outputs": { - "funded_rows": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01.parquet", - "grade_summary_parquet": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01_grade_summary.parquet", - "grade_summary_csv": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", - "grade_summary_tex": "/mnt/c/Users/carlos/Documents/Paper_CRPTO/reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex" - } -} \ No newline at end of file diff --git a/data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc b/data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc new file mode 100644 index 0000000..9fbd956 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 88a53915980f9bb906b8eb9bfb1b5fa2.dir + size: 3392967 + nfiles: 4 + hash: md5 + path: ijds-allocation-granularity-sensitivity-2026-07-16-v3 diff --git a/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc new file mode 100644 index 0000000..c6af42f --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 76c4217300b102553832aef25a0fccbc.dir + size: 57987568 + nfiles: 9 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-12-v1 diff --git a/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc new file mode 100644 index 0000000..eb443e3 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: f5a509aade79b34fc9cc929f214ecb8a.dir + size: 47312984 + nfiles: 9 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-14-v3 diff --git a/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc new file mode 100644 index 0000000..ea7b874 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 696a10c2377a7ceab1953ec1366c083b.dir + size: 51149096 + nfiles: 10 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-15-v5 diff --git a/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc new file mode 100644 index 0000000..8c4e308 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 410a84cf6cacd7c077b4bc74bedd45ee.dir + size: 76783301 + nfiles: 9 + hash: md5 + path: ijds-credit-risk-controls-2026-07-13-v1b diff --git a/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc new file mode 100644 index 0000000..9b7246f --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 9b0bdb9bbfc1fb6451b366acd6d50a14.dir + size: 517109 + nfiles: 2 + hash: md5 + path: ijds-credit-risk-controls-2026-07-14-v3 diff --git a/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc new file mode 100644 index 0000000..5afd086 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 9eeb324c1613801452841c769b369a61.dir + size: 521387 + nfiles: 3 + hash: md5 + path: ijds-credit-risk-controls-2026-07-15-v5 diff --git a/data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc b/data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc new file mode 100644 index 0000000..857822e --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc @@ -0,0 +1,6 @@ +outs: +- md5: b05f729bf21f2b35fda69e308c9ba499.dir + size: 143910 + nfiles: 8 + hash: md5 + path: ijds-endpoint-availability-sensitivity-2026-07-14-v1 diff --git a/data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc b/data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc new file mode 100644 index 0000000..c2821a4 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc @@ -0,0 +1,6 @@ +outs: +- md5: a2cd5cced325e569ee60110f663e2907.dir + size: 36529554 + nfiles: 6 + hash: md5 + path: ijds-fit-label-completion-sensitivity-2026-07-16-v2 diff --git a/data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc b/data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc new file mode 100644 index 0000000..d626b14 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 1351952d6883b981d758838791567692.dir + size: 47368489 + nfiles: 6 + hash: md5 + path: ijds-missingness-sensitivity-2026-07-15-v3 diff --git a/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc new file mode 100644 index 0000000..f3a8472 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc @@ -0,0 +1,6 @@ +outs: +- md5: edeccb00c22f2e8cecc6c00ba85fb43a.dir + size: 13015948 + nfiles: 6 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-13-v1c diff --git a/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc new file mode 100644 index 0000000..df82ab8 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: c56f2227e76a805014f6b5b2a3338560.dir + size: 16051446 + nfiles: 6 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-14-v3 diff --git a/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc new file mode 100644 index 0000000..209e966 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: a5069a03115f715029e629c15f79c626.dir + size: 16065187 + nfiles: 6 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-15-v5 diff --git a/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v5.dvc b/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v5.dvc new file mode 100644 index 0000000..97a375e --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 97d2198484b26b9e73faa305162e228b.dir + size: 138923513 + nfiles: 246 + hash: md5 + path: ijds-portfolio-structure-sensitivity-2026-07-15-v5 diff --git a/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc b/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc new file mode 100644 index 0000000..6ffec22 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc @@ -0,0 +1,6 @@ +outs: +- md5: a8660a52ac130b05b42756f0bd1a8ce8.dir + size: 142652135 + nfiles: 330 + hash: md5 + path: ijds-portfolio-structure-sensitivity-2026-07-15-v6 diff --git a/data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc b/data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc new file mode 100644 index 0000000..c6136d8 --- /dev/null +++ b/data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 0a6757e5107cd32ff47a14d2babcff3b.dir + size: 11341341 + nfiles: 10 + hash: md5 + path: ijds-rolling-origin-2017-2026-07-15-v4 diff --git a/data/processed/final_project_summary.parquet.dvc b/data/processed/final_project_summary.parquet.dvc deleted file mode 100644 index dffad09..0000000 --- a/data/processed/final_project_summary.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: bd19c46915ccfcfbf11f5801facc1448 - size: 16440 - hash: md5 - path: final_project_summary.parquet diff --git a/data/processed/hazard_ratios.parquet.dvc b/data/processed/hazard_ratios.parquet.dvc deleted file mode 100644 index e3bb076..0000000 --- a/data/processed/hazard_ratios.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 18ac73913824d4fae3dabfd391718857 - size: 10709 - hash: md5 - path: hazard_ratios.parquet diff --git a/data/processed/ifrs9_scenario_summary.parquet.dvc b/data/processed/ifrs9_scenario_summary.parquet.dvc deleted file mode 100644 index 97fd828..0000000 --- a/data/processed/ifrs9_scenario_summary.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 25396d2d604bddaea934437ce1669f31 - size: 17684 - hash: md5 - path: ifrs9_scenario_summary.parquet diff --git a/data/processed/km_curve_data.parquet.dvc b/data/processed/km_curve_data.parquet.dvc deleted file mode 100644 index 519355c..0000000 --- a/data/processed/km_curve_data.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 44c063d9cc53047e37840d65bf97ad0e - size: 52384 - hash: md5 - path: km_curve_data.parquet diff --git a/data/processed/model_comparison.json.dvc b/data/processed/model_comparison.json.dvc deleted file mode 100644 index 7c66c4b..0000000 --- a/data/processed/model_comparison.json.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 59bfec199f0e23cc13c8a9fb05ce18e5 - size: 15059 - hash: md5 - path: model_comparison.json diff --git a/data/processed/pipeline_summary.json.dvc b/data/processed/pipeline_summary.json.dvc deleted file mode 100644 index b61f014..0000000 --- a/data/processed/pipeline_summary.json.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 364ea3f664f85560fc3e230c3bee9794 - size: 4656 - hash: md5 - path: pipeline_summary.json diff --git a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier.parquet.dvc b/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier.parquet.dvc deleted file mode 100644 index ba9f382..0000000 --- a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier.parquet.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: 958d258b2ff445c7efeea89a6b68a2ba - size: 22205 - isexec: true - hash: md5 - path: portfolio_bound_aware_frontier.parquet diff --git a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier_raw.parquet.dvc b/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier_raw.parquet.dvc deleted file mode 100644 index 8f34629..0000000 --- a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier_raw.parquet.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: a966f3e68de690b0f32d0ea88a476abe - size: 22819 - isexec: true - hash: md5 - path: portfolio_bound_aware_frontier_raw.parquet diff --git a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_shortlist.parquet.dvc b/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_shortlist.parquet.dvc deleted file mode 100644 index acba396..0000000 --- a/data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_shortlist.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: c02d82ac23d427df4d1947a7aa8edca7 - size: 34640 - hash: md5 - path: portfolio_bound_aware_shortlist.parquet diff --git a/data/processed/sicr_conformal_grid.parquet.dvc b/data/processed/sicr_conformal_grid.parquet.dvc deleted file mode 100644 index f9c1118..0000000 --- a/data/processed/sicr_conformal_grid.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 8e6353c5f38aedf7915f3736972aa343 - size: 12999 - hash: md5 - path: sicr_conformal_grid.parquet diff --git a/data/processed/stage_cost_by_grade.parquet.dvc b/data/processed/stage_cost_by_grade.parquet.dvc deleted file mode 100644 index 67d990b..0000000 --- a/data/processed/stage_cost_by_grade.parquet.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 6e9f41db20e264f707d7b68ce2884ccc - size: 9619 - hash: md5 - path: stage_cost_by_grade.parquet diff --git a/dbt_project/models/marts/analytics/_analytics.yml b/dbt_project/models/marts/analytics/_analytics.yml deleted file mode 100644 index 43c3e17..0000000 --- a/dbt_project/models/marts/analytics/_analytics.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 - -models: - - name: fct_conformal_coverage - description: CRPTO conformal coverage by grade. - - name: fct_model_performance_by_cohort - description: PD performance by score band. diff --git a/dbt_project/models/marts/analytics/fct_conformal_coverage.sql b/dbt_project/models/marts/analytics/fct_conformal_coverage.sql deleted file mode 100644 index bfe212b..0000000 --- a/dbt_project/models/marts/analytics/fct_conformal_coverage.sql +++ /dev/null @@ -1,7 +0,0 @@ -select - grade, - count(*) as n_loans, - avg(case when y_true between pd_low_90 and pd_high_90 then 1.0 else 0.0 end) as coverage_90, - avg(width_90) as avg_width_90 -from {{ ref('stg_conformal_intervals') }} -group by 1 diff --git a/dbt_project/models/marts/analytics/fct_model_performance_by_cohort.sql b/dbt_project/models/marts/analytics/fct_model_performance_by_cohort.sql deleted file mode 100644 index 0fa1972..0000000 --- a/dbt_project/models/marts/analytics/fct_model_performance_by_cohort.sql +++ /dev/null @@ -1,12 +0,0 @@ -select - case - when pd_point < 0.05 then '00_low' - when pd_point < 0.10 then '01_medium' - else '02_high' - end as pd_band, - count(*) as n_loans, - avg(y_true) as default_rate, - avg(pd_point) as mean_pd, - avg(abs(y_true - pd_point)) as mean_abs_error -from {{ ref('stg_test_predictions') }} -group by 1 diff --git a/dbt_project/models/marts/credit_risk/_credit_risk.yml b/dbt_project/models/marts/credit_risk/_credit_risk.yml deleted file mode 100644 index 8637398..0000000 --- a/dbt_project/models/marts/credit_risk/_credit_risk.yml +++ /dev/null @@ -1,5 +0,0 @@ -version: 2 - -models: - - name: fct_portfolio_allocation - description: Selected CRPTO bound-aware champion policy. diff --git a/dbt_project/models/marts/credit_risk/fct_portfolio_allocation.sql b/dbt_project/models/marts/credit_risk/fct_portfolio_allocation.sql deleted file mode 100644 index 81b2274..0000000 --- a/dbt_project/models/marts/credit_risk/fct_portfolio_allocation.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - *, - realized_total_return / nullif(total_allocated, 0) as return_on_allocated_capital -from {{ ref('stg_portfolio_allocations') }} -qualify row_number() over (order by alpha01_exact_pass desc, realized_total_return desc) = 1 diff --git a/dbt_project/models/staging/_sources.yml b/dbt_project/models/staging/_sources.yml deleted file mode 100644 index 28510d7..0000000 --- a/dbt_project/models/staging/_sources.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 - -sources: - - name: crpto_files - meta: - external_location: "{{ var('crpto_data_dir') }}" - tables: - - name: test_predictions - - name: conformal_intervals - - name: portfolio_allocations diff --git a/dbt_project/models/staging/_staging.yml b/dbt_project/models/staging/_staging.yml deleted file mode 100644 index 5f0c67a..0000000 --- a/dbt_project/models/staging/_staging.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: 2 - -models: - - name: stg_test_predictions - columns: - - name: loan_id - - name: pd_point - - name: stg_conformal_intervals - columns: - - name: loan_id - - name: stg_portfolio_allocations - columns: - - name: policy_mode diff --git a/dbt_project/models/staging/stg_conformal_intervals.sql b/dbt_project/models/staging/stg_conformal_intervals.sql deleted file mode 100644 index 1b06918..0000000 --- a/dbt_project/models/staging/stg_conformal_intervals.sql +++ /dev/null @@ -1,10 +0,0 @@ -select - cast(id as varchar) as loan_id, - grade, - temporal_segment, - cast(y_true as double) as y_true, - cast(y_pred as double) as pd_point, - cast(pd_low_90 as double) as pd_low_90, - cast(pd_high_90 as double) as pd_high_90, - cast(width_90 as double) as width_90 -from read_parquet('{{ var("crpto_conformal_winner_dir") }}/conformal_intervals_mondrian.parquet') diff --git a/dbt_project/models/staging/stg_portfolio_allocations.sql b/dbt_project/models/staging/stg_portfolio_allocations.sql deleted file mode 100644 index fa43480..0000000 --- a/dbt_project/models/staging/stg_portfolio_allocations.sql +++ /dev/null @@ -1,12 +0,0 @@ -select - policy_mode, - cast(risk_tolerance as double) as risk_tolerance, - cast(gamma as double) as gamma, - cast(uncertainty_aversion as double) as uncertainty_aversion, - cast(realized_total_return as double) as realized_total_return, - cast(n_funded as integer) as n_funded, - cast(total_allocated as double) as total_allocated, - cast(alpha01_exact_pass as boolean) as alpha01_exact_pass, - cast(alpha01_weighted_miscoverage_V as double) as alpha01_weighted_miscoverage_V, - cast(alpha01_gamma_cp as double) as alpha01_gamma_cp -from read_parquet('{{ var("crpto_portfolio_dir") }}/portfolio_bound_aware_shortlist.parquet') diff --git a/dbt_project/models/staging/stg_test_predictions.sql b/dbt_project/models/staging/stg_test_predictions.sql deleted file mode 100644 index 26d085b..0000000 --- a/dbt_project/models/staging/stg_test_predictions.sql +++ /dev/null @@ -1,6 +0,0 @@ -select - loan_id, - cast(y_true as double) as y_true, - cast(y_prob_final as double) as pd_point, - cast(coalesce(pd_calibrated, y_prob_final) as double) as pd_calibrated -from read_parquet('{{ var("crpto_data_dir") }}/test_predictions.parquet') diff --git a/dbt_project/package-lock.yml b/dbt_project/package-lock.yml deleted file mode 100644 index b4c280e..0000000 --- a/dbt_project/package-lock.yml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - - name: dbt_utils - package: dbt-labs/dbt_utils - version: 1.3.3 -sha1_hash: a4da77dcded39caf20bd661f0098cbffd9735800 diff --git a/dbt_project/packages.yml b/dbt_project/packages.yml deleted file mode 100644 index 5214349..0000000 --- a/dbt_project/packages.yml +++ /dev/null @@ -1,4 +0,0 @@ -packages: - # General-purpose utilities: surrogate keys, audit columns, deduping macros. - - package: dbt-labs/dbt_utils - version: [">=1.3.0", "<2.0.0"] diff --git a/dbt_project/tests/assert_default_rate_reasonable.sql b/dbt_project/tests/assert_default_rate_reasonable.sql deleted file mode 100644 index 1037768..0000000 --- a/dbt_project/tests/assert_default_rate_reasonable.sql +++ /dev/null @@ -1,7 +0,0 @@ -with rates as ( - select avg(y_true) as default_rate - from read_parquet('{{ var("crpto_data_dir") }}/test_predictions.parquet') -) -select * -from rates -where default_rate <= 0 or default_rate >= 0.5 diff --git a/dbt_project/tests/assert_pd_in_range.sql b/dbt_project/tests/assert_pd_in_range.sql deleted file mode 100644 index 272ca8e..0000000 --- a/dbt_project/tests/assert_pd_in_range.sql +++ /dev/null @@ -1,6 +0,0 @@ -select * -from read_parquet('{{ var("crpto_data_dir") }}/test_predictions.parquet') -where y_prob_final < 0 - or y_prob_final > 1 - or coalesce(pd_calibrated, y_prob_final) < 0 - or coalesce(pd_calibrated, y_prob_final) > 1 diff --git a/docs/ACADEMIC_CONTEXT.md b/docs/ACADEMIC_CONTEXT.md index 9c31f65..9a52518 100644 --- a/docs/ACADEMIC_CONTEXT.md +++ b/docs/ACADEMIC_CONTEXT.md @@ -1,117 +1,55 @@ -# CRPTO — academic context and operating principles - -This document captures the operating reality of CRPTO so that any future -collaborator (human or agent) does not over-engineer for an industrial -deployment that is never going to happen. - -## What this project is - -- A **master's thesis** by Carlos Alfredo Vergara Rojas. -- A pipeline + Quarto book + paper draft + journal package built on the - **Lending Club Loan Data 2007–2020** (Kaggle mirror, 2.26M loans, 174 raw - columns → 21 modeling features). -- The deliverables are the **paper PDF**, the **Quarto book** rendered to - HTML (and PDF for journal), the **18 paper tables**, the **8 paper figures** - and the **MRM dossier**. - -## What this project is NOT - -- **Not going to production.** There is no live scoring service, no - microservice, no batch nightly job. No SLAs, no on-call. -- **Not a multi-author project.** Solo author. No code review board, no - PR approval workflow, no separate QA team. The CLAUDE.md operating rules - exist to keep agents disciplined, not to satisfy a corporate process. -- **Not getting new data.** Lending Club closed retail loan origination in - late 2020. The dataset is **static and complete** for the period covered. - There is no streaming pipeline, no schema drift, no concept drift on new - cohorts. If we re-train, we re-train on the same historical window. -- **Not commercial software.** MIT-licensed code, CC-BY 4.0 text. Treat - reviewers (paper, journal, MRM) as the only "stakeholders". - -## Consequences for engineering decisions - -| Decision | Industrial default | CRPTO academic stance | -| --- | --- | --- | -| CI/CD coverage | every push runs full test matrix | only render the book + lint; tests run locally via pre-push hook | -| Branch protection on `main` | required reviews, status checks | none — single author can push direct | -| Dependabot version PRs | auto-merge after CI | dismissed by default; manual review when a real CVE shows up | -| Monitoring / observability | dashboards, alerts | MLflow for experiment trace, nothing else | -| Feature stores, registries | Hopsworks, Feast | DVC + dbt on DuckDB, all local | -| Re-training schedule | weekly/quarterly retrains | never automatically; only when the paper needs revision | -| Disaster recovery | multi-region backups | git + DVC remote on DagsHub is enough | -| Secrets management | Vault, KMS | `.env` local + GitHub Actions secrets if needed | - -## Re-running the champion - -The "frozen champion" rule still holds for `dvc.yaml` stages that **search** -for hyperparameters or policies (the 276k portfolio sweep, Optuna HPO, -conformal alpha/group selection). The search results are the contribution -of the paper. - -However, **stages that consume those frozen choices may be re-run** when -needed (e.g. to validate a library migration with drift comparison). The -search itself does not get re-run. - -| Stage | Search? | Re-runnable for validation? | -| --- | --- | --- | -| `crpto.data.dataset` | no (deterministic split) | yes | -| `crpto.data.features` | no (fixed feature config) | yes | -| `crpto.pd.champion` | no (hyperparams already frozen in `configs/crpto_pd_model.yaml`) | yes, with drift check | -| `crpto.conformal.intervals` | no (alpha/groups frozen in `configs/crpto_conformal_policy.yaml`) | yes, with drift check | -| `crpto.conformal.validation` | no | yes | -| `crpto.portfolio.optimization` | no (policy frozen in `configs/crpto_optimization.yaml`) | yes, with drift check | -| `crpto.portfolio.bound_exact_eval` | **YES — search of 276k policies** | **NO. The chosen rank-1 is the paper contribution.** | -| `crpto.paper.*` | no | yes (regenerate tables/figures freely) | -| `crpto.book.render` | no | yes | - -Drift check default tolerance: max abs diff `≤ 1e-6` per loan on -`conformal_intervals_mondrian.parquet`, coverage delta `≤ 5e-4` per Mondrian -cell, portfolio robust return delta `≤ $1.00` (~6 ppm of $170,464.54). - -## GitHub strategy (single-author public repo) - -The repo is public for academic reproducibility. We keep: - -- `book-publish.yml` — auto-deploys the Quarto book to GitHub Pages on every - push to `main`. This is the single most valuable workflow because - reviewers can read the book at https://eigencharlie.github.io/Paper_CRPTO/. -- `lint.yml` — catches formatting/import issues before they accumulate. - Runs in ~30s. - -We **drop** (or never adopt): - -- Branch protection requiring reviews — no second author exists. -- Dependabot version PRs — too noisy when nobody triages them. Security - alerts (Dependabot's separate "alerts" tab) remain on because they only - fire for real CVEs. -- A full `pytest` workflow on every push — local pre-push hook already runs - the artifact-independent suite, and reviewers do not consult the test tab. -- `dbt.yml` and `book-build.yml` — `dbt parse` without parquets is trivial - and the book build is already part of book-publish. - -If the project ever moves to multi-author (very unlikely), revisit this -document and re-enable the dropped workflows + branch protection. - -## What we keep doing rigorously even in academic mode - -These remain non-negotiable because they protect the paper's -reproducibility: - -1. **Pin every dependency** in `uv.lock` and bump floors only when the - lockfile already moved. -2. **DVC for data + model artifacts**; never commit binaries to Git. -3. **Pre-commit hooks** (ruff, nbstripout, non-blocking DVC drift report, - smoke tests). -4. **Pre-push hooks** that run the artifact-independent test suite. -5. **MAPIE / Pandera / Optuna / MLflow** kept on the same major version as - `uv.lock` (no silent surprises). -6. **EXTRACTION_MANIFEST.json** hashes are the canonical source of "is the - champion intact?" — validated by the `crpto-validate-champion` skill - and the `validate-champion` pre-push hook. - -## When in doubt - -Default to *less* tooling, not more. Every CI hook, every workflow, every -service has a maintenance cost that is paid by exactly one person. If a -proposed tool does not help write the paper or render the book, it does -not belong here. +# Academic Context + +CRPTO is a static-data research project for one IJDS paper. It is not a live +credit system, a model-serving product, or a multi-author software platform. + +## Research Unit + +The archive contains accepted Lending Club loans, not all applicants. The +active design constructs status-independent candidate menus and assigns data to +model fitting, residual fitting, policy development, and OOT evaluation by +time. Monthly allocations, not individual loans, are the coupled decision +contexts. + +The file is not a verified point-in-time snapshot. Outcome availability is +therefore reconstructed at a declared September 30, 2020 cutoff. Unresolved +outcomes remain in the candidate population and enter sharp identification +bounds rather than being dropped. + +## Evidence Layers + +1. **Prediction controls:** CatBoost/Platt plus four frozen learner + specifications describe discrimination, calibration, WOE/IV, monotonicity, + PSI, and all-candidate coverage. +2. **Binary geometry:** exact residual ranks and set intersections explain how + a prevalence threshold can change interval width discontinuously. +3. **Decision construction:** outcome-blind rulers and coordinates create + comparable monthly allocation tracks without selecting a winner. +4. **Identification:** paired common-outcome bounds and exact point-cap support + determine which policy contrasts are signed and which cross zero. +5. **Sensitivity:** endpoint availability, fit-label timing, missingness + encoding, later origin, and portfolio structure are separate axes. They are + not interchangeable replications and were not crossed factorially. + +## Engineering Consequences + +- Reproducibility and claim synchronization matter more than deployment + infrastructure. +- Current code is typed and tested at scientific boundaries: temporal roles, + ID joins, outcome isolation, solver contracts, sharp bounds, and manuscript + numbers. +- Tests should protect a possible scientific failure mode, not preserve every + historical implementation detail. +- Historical DVC and extraction paths remain sealed only because immutable + hashes bind their locations. They are excluded from active recipes and paper + claims. +- The full pre-consolidation project is available at `D:\crpto_legacy`; the Git + repository should not narrate prior iterations. + +## Interpretation Boundary + +The active results are retrospective and archive-specific. They do not imply +prospective exchangeability, selected-set validity, causal funding effects, +deployment safety, fair-lending compliance, or external validity. The economic +quantity is a standardized status-indexed payoff proxy, not realized cash-flow +return, NPV, IRR, or welfare. diff --git a/docs/PROJECT_HISTORY.md b/docs/PROJECT_HISTORY.md deleted file mode 100644 index 3efbd5a..0000000 --- a/docs/PROJECT_HISTORY.md +++ /dev/null @@ -1,46 +0,0 @@ -# CRPTO project history and extraction lessons - -CRPTO is now an independent repository with its own GitHub project, DVC remote -and MLflow tracking URI. This note is intentionally historical: it records how -the standalone project was extracted and hardened, but it is not active runtime -configuration. - -## Origin - -- The first standalone CRPTO folder was extracted from a broader internal - research workspace. -- The extraction copied only the CRPTO paper lane: Lending Club data, - canonical PD artifacts, conformal diagnostics, robust portfolio optimization, - paper tables/figures, Quarto book chapters, dbt lineage and tests. -- Streamlit, FastAPI, papers 2/3, survival, causal, IFRS9, quantum/GPU labs - and insights-factory code were deliberately left out. - -## Hardening decisions - -- The public GitHub repository is `EigenCharlie/Paper_CRPTO`. -- The standalone DVC remote is - `https://dagshub.com/EigenCharlie94/Paper_CRPTO.s3`. -- The standalone MLflow tracking URI is - `https://dagshub.com/EigenCharlie94/Paper_CRPTO.mlflow`. -- The old parent DagsHub project is provenance only. It must not appear in - active configs, templates, workflows or secrets. -- The current IJDS baseline is `ijds-rebaseline-2026-06-07`. The earlier - `paper-thesis-final-economic-2026-04-06` run remains historical provenance; stages - that search for the champion remain blocked unless a revalidation plan exists. -- WSL-era absolute paths preserved inside frozen JSON status files are treated - as provenance, not active configuration. They are left intact unless a future - revalidation deliberately re-freezes the artifact family under a new run tag. - -## Lessons from the extraction - -- Mixed shell line endings can dirty `.dvc` pointer files, so `.gitattributes` - pins DVC/YAML/Quarto/Python/text files to LF. The current working mode is - Windows PowerShell native. -- Local secret duplicates such as `.env_original` and root-level `config.local` - create accidental-push risk; secrets now live in `.env` and - `.dvc/config.local` only. -- GitHub CI stays lightweight for single-author academic work. `tests-full` is - manual and strict, because it requires DVC artifacts. -- `feature_config.yml` plus `feature_config.parquet` are the live champion - feature contract as of the 2026-06-13 run-tag-approved cleanup; the legacy - `feature_config.pkl` was removed from the DVC stage and manifest. diff --git a/docs/SCOPE_AND_GOVERNANCE.md b/docs/SCOPE_AND_GOVERNANCE.md index 5f0d253..c872705 100644 --- a/docs/SCOPE_AND_GOVERNANCE.md +++ b/docs/SCOPE_AND_GOVERNANCE.md @@ -1,181 +1,99 @@ -# CRPTO scope and governance - -This repository is the standalone home for the CRPTO paper, journal package, -Quarto book and reproducibility pipeline. It is intentionally scoped to the -CRPTO paper lane only; extraction history and lessons are kept in -`docs/PROJECT_HISTORY.md`. - -## Public scope - -CRPTO covers: - -- Lending Club credit-risk data preparation used by the frozen paper run. -- Canonical PD champion, calibration contract and model-risk documentation. -- Conformal prediction intervals and diagnostics used by the paper evidence. -- Robust predict-then-optimize portfolio policy and SPO/funded-set analyses. -- Fair-lending, governance, MRM, traceability and journal appendix material. -- Quarto book, manuscript draft, tables, figures and publication exports. -- CI, dbt, DVC metadata, MLflow/DagsHub integration templates and local skills. - -CRPTO does not cover: - -- Parent-project Streamlit or FastAPI apps. -- Paper 2/Paper 3 lanes, IFRS9, survival, causal, quantum/GPU labs or the - insights factory, except where a CRPTO chapter cites prior context. -- New model-training research unless it is explicitly isolated from the frozen - champion or run under a new tag. - -## Active paper contract - -The current IJDS body point is the simple calibration-selected 90% guardrail: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` -- exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`; -- policy: `q=(p+u)/2`, `tau=0.17`, with point PD in the economic objective - and conformal `q` in the risk constraint; -- selector: nine round-number policies on November 2017, five eligible under - deterministic full-budget, effective-PD, and `B_u<=0.28` screens; no outcome - or assumption-conditional selector columns; -- independent audit: the outcome-free December replay selects the same policy; - after outcomes are opened, weighted default is `0.145650` and miscoverage is - `0.124925`, explicitly ruling out a selected-set validity claim; -- realized return: `$179,327.59`; -- weighted default and miscoverage: `0.039375` and `0.036875`; -- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`; -- endpoint budget `0.258051`; assumption-conditional Markov threshold - `0.574279`; -- paper artifacts: A35 exact-alpha audit, A36 split selector/audit, A37 - temporal evaluation, A38 grade audit, A39 month-cluster bootstrap with - loan-level sensitivity, and A40 matched comparisons. - -The selector is outcome-free with respect to OOT policy ranking, but earlier -project development inspected the static OOT corpus. The paper must describe -the result as a retrospective lockbox replay, not a pristine prospective trial. - -The previous IJDS rebaseline and pool93 frontier are retained as historical -provenance, not as active body claims: - -- run tag: `ijds-rebaseline-2026-06-07` -- policy: `bound_aware_276k_economic_champion` -- robust return: `$170,464.54` -- `V(alpha=0.01)=0.028875` -- `Gamma_CP(alpha=0.01)=0.187987` -- exact pass: `true` -- robust region: `45/45` - -The older run tag `paper-thesis-final-economic-2026-04-06` is retained as -historical provenance, not as the current manifest baseline. - -Do not overwrite these protected files without an explicit revalidation plan: - -- `models/pd_canonical.cbm` -- `models/pd_canonical_calibrator.pkl` -- `models/final_project_promotion.json` -- `models/conformal_policy_status.json` -- `data/processed/conformal_intervals_mondrian.parquet` -- `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` -- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `EXTRACTION_MANIFEST.json` - -## Safe work - -Safe changes by default: - -- Documentation, README, runbooks, Quarto prose and non-executed book renders. -- Artifact-independent tests and adapters that do not change model outputs. -- CI/workflow maintenance, dependency-review metadata and public docs. -- Regenerating CRPTO tables, figures, evidence summaries and journal package - from already-frozen artifacts. -- Adding tests around utilities, schemas, policy aliases and pipeline state. - -Potentially safe but review first: - -- dbt model changes that only read existing CRPTO DuckDB/parquet artifacts. -- DVC metadata edits that do not repro protected stages. -- Dependency floor bumps that do not affect protected model or conformal code. - -Not safe on `main`: - -- `dvc repro crpto.pd.champion` -- `dvc repro crpto.conformal.intervals` -- `dvc repro crpto.conformal.validation` -- `dvc repro crpto.portfolio.optimization` -- `dvc repro crpto.portfolio.bound_exact_eval` -- MAPIE, feature-config or conformal refactors that require fresh champion - artifacts before drift validation. - -## Refactor lanes - -The files in `docs/refactor/` are plans, not approvals to execute. The current -high-risk lanes are: - -- `MAPIE_MIGRATION_PLAN.md`: code is MAPIE 1.x-compatible and the June 2026 - drift report is green under the current stack. Protected conformal stage - reruns still require explicit approval. -- `CONFORMAL_REFACTOR_PLAN.md`: script-level extraction is underway, but - class/module modularization must preserve pickled calibrator compatibility - or create a new run tag. -- `archive/FEATURE_CONFIG_PARQUET_PLAN.md` (executed 2026-06-13, archived): - the live feature contract is now YAML/Parquet; changing it affects the - data/features contract and requires downstream validation. - -## Public GitHub rules - -The GitHub repo is public: . - -Keep in Git: - -- Source, tests, Quarto, docs, tables, figures, JSON status files, DVC lock - files and DVC pointer files. - -Keep out of Git: - -- `.env`, `.env.*` except templates, `.dvc/config.local`, raw CSVs, processed - parquet/DuckDB files, model binaries, MLflow runs, local caches and tokens. - -GitHub repository security currently expects: +# Scope And Governance -- Dependency graph and Dependabot security updates enabled. -- Secret scanning enabled. -- No required branch protection in the current single-author academic mode. - If CRPTO becomes multi-author, re-enable branch protection with at least - `lint` and `book-publish` as required checks. +## Public Scope -The default CI must remain lightweight: `lint` and `book-publish` run on push. -The artifact-aware `tests-full` workflow is manual and should be run before -journal milestones or any protected-stage revalidation. +This repository contains the active CRPTO IJDS research object: -## Environment +- exact data-role, endpoint, and feature contracts; +- frozen prediction and binary conformal coverage controls; +- outcome-blind portfolio construction and exact comparator analysis; +- registered aggregate evidence, tables, and figures; +- the anonymous paper, supplement, and official submission build; +- tests and CI needed to reproduce and audit those outputs. -Use the Windows-native project environment: +It does not contain a production service, a live lending policy, an experiment +dashboard, or a narrative of prior CRPTO versions. -- Windows PowerShell: `.venv/Scripts/python.exe` -- Python tools: `uv run ...` -- Quarto renders: `uv run -- quarto ...` +## Authorities -Do not route normal CRPTO work through non-Windows shells. If a shell or tool -creates a non-Windows virtualenv layout, treat it as a misconfigured local -environment and recreate the venv from PowerShell. +| Concern | Authority | +|---|---| +| Active prose claims | `docs/research/active_claims_2026-07-14.md` | +| Lineage and DVC identities | `configs/ijds_active_evidence_sources.yaml` | +| Executable qualitative claims | `configs/ijds_claim_ledger.yaml` | +| Numeric paper evidence | `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` | +| Active code surface | `configs/crpto_publication_targets.yaml` | +| Historical protected hashes | `EXTRACTION_MANIFEST.json` | -## Release checklist +No memo, old manuscript, stopped protocol, or unregistered run may override +these files. -Before considering a CRPTO change ready: +## Safe Operations -1. `just lint` -2. `just smoke` -3. `uv run pytest tests/test_utils/test_pipeline_state.py tests/test_utils/test_optuna_storage.py -q` -4. `uv run -- quarto render book --to html --no-execute` -5. Confirm `git status --short` does not include data, models or secrets. +- Read registered roots and rebuild active aggregate evidence. +- Render the body, supplement, generated TeX, and PDFs. +- Run tests, lint, static typing, source verification, and visual QA. +- Refactor current code when scientific outputs and lineage identities remain + stable. +- Add a new, predeclared, contained experiment with a distinct run tag. -For any protected-stage or dependency migration, add a branch-specific drift -report before merge. +## Protected Operations + +Explicit permission is required before running: + +- `crpto.pd.champion` +- `crpto.conformal.intervals` +- `crpto.conformal.validation` +- `crpto.portfolio.optimization` +- `crpto.portfolio.bound_exact_eval` + +Do not modify `EXTRACTION_MANIFEST.json` or overwrite protected artifacts. A +new result belongs under a new run tag; it must not replace a frozen root. + +## Sealed Compatibility + +`dvc.yaml`, `dvc.lock`, and files whose exact paths are fixed by the extraction +manifest form a non-executable compatibility capsule. Their only current role +is hash and provenance verification. Active commands must not call them, and +the manuscript must not cite their historical results. + +The complete project history is preserved outside the repository in +`D:\crpto_legacy`, including a Git mirror, verified bundle, snapshots, and the +pre-consolidation material worktree. + +## Data And Secrets + +Keep raw CSVs, DVC cache, model binaries, local PDFs, `.env` files, tokens, and +temporary benchmark outputs out of Git. Commit only DVC pointers and aggregate +artifacts allowed by the active publication contract. Never place credentials +in YAML, code, notebooks, logs, or submission files. + +## Change Classification + +| Change | Required validation | +|---|---| +| Prose/citation only | claim sync, TeX generation, manuscript build | +| Deterministic builder | focused tests, idempotence, evidence reconciliation | +| Current code refactor | full tests, lint, mypy, ty, active checks | +| Data/model/conformal behavior | new protocol/run tag plus explicit drift and scientific review | +| Submission closeout | all gates, strict protected hashes, DVC remote, visual QA | + +## Closeout + +Before synchronization: + +```powershell +just test +just lint +just type-check +just type-check-fast +just drift-gate +just ijds-active-check +just validate-champion-strict +just submission-build +just submission-check +just ijds-dvc-verify-remote +``` + +The project remains `prefreeze_active`; a clean closeout is not a submission +freeze. diff --git a/docs/THESIS_PDF_SCOPE.md b/docs/THESIS_PDF_SCOPE.md deleted file mode 100644 index 038427b..0000000 --- a/docs/THESIS_PDF_SCOPE.md +++ /dev/null @@ -1,90 +0,0 @@ -# Alcance Del PDF De Tesis APA - -Fecha: 2026-06-08. - -Este documento fija la decision operativa sobre el PDF largo de tesis. El libro -Quarto HTML es el companion vivo del proyecto; `paper/CRPTO_ijds.pdf` y -`paper/supplement_ijds.pdf` son borradores locales de verificacion para IJDS. El -PDF completo `CRPTO.pdf` del libro no se mantiene como artefacto rutinario. - -## Decision - -No vamos a sostener un PDF automatico de todo el libro Quarto. El libro completo -tiene demasiada superficie para un export unico: codigo largo, tablas anchas, -figuras pensadas para HTML, appendices granulares y paginas que funcionan bien -como dossier navegable pero no como documento APA paginado. Mantener ese PDF -agrega ruido editorial y no mejora el paper IJDS ni la tesis de maestria en su -estado actual. - -Las superficies activas son: - -| Superficie | Estado | Regla | -|---|---|---| -| Libro Quarto HTML | Mantener | Companion primario, navegable y amplio. | -| `CRPTO_ijds.pdf` | Mantener | Borrador HTML-print para revisar el paper; el PDF final usa plantilla IJDS. | -| `supplement_ijds.pdf` | Mantener | Borrador HTML-print para revisar el supplement. | -| `CRPTO.pdf` libro completo | No mantener | `just book-pdf` es no-op intencional. | -| PDF tesis APA | Diferir | Se construye despues como documento curado con secciones seleccionadas. | - -## Cuando construir el PDF de tesis - -El PDF APA de tesis se activa solo cuando exista una decision academica sobre: - -1. Capitulos que entran al documento de maestria. -2. Norma APA y reglas institucionales vigentes. -3. Longitud objetivo y material que debe ir a apendices. -4. Figuras y tablas que realmente soportan la defensa. -5. Tratamiento de codigo: oculto, resumido o movido a apendice/repositorio. - -Mientras falten esas decisiones, el trabajo correcto es fortalecer el libro HTML, -el paper IJDS y el supplement. - -## Arquitectura recomendada - -Cuando se active, el PDF de tesis debe ser un proyecto curado, no el libro -entero impreso: - -| Componente | Recomendacion | -|---|---| -| Fuente | Nuevo perfil o carpeta `thesis/` que importe secciones seleccionadas del libro. | -| Ejecucion | `execute: false` / `freeze` contra artefactos ya validados; nunca reabre champion. | -| Estilo | APA 7, portada institucional, resumen, palabras clave, tabla de contenido, lista de figuras/tablas si aplica. | -| Codigo | Oculto en el cuerpo; fragmentos solo si son metodologicamente necesarios. | -| Tablas | Versiones compactas en cuerpo; tablas anchas a apendice landscape, CSV o HTML companion. | -| Figuras | PNG/PDF paper-grade con dimensiones fijas; ninguna figura debe ocupar pagina completa y quedar cortada. | -| Referencias | `book/references.bib` + `book/apa.csl`, auditadas antes del cierre. | -| Apendices | Solo material que un jurado pueda necesitar; no volcar A0--A34 completo sin curadoria. | - -## Checklist de activacion - -Antes de crear el PDF APA: - -```powershell -just lint -just smoke -just validate-champion -just paper-submission-pdf -just book -``` - -Luego, para el documento de tesis: - -1. Definir outline con maximo 6--8 capitulos principales. -2. Crear una matriz seccion -> claim -> evidencia -> figura/tabla. -3. Decidir que tablas se resumen y cuales quedan fuera del PDF. -4. Renderizar un PDF piloto de 20--30 paginas. -5. Revisar visualmente pagina por pagina: tablas, figuras, captions, - referencias cruzadas, saltos de pagina y overflow. -6. Solo despues escalar al documento completo. - -## No objetivos - -- No convertir el libro completo en PDF por obligacion tecnica. -- No usar el PDF de tesis para reabrir la busqueda del champion. -- No duplicar el paper IJDS dentro de la tesis sin adaptarlo a la narrativa de - maestria. -- No mantener `CRPTO.pdf` si su unica utilidad es detectar problemas de layout - que ya sabemos que nacen de imprimir un dossier HTML de 400+ paginas. - -La tesis debe ser mas selectiva que el libro y mas pedagogica que el paper. El -PDF APA se construira cuando esa seleccion este clara. diff --git a/docs/refactor/CONFORMAL_REFACTOR_PLAN.md b/docs/refactor/CONFORMAL_REFACTOR_PLAN.md deleted file mode 100644 index dfb03fd..0000000 --- a/docs/refactor/CONFORMAL_REFACTOR_PLAN.md +++ /dev/null @@ -1,171 +0,0 @@ -# Refactor plan — `src/models/conformal.py` - -**Status**: Executed 2026-06-13. `src.models.conformal` is now a package -facade with focused submodules for scores, PD intervals, classification and -regression. The legacy public import path and pickle module path remain -stable. - -> **2026-06-13 finding (supersedes the "needs a calibrator re-pickle window" -> caveat).** The frozen calibrator `models/pd_canonical_calibrator.pkl` does -> **not** reference any class from `conformal.py`. Inspected with -> `pickletools` and by loading it: its object graph contains only -> `src.models.venn_abers.VennAbersScoreCalibrator` and the upstream -> `venn_abers.venn_abers.VennAbers`. Therefore splitting `conformal.py` is -> pickle-safe; the only invariant is to keep `src/models/venn_abers.py`'s -> module path stable (there is no reason to move it). Verify with: -> `uv run python -c "import pickle,pathlib; o=pickle.loads(pathlib.Path('models/pd_canonical_calibrator.pkl').read_bytes()); print(type(o).__module__)"` -> → `src.models.venn_abers`. Acceptance gate is `just drift-gate` (green). - -## What is done - -- Full split executed: - - `src/models/conformal/__init__.py` re-exports the public API. - - `src/models/conformal/_scores.py` owns score quantiles/scaling. - - `src/models/conformal/pd_intervals.py` owns PD intervals and Venn-Abers. - - `src/models/conformal/classification.py` owns sets, partitions and - score-space cross conformal. - - `src/models/conformal/regression.py` owns regression/residual intervals. -- `validate_coverage` and `summarize_prediction_sets` moved to - `src/models/conformal_diagnostics.py` (pure data summaries, no pickled - state). `src.models.conformal` re-exports them so legacy imports keep - working. -- `tests/test_models/test_calibrator_pickle_compat.py` added. Any future - refactor that mutates `__module__` for the pickled classes will fail - this test, forcing explicit pickle compat shims. -- `tests/test_models/test_conformal_artifact_properties.py` added. Validates - the structural invariants of the frozen - `conformal_intervals_mondrian.parquet` (monotone bounds, 90/95 coverage, - Mondrian conditional coverage by grade). - -## What remains - -No conformal split work remains. Future conformal changes should be tied to a -new methodological reason, not module-size cleanup. - -## Context - -`src/models/conformal.py` is 845 LOC and exposes 18 public functions plus -3 classes. The original review (plan §6, item 4) flagged it as a "monolithic -module" worth splitting into focused submodules. This document captures the -proposed structure, the constraints that prevented an in-place refactor in -the bootstrap commit, and the green-light criteria for executing it later. - -## Why it was not refactored in the bootstrap - -`models/pd_canonical_calibrator.pkl` (the frozen champion's calibrator) is a -pickle that stores the **fully-qualified module path** of every class it -serialises. The pickle very likely references -`src.models.conformal.ProbabilityRegressor` and -`src.models.conformal.PrefitClassifierAdapter`. Moving those classes to a -submodule changes their `__module__` attribute, which breaks `pickle.load` -of the existing artefact, which in turn invalidates the run tag -`paper-thesis-final-economic-2026-04-06`. - -Verifying that the move is pickle-safe requires either: - -1. Loading the calibrator, re-pickling under the new namespace, and - comparing predictions bit-for-bit on `data/processed/calibration_fe.parquet`, - or -2. Re-running the `crpto.pd.champion` stage with the new module layout and - confirming the new SHA256 in `EXTRACTION_MANIFEST.json` matches. - -Neither option is in scope for the bootstrap commit because the champion is -frozen and the DVC stage is on the deny-list of `.claude/settings.json`. - -## Proposed structure - -``` -src/models/conformal/ -├── __init__.py # re-export everything for backwards-compat -├── _adapters.py # ProbabilityRegressor, PrefitClassifierAdapter, -│ # PrefitCalibratedClassifierAdapter -├── _scores.py # _conformal_quantile, _resolve_score_scale_family, -│ # _compute_score_scale -├── pd_intervals.py # create_pd_intervals, create_pd_intervals_mondrian, -│ # create_pd_intervals_venn_abers, -│ # conditional_coverage_by_group, -│ # apply_probability_calibrator -├── regression.py # create_regression_intervals, create_residual_intervals -├── classification.py # create_classification_sets, -│ # _create_margin_classification_sets, -│ # summarize_prediction_sets, -│ # create_classification_sets_mondrian, -│ # build_mondrian_partition_labels, -│ # create_cross_conformal_score_intervals -└── diagnostics.py # validate_coverage -``` - -`__init__.py` re-exports every public symbol so existing call sites keep -working: - -```python -from ._adapters import ( - PrefitCalibratedClassifierAdapter, - PrefitClassifierAdapter, - ProbabilityRegressor, -) -from .classification import ( - build_mondrian_partition_labels, - create_classification_sets, - create_classification_sets_mondrian, - create_cross_conformal_score_intervals, - summarize_prediction_sets, -) -from .diagnostics import validate_coverage -from .pd_intervals import ( - apply_probability_calibrator, - conditional_coverage_by_group, - create_pd_intervals, - create_pd_intervals_mondrian, - create_pd_intervals_venn_abers, -) -from .regression import create_regression_intervals, create_residual_intervals -``` - -## Pickle compatibility strategy - -When we are ready to execute the split: - -1. **Pin pickle module aliases first.** Add to `src/models/conformal/__init__.py`: - - ```python - import sys - # Backwards-compat: pickles created before the split reference - # ``src.models.conformal.ProbabilityRegressor`` directly. Keep that path - # alive after the split. - sys.modules.setdefault( - "src.models.conformal", sys.modules[__name__] - ) - ``` - - This is a no-op for Python's normal import machinery but ensures pickle's - `find_class` resolves the legacy fully-qualified name to the new package. - -2. **Add a calibrator round-trip test** before any restructuring: - - ```python - def test_calibrator_pickle_round_trip_after_refactor(): - import joblib - cal = joblib.load("models/pd_canonical_calibrator.pkl") - cal.predict(np.array([0.1, 0.5, 0.9])) # must not raise - ``` - -3. **Run on a non-champion branch** and compare predictions on - `data/processed/test_predictions.parquet` for the first 1000 rows. - Tolerance: bit-exact (numpy `array_equal`). - -## Acceptance criteria - -The refactor lands on `main` when ALL of the following are true: - -- `tests/test_models/test_conformal*` (existing) — all green. -- A new `test_calibrator_pickle_round_trip` — green. -- `dvc status` against the frozen champion — clean (no drift). -- The first 1000 predictions of `test_predictions.parquet` — byte-exact - match against the pre-refactor commit. - -## Timing - -Schedule for the next paper-iteration window when the champion can be -re-validated. Until then, the monolithic `conformal.py` remains the source -of truth. diff --git a/docs/refactor/MAPIE_MIGRATION_PLAN.md b/docs/refactor/MAPIE_MIGRATION_PLAN.md deleted file mode 100644 index c7d4350..0000000 --- a/docs/refactor/MAPIE_MIGRATION_PLAN.md +++ /dev/null @@ -1,114 +0,0 @@ -# MAPIE 0.9 → 1.x migration plan - -**Status (2026-06-10)**: **RESOLVED — gate GREEN with zero drift.** The -April candidate binary was restored and the canonical PD identity was -unified to it (april-lineage unification, see -`drift_report_mapie_2026-06.md` and `EXTRACTION_MANIFEST.json`). The -harness (`tests/test_models/test_conformal_mapie_drift.py`, opt-in via -`CRPTO_RUN_CHAMPION_DRIFT=1` because it is slow) reproduces the frozen -champion intervals bit-exactly under the current MAPIE 1.4 / numpy / -catboost stack, including re-learning the identical floor multipliers. -The MAPIE 1.x migration is therefore revalidated; the remaining items in -this plan (conformal module split, see `CONFORMAL_REFACTOR_PLAN.md`) can -proceed against this green gate whenever scheduled. The `_mapie_compat.py` -shim sketched below was created during the gate work but never adopted by -any call site (the helper surface already uses 1.x names directly), so it -was removed as dead code in the R0 cleanup (2026-06-12); the snippet stays -only as historical patch shape. - -**Historical status (2026-06-09)**: gate RED — the frozen intervals were -generated by the April search candidate while `pd_canonical.cbm` was the -June rebaseline retrain (corr 0.9917, not bit-exact). The drift was -upstream of MAPIE (the champion Mondrian path uses plain numpy quantiles). - -**Previous status**: Partially code-compatible, not champion-revalidated. The CRPTO -runtime is already on MAPIE 1.x and the main helper imports use the 1.x class -names, but the frozen champion conformal artefact has not been regenerated or -drift-tested under a fresh MAPIE 1.x run. - -## Context - -MAPIE 1.0 rewrote the public API: - -- `MapieRegressor` / `MapieClassifier` → `SplitConformalRegressor`, - `CrossConformalRegressor`, `SplitConformalClassifier` (and - `CrossConformalClassifier`). -- `MondrianCP` becomes a standalone wrapper instead of an argument. -- The conformity score selection moved from string `method=` arguments to - explicit `conformity_score=` objects. - -`pyproject.toml` already pins `mapie>=1.4`, so the runtime is on MAPIE 1.x. -`src/models/conformal.py` has been moved to the 1.x public class names for the -helper surface. This is **not** the same as validating the CRPTO champion: -protected stages such as `crpto.conformal.intervals` and -`crpto.conformal.validation` remain blocked until the drift harness below is -run on a branch. - -## Why the champion was not re-run in the bootstrap - -The champion's conformal artefact -`data/processed/conformal_intervals_mondrian.parquet` was generated under -MAPIE 0.9 with very specific quantile/scoring choices. Switching to the 1.x -API can change numerical results by `~1e-6` even with identical seeds because -default conformity score families changed (`absolute_residual` → `absolute`, -`apsbinary` → `aps`, …). Any drift invalidates the run tag -`paper-thesis-final-economic-2026-04-06`. - -The deny-list in `.claude/settings.json` blocks `dvc repro -crpto.conformal.intervals` for exactly this reason. - -## Acceptance criteria - -Execute the migration when ALL of the following can be verified: - -1. **Compatibility shim ready.** Wrap the old call sites in a thin adapter - so the diff focuses on imports rather than logic: - - ```python - # src/models/_mapie_compat.py - try: - from mapie.regression import SplitConformalRegressor # 1.x - from mapie.classification import SplitConformalClassifier - except ImportError: # pragma: no cover - from mapie.regression import MapieRegressor as SplitConformalRegressor - from mapie.classification import MapieClassifier as SplitConformalClassifier - ``` - -2. **Drift harness.** Add `tests/test_models/test_conformal_mapie_drift.py` - that loads `conformal_intervals_mondrian.parquet`, recomputes the - intervals against MAPIE 1.x with the same seed, and asserts max abs diff - ≤ `1e-6` per loan and coverage delta ≤ `5e-4` per Mondrian cell. - -3. **Run on a branch.** `feat/mapie-1.x-migration`. Re-run - `crpto.conformal.intervals` and `crpto.conformal.validation` outside - `main`. Compare `models/conformal_policy_status.json` before/after. - -4. **MRM sign-off.** If drift exceeds tolerance, the change is *not* a - refactor — it is a model change and needs a fresh run-tag. - -## Patch shape (reference) - -```python -# Before (MAPIE 0.9 style) -from mapie.regression import MapieRegressor -mapie = MapieRegressor(estimator, method="plus", cv=5) -mapie.fit(X_cal, y_cal) -y_pred, y_pis = mapie.predict(X_test, alpha=0.1) - -# After (MAPIE 1.x style) -from mapie.regression import SplitConformalRegressor -from mapie.conformity_scores import AbsoluteConformityScore -mapie = SplitConformalRegressor( - estimator=estimator, - conformity_score=AbsoluteConformityScore(), - confidence_level=0.9, -) -mapie.fit_conformalize(X_cal, y_cal) -y_pred, y_pis = mapie.predict_interval(X_test) -``` - -## Timing - -Pair with the conformal module refactor (see `CONFORMAL_REFACTOR_PLAN.md`). -Both touch `src/models/conformal.py` and benefit from a single drift -validation pass. diff --git a/docs/refactor/README.md b/docs/refactor/README.md deleted file mode 100644 index b8d70ab..0000000 --- a/docs/refactor/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# `docs/refactor/` - -Refactor plans, drift reports and execution memory for code paths that touch -the frozen paper pipeline. - -> The consolidated 2026-06 backlog was executed and archived: -> [`archive/NEXT_WORK_PLAN_2026-06.md`](archive/NEXT_WORK_PLAN_2026-06.md). -> The 2026-07-05 audit lanes (F1–F5) are closed; see -> [`docs/research/crpto_full_audit_2026-07-05.md`](../research/crpto_full_audit_2026-07-05.md) -> for the execution record and the post-submission backlog. - -Each plan documents: - -1. Why the change is desirable. -2. Why it was deferred or gated. -3. Acceptance criteria for execution. -4. Patch shape / file layout where useful. - -| Plan | Touches the champion? | Current status | -| --- | --- | --- | -| [`CONFORMAL_REFACTOR_PLAN.md`](CONFORMAL_REFACTOR_PLAN.md) | Yes (calibrator pickle) | Full public split executed 2026-06-13; `src.models.conformal` is now a package facade with strict-typed submodules. | -| [`MAPIE_MIGRATION_PLAN.md`](MAPIE_MIGRATION_PLAN.md) | Yes (intervals parquet) | Runtime is already MAPIE 1.x and the drift report is green; protected reruns still require explicit approval. | -| [`archive/FEATURE_CONFIG_PARQUET_PLAN.md`](archive/FEATURE_CONFIG_PARQUET_PLAN.md) | Yes (downstream stages) | Executed 2026-06-13 and archived; `feature_config.pkl` retired from the live DVC DAG and manifest. | -| [`ijds_tooling_decisions_2026-07-09.md`](ijds_tooling_decisions_2026-07-09.md) | No (tooling/refactor only) | Final live contract: `uv`, Ruff, mypy, `ty`, pytest, `just`, DVC and hook validation; Pyrefly and Commitizen are not adopted. | - -Executed lanes now in `main`: - -- **R0 dead-code cleanup**: removed dead modules and consolidated policy - matching helpers while preserving frozen artifact hashes. -- **R3 type-check cleanup**: `src/` and live scripts now pass `just type-check` - with the gradual mypy profile. -- **R4/R5 tests and operations**: added contract tests and named operational - gates for drift/bounds workflows. -- **R1 entrypoint extraction**: `scripts/train_pd_model.py`, - `scripts/generate_conformal_intervals.py` and - `scripts/optimize_portfolio_tradeoff.py` now have focused helpers for - config/replay setup, feature/input preparation, conformal tuning - selection, trade-off grid/input preparation and PD calibration selection. - These were code-only refactors; tracked DVC dependency hashes were re-keyed - without re-running protected stages. -- **2026-06-13 A/B closure**: conformal module split, YAML-first - `feature_config` reader, explicit DVC split stage, `test_predictions` as - `crpto.pd.champion` out, and executable `params.yaml` sync checker. No - protected DVC stage was reproduced. -- **2026-06-13 A2 phase 4**: `crpto.data.features` now writes - `feature_config.yml` and `feature_config.parquet`, no longer writes - `feature_config.pkl`, and `EXTRACTION_MANIFEST.json` tracks the YAML/Parquet - contract. The downstream champion/conformal stages were re-keyed with - `dvc commit -f`; `just drift-gate` remained bit-exact. - -Pick a plan up only when: - -- A new run-tag is being prepared, or -- The plan's acceptance criteria can be met without disturbing the frozen - champion artifacts. diff --git a/docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md b/docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md deleted file mode 100644 index 708694d..0000000 --- a/docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md +++ /dev/null @@ -1,228 +0,0 @@ -# CRPTO sensitivity / robustness run — design spec - -**Branch:** `experiments/ijds-sensitivity-suite` -**Run tag:** `ijds-sensitivity-2026-06-14` (distinct from the frozen champion tag `ijds-rebaseline-2026-06-07`) -**Status:** DESIGN ONLY. No protected stage has been run. Nothing here overwrites a frozen artifact. -**Author note:** this spec was produced from the expert audit of theory/claims/artifacts. It enumerates the experiments that would require a dedicated branch + run tag and would strengthen a claim, metric, bound, or theorem, and specifies exactly how to run each one safely. - ---- - -## 0. Why a separate run tag at all - -The paper's headline (`$170,464.54`, `V=0.028875`, `Gamma_CP=0.187987`, `B_u=0.278393`, `45/45`) -is the frozen champion under `ijds-rebaseline-2026-06-07`, hash-pinned in -`EXTRACTION_MANIFEST.json`. Every experiment below produces *new* artifacts under the -new run tag and **never** overwrites the frozen ones. The champion remains the headline; -these are diagnostics that either confirm it or are reported as separate sensitivity -evidence. - -## 1. The governing design principle: keep the PD model frozen - -CatBoost training is **not bit-reproducible** (multi-threaded float reduction). Retraining -`pd_canonical.cbm` therefore drifts and would fail `just drift-gate` by construction — this -is the lesson of the april-lineage unification. **Inference with a frozen model is -deterministic**, and the conformal layer is deterministic given the PD scores and the -calibration set. - -This splits the suite into tiers: - -| Tier | Definition | Drift risk | Permission | -|---|---|---|---| -| **T0 — conformal-only** | Frozen `pd_canonical.cbm` + frozen calibrator; re-fit only the conformal quantiles and/or re-score the frozen funded set. No PD retrain, no 276k search. | None (deterministic) | Run under the run tag; restore-before-merge | -| **T1 — PD retrain** | Retrains the PD model (new seed, new temporal fold). | High (CatBoost non-reproducible) | Explicit per-run approval; drift documented, not gated | -| **T2 — re-search** | Re-runs the 276k `bound_exact_eval` policy search as a *new champion*. | Replaces the contribution | Do not, except as a deliberate v2 protocol | - -**Almost everything worth running is T0.** Only temporal re-folding and seed-stability are T1. - -## 2. Experiment catalogue - -### E1 — Group-weighted / localized conformal to control funded-set coverage (T0, highest theory value) -- **Claim it strengthens:** the central coverage claim. Today `V(0.01)=0.028875 > alpha=0.01` - (funded-set under-coverage), so the certificate holds only at `V <= sqrt(alpha)`. If a - group-weighted or localized split-conformal recalibration achieves `E[V] <= alpha` on the - funded set, the certificate would hold at the **nominal** `alpha` — a strictly stronger bound. -- **Method:** re-fit weighted/localized conformal quantiles (weights = funded-set exposure - weights, or grade/score-decile group weights) on the frozen calibration scores; re-derive - `u_i(alpha)`; re-score the **frozen** funded set; report `V`, `Gamma_CP`, funded coverage. - References already in bib: `barber2023beyond`, `guan2023localized`, `bhattacharyya2026groupweighted`, - `jonkers2024wcps`. -- **Touches:** conformal recalibration code only (frozen PD, frozen funded set). T0. -- **Outputs:** `reports/crpto/experiments/ijds-sensitivity-2026-06-14/E1_weighted_conformal.parquet`. -- **Report:** a supplement table — does weighted/localized CP close the `V > alpha` gap, and at - what width cost? If yes, the body can claim nominal-`alpha` coverage for the funded set. -- **Risk/compute:** low; minutes. **Highest payoff of the suite.** - -### E2 — Calibration-partition sensitivity (empirical Beta-band confirmation) (T0) -- **Claim it strengthens:** confirms empirically the analytical Beta band just added to the body - (the under-coverage is structural, not a draw artifact). -- **Method:** for `K=50` seeds, bootstrap/subsample the 237,584 calibration loans, re-fit the - Mondrian conformal quantiles, re-derive `u_i(alpha)`, re-score the **frozen** funded set; - collect the distribution of `V`, `Gamma_CP`, funded coverage. -- **Touches:** conformal fit on resampled calibration + re-score frozen funded set. Frozen PD. T0. -- **Outputs:** `.../E2_calibration_sensitivity.parquet` (K rows). -- **Report:** mean/sd/quantiles of `V` across calibration draws; compare to the Beta prediction - (sd ~ 0.0002 on the endpoint side). Expected: `V` band dominated by the test side, endpoints - near-invariant — i.e., the body's claim verified. -- **Risk/compute:** low; ~K conformal fits. - -### E3 — Rolling-origin / walk-forward temporal validation (T1, high value, high risk) -- **Claim it strengthens:** "the certificate is not specific to the Jan-2018 cutoff." Re-run the - certificate on K rolling temporal folds. -- **Method:** for each fold, re-split by origin date, **retrain PD**, re-fit conformal, re-optimize, - re-certify; report `V`/return/coverage per fold. -- **Touches:** `data.splits`, `pd.champion` (retrain — drift), conformal, optimization, bound_eval. **T1.** -- **Outputs:** `.../E3_walkforward//...`. -- **Report:** per-fold certificate stability. Drift across folds is expected and **reported as such** - (it is genuine temporal variation, not a reproducibility bug). -- **Risk/compute:** high; K full pipeline runs; PD drift must be reported, never gated. - -### E4 — PD seed-stability characterization (T1) -- **Claim it strengthens:** turns the CatBoost non-reproducibility from a hidden caveat into a - measured band, supporting the reproducibility section. -- **Method:** retrain PD with `K=10` seeds (same config); report AUC/Brier/ECE distribution and the - induced spread in the downstream certificate (`V`, return) when the rest of the chain is held fixed. -- **Touches:** `pd.champion` x K. **T1.** -- **Outputs:** `.../E4_pd_seed_stability.parquet`. -- **Report:** "the headline AUC 0.7139 sits in a [lo, hi] band over seeds; the certificate moves by - [x]" — a quantified reproducibility statement. -- **Risk/compute:** medium-high; K PD trainings. - -### E5 — Conformal method / partition ablation at alpha=0.01 (T0) -- **Claim it strengthens:** justifies the score-decile Mondrian choice (or finds a better one). -- **Method:** re-fit global split / grade-Mondrian / score-decile-Mondrian / CQR at `alpha=0.01` on - the frozen calibration; re-score the **frozen** funded set; compare `V`, funded coverage, width. -- **Touches:** conformal variants on frozen PD + frozen funded set. T0. -- **Outputs:** `.../E5_conformal_ablation.parquet`. -- **Report:** an ablation table; either confirms the choice or motivates a switch (overlaps E1). -- **Risk/compute:** low. - -### E6 — Per-alpha end-to-end certificate (T0 if re-scoring frozen funded set; T1-lite if re-optimizing) -- **Claim it strengthens:** the deep-tail / flat-`V` finding and the alpha-gamma figure, with a true - per-`alpha` recomputation rather than one optimization re-scored at several `alpha`. -- **Method:** for `alpha in {0.005, 0.01, 0.02, 0.05, 0.10}`, re-derive endpoints and re-score the - frozen funded set (T0); optionally also re-optimize per `alpha` (touches optimization, gated). -- **Outputs:** `.../E6_per_alpha.parquet`. -- **Report:** the genuine `V(alpha)` / `Gamma_CP(alpha)` curve; confirms the flat-`V` deep-tail story. -- **Risk/compute:** low (re-score) / medium (re-optimize). - -### E7 — Calibration-set-size sensitivity (T0) -- **Claim it strengthens:** the F2 sparse-cell reliability discussion at the tight `alpha=0.01`. -- **Method:** subsample calibration to `n in {25k, 50k, 100k, 237k}`, re-fit conformal, re-score the - frozen funded set; track per-cell endpoint reliability and `V`, especially in the smallest cells. -- **Touches:** conformal on subsampled calibration + frozen funded set. T0. -- **Outputs:** `.../E7_calibration_size.parquet`. -- **Report:** how endpoint reliability and `V` degrade as cells thin — empirical backing for F2. -- **Risk/compute:** low. - -## 3. Priority (value x tractability) - -1. **E1** — could upgrade the bound from `sqrt(alpha)` to nominal `alpha`. T0. Do first. -2. **E2** — empirically confirms the Beta band; directly answers the referee. T0. -3. **E5** — justifies the Mondrian choice. T0. -4. **E6** — rigorous alpha-sweep. T0 (re-score variant). -5. **E7** — F2 sparse-cell backing. T0. -6. **E4** — reproducibility band. T1. -7. **E3** — temporal robustness. T1, highest cost. - -The five T0 experiments (E1, E2, E5, E6, E7) carry essentially no drift risk and deliver most of -the value; the two T1 experiments (E3, E4) are the only ones that need per-run approval and explicit -drift reporting. - -## 4. Safe execution protocol (mandatory for every experiment) - -1. Work only on `experiments/ijds-sensitivity-suite` (or a child branch). -2. All outputs go under `reports/crpto/experiments/ijds-sensitivity-2026-06-14/` — never under the - frozen champion paths. -3. New experiment code lives under `scripts/experiments/` and loads the frozen - `pd_canonical.cbm` / `pd_canonical_calibrator.pkl` read-only; it never writes them. -4. For any T1 step: capture a drift report against the frozen champion and **restore before merge** - (`git restore data/processed/ models/ dvc.lock && dvc checkout`); `just validate-champion` must be - green post-restore to certify the champion was not contaminated. -5. Only **code + the sensitivity reports/tables** merge to `main`; regenerated frozen artifacts never do. -6. `just drift-gate` stays the tripwire: a RED on the champion chain means contamination — stop. - -## 5. NO-DO list - -- Do not run `crpto.portfolio.bound_exact_eval` as a *new champion* (T2 / 276k search). -- Do not overwrite any path in `EXTRACTION_MANIFEST.json`. -- Do not retrain PD and silently let the new numbers flow into the paper — T1 drift is reported, not promoted. -- Do not merge regenerated frozen artifacts to `main`. - -## 6. What each experiment could change in the paper - -- E1 success -> body upgrades "operative safety level `V <= sqrt(alpha)`" to "nominal `alpha`-coverage - on the funded set under group-weighted recalibration" (a materially stronger headline). -- E2 -> a one-line empirical confirmation of the Beta band ("across 50 calibration draws `V` stays in - [lo, hi], matching the analytical band"). -- E5/E6/E7 -> supplement ablation/sweep tables that pre-empt method-choice and sparse-cell questions. -- E3/E4 -> temporal-fold and seed-stability tables (reproducibility/robustness appendix). - ---- - -## 7. Execution log — 2026-06-14 - -**E1 / E6 resolved analytically from the frozen funded-set artifact (`crpto_tableA7_funded_set_loans.csv`), no protected-stage rerun, no reimplementation risk.** - -Sanity: `V(0.01)=0.028875` and `Gamma_CP(0.01)=0.187987` reproduce exactly from A7. - -Key finding (binary-outcome reading of `V`): of 341 funded loans, 19 defaulted. A -non-default cannot miss (`y=0 <= u`); a default misses unless its endpoint reached -`u=1`. The `alpha=0.01` endpoint capped 2 of 19 defaults at `u=1` (weight 0.004) and -left 17 uncapped (weight 0.028875 = `V`). So `V` = funded weighted default rate -(0.032875) minus endpoint-capped defaults. - -Consequence for **E1**: no calibration reweighting (group-weighted / localized / MDCP) -can drive `V` to nominal `alpha=0.01` without capping most funded defaults at `u=1`, -which voids `Gamma_CP` and the economics. The `sqrt(alpha)` level is the honest -guarantee. This is a *negative* result for E1 but a stronger, more honest paper -statement than a re-run would have produced. **Added to the body and supplement.** - -Consequence for **E6**: the stored `pd_high_90/95` are two-sided intervals, not -one-sided endpoints, so a faithful per-`alpha` one-sided curve would need a conformal -refit; the binary-outcome reading above already explains the flat-`V` behaviour -without it. Not pursued (reimplementation risk > marginal value). - -**E2 / E5 / E7** (calibration bootstrap, partition ablation, calibration size) would -each confirm the same structural conclusion via a conformal refit; given E1's -analytical result they are demoted to optional. **E3 / E4** (T1, PD retrain) remain the -only experiments that could add genuinely new information (temporal-fold and seed -stability) and still require explicit per-run approval + drift management. - -## 8. Execution log — E3/E4 run (2026-06-15) - -Ran under run tag `ijds-sensitivity-2026-06-14`, fully isolated output paths -(`models/experiments/ijds-sensitivity-2026-06-14/`), ~12-13 min per full PD training. -Three frozen artefacts leaked to default paths (decision_threshold.json, -decision_threshold_v2.json, pd_model_contract.json — git-tracked) plus -`data/processed/test_predictions.parquet` (DVC out); all restored via -`git restore` + `dvc checkout`. **Post-restore: validate-champion 8/8, drift-gate -4/4, dvc status clean — champion intact.** - -**E4 — PD seed stability** (champion config, varying `model.params.random_seed`): - -| seed | OOT AUC | Brier | ECE | -|---|---:|---:|---:| -| 42 | 0.71268 | 0.15459 | 0.00615 | -| 52 | 0.71227 | 0.15465 | 0.00583 | -| 62 | 0.71210 | 0.15470 | 0.00597 | - -AUC spread 0.00058 (<0.001); Brier 0.1546 +/- 0.0001. The CatBoost non-reproducibility -across seeds is negligible — confirms that distributing a frozen binary loses no -discrimination and that the bit-exact certified chain is the right reproducibility object. - -**E3 — walk-forward temporal validation** (expanding windows, 80k eval each): - -| window | fit rows | validation AUC | best_iter | -|---|---:|---:|---:| -| 1 | 200,000 | 0.7332 | 499 | -| 2 | 372,850 | 0.7330 | 491 | -| 3 | 545,700 | 0.7173 | 621 | - -Internal validation AUC stays in [0.717, 0.733]; the dip in the most recent window -foreshadows the harder post-2018 OOT regime (the OOT test AUC of 0.7127 is lower still, -consistent with the deliberately adversarial out-of-time design). - -**Verdict:** both confirm stability rather than overturning anything. E4 added to the -supplement reproducibility appendix; E3 noted as temporal-stability evidence. No body -change (page budget; lineage offset 0.712 vs headline 0.7139 kept out of the body to -avoid conflation). T1 experiments are now complete; no further protected runs planned. diff --git a/docs/refactor/archive/FEATURE_CONFIG_PARQUET_PLAN.md b/docs/refactor/archive/FEATURE_CONFIG_PARQUET_PLAN.md deleted file mode 100644 index 6072f5a..0000000 --- a/docs/refactor/archive/FEATURE_CONFIG_PARQUET_PLAN.md +++ /dev/null @@ -1,100 +0,0 @@ -# `feature_config.pkl` → Parquet + schema migration plan - -**Status**: Executed as of 2026-06-13. The live pipeline now writes -`data/processed/feature_config.yml` plus -`data/processed/feature_config.parquet`, and no longer writes or tracks -`feature_config.pkl`. The downstream champion/conformal stages were re-keyed -without re-running CatBoost; `just drift-gate` stayed bit-exact. - -## Context - -`data/processed/feature_config.pkl` used to be a Python pickle produced by -`scripts/materialize_feature_artifacts.py` and consumed by scripts that need -the feature contract (feature order, categorical features, challenger pools -and IV scores). Pickles have well-known portability and security drawbacks: - -- The class types and module paths are baked in. Any rename in `src/features` - breaks deserialisation. -- Reviewers cannot inspect the artefact without running Python. -- Pickles can execute arbitrary code on load, which trips automated - security scanners (Bandit's `B301`). - -## Target representation - -Two artefacts replaced the single pickle: - -1. **`data/processed/feature_config.yml`** — the canonical human-readable - mapping used by live consumers. -2. **`data/processed/feature_config.parquet`** — a long-form table with - columns `section`, `kind`, `ordinal`, `key`, `value_json`, validated by a - Pandera schema for reviewers and MRM inspection. - -A loader helper: - -```python -# src/features/feature_config_io.py -def load_feature_config( - repo_root: Path | None = None, -) -> FeatureConfig: - """Read YAML, Parquet, or an explicit legacy pickle audit path.""" -``` - -The project kept the existing `dict[str, Any]` API because all live consumers -already expect dictionaries and the object contains no fitted transformers. -That is simpler than introducing a dataclass solely for this academic repo. - -## Execution notes - -`scripts/materialize_feature_artifacts.py` is a stage in `dvc.yaml`: - -```yaml -crpto.data.features: - cmd: python scripts/materialize_feature_artifacts.py --config configs/crpto_pd_model.yaml - outs: - - data/processed/train_fe.parquet - - data/processed/test_fe.parquet - - data/processed/calibration_fe.parquet -``` - -Changing the output format required one re-run of `crpto.data.features`. The -feature Parquets remained byte-identical, `feature_config.yml` kept its -existing SHA256, and only `feature_config.parquet` plus the JSON serialization -of `feature_manifest_v2.json` changed. The champion and conformal stages were -not reproduced; their DVC dep hashes were accepted with `dvc commit -f` after -`just drift-gate` showed zero numerical drift. - -## Acceptance criteria - -1. **Dual-write phase.** Completed before phase 4: YAML was written next to - the pickle and consumers moved to YAML-first loading. - -2. **Reader migration.** Each consumer adds a fallback: - - ```python - try: - cfg = load_feature_config_parquet(...) - except FileNotFoundError: - cfg = joblib.load("data/processed/feature_config.pkl") - ``` - - Completed. Live consumers now point at `feature_config.yml`; explicit - `prefer="pickle"` remains only for legacy audits. - -3. **Round-trip test.** Completed. Tests cover YAML/Parquet equivalence on the - current frozen contract and retain a small explicit-pickle escape hatch. - -4. **Drop the pickle.** Completed. `feature_config.pkl` is no longer a DVC out, - no longer in `EXTRACTION_MANIFEST.json`, and is absent from the local - `data/processed` checkout after `dvc checkout`. - -## Risks - -- Historical search scripts may still write pickle snapshots inside their - external experiment bundles. That is separate from the live CRPTO feature - contract and remains outside the frozen champion DAG. -- The Parquet table stores JSON values with explicit `ordinal` fields so list - order is stable. - -## Timing - -Completed in the 2026-06-13 run-tag-approved cleanup window. diff --git a/docs/refactor/archive/NEXT_WORK_PLAN_2026-06.md b/docs/refactor/archive/NEXT_WORK_PLAN_2026-06.md deleted file mode 100644 index c89ecae..0000000 --- a/docs/refactor/archive/NEXT_WORK_PLAN_2026-06.md +++ /dev/null @@ -1,563 +0,0 @@ -# CRPTO — Plan de trabajo restante (2026-06-13) - -> **Para Codex.** Este documento es el backlog ejecutable consolidado tras -> cerrar R0–R5 (PRs #52–#66) y la doc de gates (#67). Está ordenado por -> relación **valor / riesgo** y agrupado en lanes. Cada ítem tiene: -> contexto, qué hacer, criterio de aceptación, gate de verificación y riesgo. -> Lee primero `docs/refactor/README.md` (estado de lanes) y `CLAUDE.md` -> (reglas de operación, especialmente la **regla 12: drift-gate**). - -## Estado base (verificado 2026-06-13, `main` @ `45345a7`) - -- `just lint`, `just type-check` (**0 errores mypy en 104 archivos**), - `just smoke`, `just validate-champion`, `just test`, `just dvc-status` → - todos verdes. -- `just drift-gate` (`CRPTO_RUN_CHAMPION_DRIFT=1`) → **4/4 verde**: la cadena - del certificado es bit-exacta reproducible. Esta es la red de seguridad. -- Cero dead code, cero dead config, cero duplicación trivial accionable. -- Paper: 24/25 páginas, teoría formal + menú de bounds A21 auditado, - figuras Type 0 (TrueType embebido), bibliografía completa. - -**Conclusión de la auditoría:** el código está en estado de submission. Lo -que queda son (a) dos lanes de refactor estructural deliberadamente gated, -(b) gaps de gobernanza del DAG que requieren champion-lock, (c) pulido -editorial del paper en ventanas de calendario, (d) un puñado de -simplificaciones de bajo valor que se documentan como **NO HACER** para -evitar churn inútil. - -## Ejecución Codex (2026-06-13) - -Este plan fue ejecutado en la rama -`codex/implement-next-work-lanes-2026-06` después de revisar la gobernanza, -los PRs #52--#67 y el estado actual de `main`. - -- **A1 cerrado.** `src.models.conformal` pasó de archivo monolítico a paquete - con fachada pública y submódulos `_scores.py`, `pd_intervals.py`, - `classification.py` y `regression.py`. Los imports legacy siguen siendo - `from src.models.conformal import ...`, `conformal_adapters.py` conserva el - `__module__ = "src.models.conformal"` y los módulos nuevos entraron a mypy - strict. -- **A2 fases 1--3 cerradas.** `load_feature_config(prefer="yaml")` lee YAML - con fallback pickle, los consumidores PD/conformal usan YAML-first de forma - explícita, y `tests/test_features/test_feature_config_equivalence.py` - compara el pkl/yml congelado. El pkl no se borra ni sale del manifest. -- **B1 cerrado como metadata-only.** Los splits quedaron bajo un stage DVC - explícito `crpto.data.splits`, `calibration.parquet` quedó declarado como - dep real de `crpto.data.features`, y `test_predictions.parquet` pasó a ser - out de `crpto.pd.champion`. Se eliminaron los `.dvc` standalone - correspondientes y se hizo `dvc commit -f` sobre artefactos existentes; no - se ejecutó `dvc repro` ni se regeneró ningún artefacto protegido. -- **B2 cerrado como contrato generado.** `scripts/build_params_view.py` y - `just params-check` reconstruyen la vista `params.yaml` desde configs y - `models/final_project_promotion.json`; el test - `tests/test_configs/test_params_view_generator.py` evita volver a un espejo - manual silenciosamente stale. -- **C1/C2 ya estaban cerrados en el paper.** `paper/CRPTO_ijds.qmd` y - `paper/submission/CRPTO_ijds_submission.tex` ya contienen la lectura de - Markov óptimo bajo Assumption 1 sola y la frase de A9 donde ambos slices - temporales pasan el exact check. La tabla vigente reporta - `selection_slice_2018` con `V=0.030475`, coverage `0.9550`, pass `True`; y - `confirmation_slice_2019_2020` con `V=0.082100`, coverage `0.9259`, pass - `True`. -- **Estado en PR #68:** C3/C4 seguían calendar/remote-gated. En la corrida - run-tag aprobada de esta sesión, C3 se ejecutó; C4 sigue prohibido. - -## Ejecución Codex (2026-06-13, run-tag aprobado) - -Carlos autorizó completar lo pendiente excepto freeze/submission. En la rama -`codex/run-tag-approved-cleanup-2026-06` se cerró **A2 fase 4**: - -- `crpto.data.features` se re-materializó una vez para retirar - `data/processed/feature_config.pkl` y producir - `data/processed/feature_config.yml` + `data/processed/feature_config.parquet`. -- `feature_config.yml` conservó su SHA256 exacto; los Parquets de features y - `feature_manifest_v2.parquet` también conservaron sus hashes DVC. -- `feature_manifest_v2.json` cambió sólo como serialización JSON generada por - el stage; se registró en `EXTRACTION_MANIFEST.json` junto con el nuevo - `feature_config.parquet`. -- Los consumidores vivos (`configs/crpto_pd_model.yaml`, `pd_model.py`, - `train_pd_model.py`, `generate_conformal_intervals.py`) apuntan a YAML. El - loader mantiene `prefer="pickle"` sólo como escape hatch explícito para - auditoría legacy. -- `crpto.pd.champion` y `crpto.conformal.intervals` **no se re-ejecutaron**: - sólo se re-keyearon sus deps con `dvc commit -f` después de confirmar - `just drift-gate` con diff `0.000e+00`. -- Hallazgo adicional ejecutado: el sandbox histórico de regret-auditability - dejó de leer el pickle directo y usa el helper YAML/Parquet (`prefer="auto"`) - si alguien lo corre en rama. - -Resultado local: `just validate-champion`, `just drift-gate`, -`just dvc-status` y los tests focales de feature-config quedaron verdes. - -**C3 ejecutado en la misma sesión:** primero -`uv run dvc push -j 1 data/processed/loan_master.parquet` respondió -`Everything is up to date`; luego `uv run dvc push -j 2` completó -correctamente y subió `1 file` (el nuevo artefacto faltante en el remote tras -A2 fase 4). - -## Ejecución Claude/Codex (2026-06-14, paper polish sin freeze) - -Claude cerró en PR #72 la **Proposición A.1**: bajo Assumption 1 sola, -Markov es el statement correcto y ningún argumento de segundo momento agnóstico -mejora el umbral del body. Codex tomó el siguiente paso natural de la nota de -Claude: - -- **A.2 cluster-aware formalizada en el supplement.** La sección A21 ahora - contiene una Proposición A.2 con prueba Hoeffding condicional: si los - agregados de clusters son independientes tras fijar calibración, partición y - allocation, entonces el bound depende de `sum_g W_g^2`. También cuantifica el - umbral de tightening (`sum_g W_g^2 < 0.0070` para `alpha=0.01`, - `delta=0.10`) y muestra por qué los clusters observados no aprietan Markov - (`0.2407`, `0.3572`, `0.0914`). Body y TEX quedaron sincronizados: A.1/A.2 - se leen como frontera explícita entre "sin estructura" y "con estructura". -- **Figuras 13/14 listas para B/N.** `scripts/build_crpto_journal_package.py` - ya no depende solo del color: Fig. 13 usa estilos de línea y marcadores - redundantes; Fig. 14 usa colormap secuencial de grises, texto dinámico - negro/blanco según luminancia y champion marker con contorno. Se regeneraron - los PNG/PDF en `reports/crpto/figures/` y - `book/assets/figures/publication/`; las vistas convertidas a escala de grises - fueron inspeccionadas y siguen legibles. -- **Reproducibility capitalizado.** La sección de reproducibilidad del QMD y el - TEX de submission mencionan que el champion feature contract vive como - YAML/Parquet en vez de pickle opaco; solo modelo y calibrador permanecen como - binarios. - -No se ejecutó freeze ni submission. No se reabrió búsqueda, HPO, champion, -intervalos conformal, validación conformal ni optimización portfolio. - -## Ejecución Codex (2026-06-14, puente teórico y QA visual) - -Claude dejó el commit `efb56d1` como reparación posterior a PR #73: el -supplement ahora presenta A.1 antes de A.2, de modo que la proposición de -optimalidad de Markov precede al tightening cluster-aware. Codex cerró los -pendientes editoriales derivados de esa auditoría: - -- **Puente teórico en body y TEX.** Theorem 1, Proposition A.1 y Proposition - A.2 ahora se leen como tríptico: garantía principal bajo weighted funded-set - validity, optimalidad first-moment sin estructura adicional y sensibilidad - cluster-aware bajo independencia cross-cluster explícita. -- **Defensa de temporalidad en A.2.** El supplement explica por qué la - partición period-grade es la defensa más razonable para un panel temporal: - separa cohortes de calendario mientras condiciona por grado; period-only - ignora mezcla de riesgo y grade-only cruza dependencia temporal. -- **Menos repetición y menos "AI slop".** Se comprimió el menú de bounds para - que la frase "tightening requiere assumptions adicionales" aparezca como - pricing de supuestos, no como eco defensivo. Abstract, introducción, - contribuciones, resultados y conclusión quedaron más naturales y con el - claim estrecho: certificado conformal-robust post-hoc, no nuevo learner ni - live deployment. -- **QA visual de submission.** Se regeneró `paper-submission`, se recompiló el - PDF IJDS y se inspeccionaron páginas clave en navegador local mediante vistas - PNG; en particular Fig. 13 y Fig. 14 siguen legibles en escala de grises, con - labels y colorbar distinguibles. - -No se ejecutó freeze ni submission. No se tocaron stages protegidos ni -artefactos congelados del champion. - -## Hallazgo post-merge (2026-06-14, Dependabot torch) - -GitHub reportó el alert Dependabot #13 sobre `torch` en `uv.lock` -(`GHSA-rrmf-rvhw-rf47`, CVE-2025-3000: `torch.jit.script`). La revisión local -dejó este estado: - -- `torch` vive en el extra opcional `spo` y también llega vía `pyepo`; no es - parte del champion, conformal, portfolio LP ni paper-export. -- El advisory no publica versión parcheada (`first_patched_version = null`, - rango vulnerable `<= 2.12.0`), así que no hay upgrade que cierre el alert. -- `rg` no encuentra uso de `torch.jit`, `jit.script` ni TorchScript en - `scripts/`, `src/`, `tests/`, `docs/`, `book/` o `configs/`. -- Intento de cerrar el alert como riesgo tolerado falló por permisos del PAT - (`403 Resource not accessible by personal access token`). - -**Acción vigente:** no tocar dependencias ni lockfile por este alert hasta que -PyTorch publique versión fija o hasta que TorchScript entre en un code path -real. Si aparece una versión parcheada, abrir PR pequeño que actualice -`torch`/`uv.lock`, corra `just setup --extra spo` o equivalente local, y -verifique los tests SPO (`tests/test_scripts/test_run_spo_real.py`) más los -gates universales. - ---- - -## AUDITORÍA POST-EJECUCIÓN (2026-06-13, Claude) — leer antes de continuar - -Audité los cuatro lanes ejecutados por Codex (PR #68, `c5d1d47`). **Todos los -gates pasan**: `validate-champion` (hashes congelados intactos), -`dvc-status` limpio, `drift-gate` **4/4 verde** (la cadena del certificado es -bit-exacta tras el split de conformal), `lint`, `type-check` (0 errores, 109 -archivos) y la suite completa. Veredicto por lane: - -| Lane | Calidad | Nota | -| --- | --- | --- | -| **A1** conformal split | Excelente | Fachada `conformal/__init__.py` re-exporta todo; `conformal_adapters.py` fuerza `__module__` para pickle compat (defensa correcta aunque el calibrador no referencie esas clases); drift-gate 4/4. | -| **A2** feature_config YAML | Buena | `prefer="yaml"` con fallback pickle; test de equivalencia pkl↔yml; pkl intacto en PR #68. **Superado por A2 fase 4:** el pkl ya fue retirado en la corrida run-tag aprobada. | -| **B1** DVC DAG | Correcto + **gap reparado** | metadata-only, hashes intactos. Ver abajo. | -| **B2** params view | Excelente | Generador `--check` verificable; mejor que el plan original (que difería esto). | - -### Gap de gobernanza de B1 — REPARADO por Claude (mismo PR de auditoría) - -B1 convirtió los splits congelados (antes `.dvc` standalone inmutables) en -**outputs regenerables** del nuevo stage `crpto.data.splits`. Esto es -técnicamente seguro (los cuatro artefactos siguen en `EXTRACTION_MANIFEST.json`, -así que `validate-champion` atrapa cualquier cambio de bytes), **pero el nuevo -stage quedó fuera de la deny-list** de `.claude/settings.json` — a diferencia -de los otros cinco stages protegidos. Un `dvc repro crpto.data.splits` (no -bloqueado) podría regenerar `train/test/calibration.parquet` y cascadear hasta -el champion; `prepare_dataset.py` es *mayormente* determinista -(`random_state=42` + sort estable) pero no hay garantía bit-exact de -parquet (orden de filas con misma fecha, metadata, compresión). - -**Reparación aplicada:** -1. `crpto.data.splits` añadido a la deny-list de `.claude/settings.json`. -2. Tabla "Qué stages son seguros re-correr" de `CLAUDE.md` actualizada: - `crpto.data.splits` marcado ❌ NO, y nota de que `crpto.pd.champion` ahora - produce `test_predictions.parquet`. - -### Nota de proceso (para Carlos) - -B1 y B2 estaban marcados originalmente como "requiere aprobación de Carlos". -Codex los ejecutó como parte de la corrida del plan; el resultado es seguro y -B2 es una mejora clara, así que no se revierten. **Con la actualización de -autorización de abajo (2026-06-13), esta clase de trabajo ya no necesita pedir -permiso caso por caso.** - ---- - -## AUTORIZACIÓN Y FILOSOFÍA DE TRABAJO (2026-06-13) — leer y respetar - -Carlos otorgó **aprobación de run-tag** a Claude y a Codex. Esto cambia el -alcance: **ya se pueden regenerar artefactos congelados** (re-correr stages -protegidos, crear un run-tag nuevo, completar A2 fase 4). La filosofía es -explícita: - -> "Todo se trabaja bajo ramas. Una vez se implementa y valida todo, se decide -> si se mantiene o se revierte. Los pasos se dejan una vez está claro que -> quedaron bien hechos; si no, se revierten. La idea siempre ha sido mejorar -> el paper al máximo (y si indirectamente mejora el libro Quarto, adelante). -> **Lo único que NO se hace todavía: freeze ni submission** — queda tiempo y -> seguiremos mejorando cosas con los pasos bien hechos." - -### Por qué un run-tag requería aprobación (y por qué ahora la tienes) - -Una rama de git versiona **código**. Pero los artefactos pesados del champion -(`pd_canonical.cbm`, los intervalos conformales, `test_predictions.parquet`) -viven en **DVC**, no en git, y sus bytes exactos están congelados en -`EXTRACTION_MANIFEST.json`. Regenerarlos (un "run-tag") es especial porque: - -1. **CatBoost no es bit-reproducible entre corridas** (lo descubrimos: tres - entrenamientos del mismo config dieron AUC 0.7124/0.7127/0.7139). Un re-run - produce un modelo *distinto al byte*, aunque el config sea idéntico. -2. **El certificado del paper depende de esos bytes exactos.** $170,464.54, - V=0.028875, Γ_CP=0.187987 salen de un funded set calculado sobre *esos* - intervalos. Si el modelo cambia, los números del paper cambian, y habría - que actualizar el manifest, las tablas, las figuras y el texto. -3. Por eso no es "revertible con `git revert`": toca artefactos DVC + la - identidad numérica del paper. La aprobación = autorización para mover esa - identidad de forma controlada, con un run-tag nuevo y todo re-derivado - coherentemente. - -### El contrato de "bien hecho vs. revertir" - -Cada cambio que regenere artefactos se hace **en rama** y se evalúa contra: - -- **`just drift-gate`** — recomputa la cadena del certificado desde los - binarios y exige diff bit-exacto. Si el run-tag es intencional (modelo - nuevo), este gate dará rojo *por diseño*; entonces el criterio pasa a: -- **Coherencia total del nuevo linaje**: el nuevo modelo → nuevos intervalos → - nuevo certificado → manifest actualizado → tablas/figuras/paper actualizados, - todo de una sola corrida, sin mezclar linajes (el error que arreglamos en - junio). El test `tests/test_configs/test_lineage_consistency.py` y - `validate-champion` deben quedar verdes contra el **nuevo** estado. -- **Mejora demostrable**: el run-tag solo se mantiene si mejora algo medible - (mejor calibración, narrativa más limpia, deuda eliminada). Si no mejora o - empeora un número del paper sin contrapartida, **se revierte**. - ---- - -## QUÉ HACER AHORA — instrucciones para Codex (2026-06-13, run-tag aprobado) - -El refactor de código (A1, A2 fases 1-4, B1, B2) está **cerrado y verificado**. -Con la aprobación de run-tag, esto es lo que puedes hacer ya, ordenado por -seguridad. **Todo en rama; cada paso se mantiene solo si queda bien hecho.** - -**Permitido y seguro (reversible con `git revert` + `dvc checkout`):** -1. Cualquier refactor de código adicional que respete LANE D (la lista - NO-HACER sigue vigente: no descomponer más los `main()`, no consolidar - `_safe_float`, no tocar los search scripts). Gate: `drift-gate` verde. -2. Regenerar tablas/figuras/evidencia del paper (stages `crpto.paper.*`, no - protegidos). Mejoras editoriales del paper y, si aplica, del libro Quarto. -3. Mejoras de bibliografía, narrativa, claims, supplement — siempre verificando - números contra artefactos (`test_lineage_consistency`). - -**Permitido con run-tag (regenera artefactos congelados; ver "cómo revertir"):** -4. **Re-run validado de la cadena** — ahora que el DAG está completo (B1), se - puede `dvc repro` de `crpto.data.splits → features → champion → conformal` - en rama para confirmar reproducibilidad (o documentar el drift esperado y - decidir si se promueve a run-tag nuevo). -5. Cualquier mejora que requiera re-entrenar/re-derivar, **siempre que el - linaje quede coherente de punta a punta en una sola corrida**. - -**Prohibido hasta nueva orden:** -6. **Freeze de submission y submission misma** (Lane C4). Queda tiempo; no se - congela ni se envía todavía. - -**Antes de cualquier commit**, correr el gate universal del final. Si tocaste -artefactos congelados con intención de run-tag, el drift-gate rojo es esperado -— entonces aplica el contrato "bien hecho vs. revertir" de arriba. - ---- - -## CÓMO REVERTIR SI UN RE-RUN SALE MAL - -El estado *known-good* es `main` con el run-tag congelado -`ijds-rebaseline-2026-06-07` (manifest actual). Si un run-tag nuevo sale mal o -no se quiere mantener: - -1. **Si está solo en rama (no mergeado):** `git checkout main` descarta el - código; `dvc checkout` restaura los artefactos del cache al estado de - `dvc.lock` de main. Fin — el champion vuelve intacto. -2. **Si el cache local se ensució:** `git checkout main && dvc checkout`; - si algún artefacto no está en cache, `dvc pull` lo trae del remote (que - conserva los bytes congelados de `ijds-rebaseline-2026-06-07`). -3. **Punto de no retorno:** solo si se mergea a main **y** se hace `dvc push` - del nuevo run-tag sobreescribiendo el remote. Por eso: **no hacer `dvc push` - de un run-tag nuevo hasta que esté validado y aprobado para mantenerse.** - El remote es el último respaldo del linaje de abril/junio. - -Regla práctica: trabaja el run-tag en rama, valida coherencia completa, y solo -entonces decide mergear + push. Mientras viva en rama, es 100% reversible. - ---- - -## LANE A — Refactor estructural seguro (desbloqueado por hallazgo nuevo) - -### A1. Split de `src/models/conformal.py` (731 LOC → submódulos) - -**Estado 2026-06-13:** ejecutado. La fachada ahora es el paquete -`src/models/conformal/__init__.py`; `src/models/conformal.py` fue retirado. - -**HALLAZGO QUE DESBLOQUEA ESTE LANE (2026-06-13):** el plan -`CONFORMAL_REFACTOR_PLAN.md` asumía que el calibrador congelado -"very likely references `src.models.conformal.ProbabilityRegressor`". **Es -falso.** Inspeccioné `models/pd_canonical_calibrator.pkl` con `pickletools` -y cargándolo: su grafo de objetos solo contiene -`src.models.venn_abers.VennAbersScoreCalibrator` y la clase upstream -`venn_abers.venn_abers.VennAbers`. **El split de `conformal.py` NO afecta el -pickle.** El único invariante es: **no mover `src/models/venn_abers.py` ni -cambiar su `__module__`** (no hay razón para tocarlo). - -Comando de verificación del hallazgo (reproducible): -```powershell -uv run python -c "import pickle,pathlib; o=pickle.loads(pathlib.Path('models/pd_canonical_calibrator.pkl').read_bytes()); print(type(o).__module__+'.'+type(o).__qualname__)" -# => src.models.venn_abers.VennAbersScoreCalibrator -``` - -**Qué hacer.** Ejecutar la "Proposed structure" de -`CONFORMAL_REFACTOR_PLAN.md`: partir `conformal.py` en submódulos enfocados -con una fachada `conformal.py` que re-exporta todo para compat. Estructura -sugerida (ajustar a lo que ya existe — `conformal_adapters.py`, -`conformal_diagnostics.py`, `conformal_artifacts.py` ya están separados): -- `_scores.py` — `_conformal_quantile`, `_resolve_score_scale_family`, - `_compute_score_scale`. -- `pd_intervals.py` — `create_pd_intervals`, - `create_pd_intervals_mondrian`, `create_pd_intervals_venn_abers`, - `conditional_coverage_by_group`, `apply_probability_calibrator`, - `build_mondrian_partition_labels`. -- `regression.py` / `classification.py` — el resto. -- `conformal.py` — fachada que re-exporta todos los símbolos públicos. - -**Criterio de aceptación:** -1. Todos los call-sites siguen importando desde `src.models.conformal` sin - cambios (la fachada los cubre). -2. `tests/test_models/test_calibrator_pickle_compat.py` verde (prueba que - el pickle sigue cargando). -3. **`just drift-gate` VERDE** — la prueba dura de que ni un bit del cálculo - cambió. -4. `just type-check` verde; promover `src.models.conformal` y los nuevos - submódulos a la lista strict de `pyproject.toml` (junto a los que ya - están; ver el comentario "Pending promotions" que dejé ahí). - -**Gate de verificación:** -```powershell -just lint; just type-check; just smoke; just validate-champion -just drift-gate # OBLIGATORIO — esta es la prueba real -just dvc-status -``` - -**Riesgo:** Bajo-medio. El split es mecánico (mover funciones puras + fachada). -El drift-gate atrapa cualquier perturbación numérica al bit. NO se re-corre -ningún stage protegido. Hacer en rama dedicada, un PR. - -**Por qué vale la pena:** `conformal.py` es el módulo más denso del núcleo y -el más citado en la tesis; un split limpio lo hace mantenible para la -defensa y deja el último módulo grande de `src/` bajo strict mypy. - ---- - -### A2. `feature_config.pkl` → Parquet/YAML (retirar el pickle) - -**Estado 2026-06-13:** fases 1--4 ejecutadas. `feature_config.pkl` fue retirado -del DAG vivo y del manifest; `feature_config.yml` + `feature_config.parquet` -son ahora el contrato activo. - -**Estado actual (verificado):** `src/features/feature_config_io.py` lee YAML -estricto por defecto, puede leer Parquet explícitamente, y conserva pickle sólo -como escape hatch de auditoría legacy. `materialize_feature_artifacts.py` -escribe YAML/Parquet y ya no escribe pkl. Los consumidores vivos -(`feature_engineering.py`, `pd_model.py`, `generate_conformal_intervals.py`, -`train_pd_model.py`) pasan por `load_feature_config` y apuntan a YAML. - -**Qué hacer (fases, según `FEATURE_CONFIG_PARQUET_PLAN.md`):** -1. Cerrado: equivalencia pkl↔yml en PR #68 y equivalencia YAML↔Parquet en la - fase 4. -2. Cerrado: consumidores vivos leen YAML por defecto. -3. Cerrado: `just drift-gate` verde con diff bit-exacto `0.000e+00`. -4. Cerrado: `feature_config.pkl` salió de `dvc.yaml`, `dvc.lock`, - `EXTRACTION_MANIFEST.json` y del checkout local; `feature_config.parquet` - quedó registrado con hash propio. - -**Criterio de aceptación:** cumplido. Consumidores leen YAML por defecto, -equivalencia YAML/Parquet testeada, drift-gate verde, manifest consistente con -lo que queda en disco. - -**Riesgo:** Medio. Toca la ruta de features que alimenta el champion. El -drift-gate es el juez. Rama dedicada. **Requiere `dvc commit` con patch -mínimo** si cambia algún hash de dep (ver procedimiento en -`docs/refactor/README.md` y el helper de patch mínimo de `dvc.lock`). - -**Por qué vale la pena:** elimina la última dependencia de pickle en la ruta -de datos (Bandit B301, portabilidad, inspeccionabilidad para reviewers MRM). - ---- - -## LANE B — Gaps de gobernanza del DAG - -> Ejecutado el 2026-06-13 como cambio metadata-only: `dvc commit -f` aceptó -> artefactos existentes y `uv run dvc status --no-updates` quedó limpio. No se -> ejecutó ningún stage protegido. - -### B1. Promover splits y `test_predictions.parquet` a stage outputs -`data/processed/{train,test,calibration}.parquet` (producidos por -`src/data/prepare_dataset.py`) y `test_predictions.parquet` (exportado por -`train_pd_model.py`) son artefactos `.dvc` standalone, no outputs de stage. -La implementación correcta fue: - -- `crpto.data.splits` produce `train.parquet`, `test.parquet` y - `calibration.parquet` desde `loan_master.parquet`. -- `crpto.data.features` declara los tres splits como deps. -- `crpto.pd.champion` declara `test_predictions.parquet` como out. -- Los `.dvc` standalone de esos cuatro artefactos se retiraron. - -### B2. Unificación estructural de `params.yaml` -`params.yaml` es espejo documental de `configs/crpto_*.yaml` (el valor stale -de learning_rate ya se corrigió). La unificación real (generar `params.yaml` -desde configs o viceversa) elimina el riesgo de desincronización pero toca -las `params:` deps de stages protegidos. El cierre ejecutado fue un generador -no destructivo: `scripts/build_params_view.py --check` reconstruye el YAML -desde configs/promotion y falla si el tracked `params.yaml` queda stale. - ---- - -## LANE C — Paper IJDS (ventanas de calendario, ver `IJDS_SUBMISSION_ROADMAP_2026-08-10.md`) - -El roadmap del paper tiene 15 tracks y ventanas semanales hasta Ago 10. -Lo que la auditoría de código confirma como **ya cerrado**: teoría formal, -menú de bounds A21 (incluida la fila agnóstica que prueba Markov óptimo), -figuras Type 0, bibliografía completa, números de un solo linaje verificado -al bit. Lo que queda es editorial puro: - -### C1. Capitalizar el resultado del bound agnóstico en el body (opcional) - -**Estado 2026-06-13:** ya cerrado en `paper/CRPTO_ijds.qmd` y -`paper/submission/CRPTO_ijds_submission.tex`. - -El Remark 1 dice que los tightenings existen "whenever the additional -assumption holds". El supplement A21c ahora prueba lo **inverso y más fuerte**: -bajo Assumption 1 sola, ningún tightening de segundo momento mejora Markov -(Cantelli agnóstico = 0.3085 > Markov 0.1000). Vale media frase en el body -("the supplement shows no second-moment tightening exists under Assumption 1 -alone, so Markov is optimal for the stated guarantee"). **Restricción de -presupuesto: el paper está en 24/25 páginas — compensar cualquier adición.** -Re-sync QMD↔TEX y recompilar (`just paper-ijds`, `latexmk`). Ventana: Jul 3–10 -(theorem audit). - -### C2. Narrativa A9 (claim hardening, Jun 17–24) - -**Estado 2026-06-13:** ya cerrado y verificado contra -`reports/crpto/tables/crpto_tableA9_strict_temporal_holdout.csv`. - -Tras la unificación de linaje, la tabla A9 muestra que **ambos slices -temporales pasan el exact check** (antes el de confirmación fallaba). La §7 -(Robustness) puede fortalecer la frase de validación temporal. Verificar el -número actual en `crpto_tableA9_strict_temporal_holdout.csv` antes de redactar. - -### C3. Reproducibility package + `dvc push` (Jul 25–31) -**Estado 2026-06-13:** ejecutado. El push focal a -`data/processed/loan_master.parquet` confirmó que el objeto grande ya estaba -sincronizado; `uv run dvc push -j 2` terminó OK y subió `1 file`. -El cover letter ya puede citar el drift harness por nombre como evidencia -ejecutable. - -### C4. Freeze de submission — **PROHIBIDO hasta nueva orden** (no es Ago 6-10) -QA doble-anonimato (`SCHOLARONE_FINAL_CHECKLIST.md`), recompilación final -`informs4`, conteo de páginas ≤25, metadata sin autor. **No ejecutar.** Carlos -fue explícito: queda tiempo y se seguirá mejorando; el freeze/submission es lo -único vedado por ahora. Todo lo demás del paper (mejorar narrativa, claims, -supplement, libro Quarto) **sí** se puede trabajar. - ---- - -## LANE D — NO HACER (no aporta, o cuesta más de lo que vale) - -Distinto de lo PROHIBIDO (freeze/submission): esto simplemente **no mejora el -proyecto**, así que no gastes ciclos en ello. Si más adelante hay una razón -nueva, reabrir con justificación. - -1. **Consolidar `_safe_float`** (aparece en 4 scripts). Verifiqué las 4 - implementaciones: **tienen semántica distinta** — - `analyze_crpto_evidence` retorna `float | None` y chequea `pd.NA`/`nan`; - `generate_governance_status._safe_float_value` usa default 0.0; - `run_comparison` y `validate_conformal_policy` usan default `nan` con - except distinto. NO son duplicación real; consolidarlas cambiaría - comportamiento. Sin valor. - -2. **Descomponer más los `main()` de 917/925 LOC** en - `train_pd_model.py` / `generate_conformal_intervals.py`. Lo que queda es - el **núcleo numérico irreducible** del pipeline. R1 ya extrajo todo lo - estructural (config/replay, splits, tuning selection). Más extracción ahí - es bajo valor (se lee como receta top-down) — y aunque ahora se puede - validar con drift-gate, el retorno de mantenibilidad es marginal. Solo - hacerlo si una mejora concreta lo exige. - -3. **Simplificar la duplicación interna de `_build_optuna_sampler_pruner`** - (los branches `else` duplican `tpe`/`median`). Es código del path de - entrenamiento; el valor es estético. Bajo valor. - -4. **Reescribir los search scripts** (`run_regret_auditability_sandbox.py` - 2342 LOC, etc.). Son evidencia histórica congelada con tests, no código - vivo. Refactor = riesgo sin retorno. - -5. **Mover los helpers extraídos en R1 a `src/`**. Son orquestación de un - solo uso. Vivir dentro del script es correcto para single-author - orchestration; moverlos a `src/` añade indirección sin reuso real. - ---- - -## Secuencia recomendada - -1. **Cerrado y verificado (2026-06-13):** A1, A2 fases 1--4, B1, B2. -2. **Ejecutable ya, en rama, con run-tag aprobado:** mejoras editoriales del - paper/libro y cualquier re-derivación coherente. Ver "QUÉ HACER AHORA". -3. **Prohibido hasta nueva orden:** freeze de submission y submission (C4). - -## Gate universal (correr tras cada PR; obligatorio antes de mergear) - -```powershell -just lint -just type-check -just smoke -just validate-champion -just drift-gate # el juez de la cadena del certificado -just dvc-status -``` -Si cualquiera da rojo en una lane que NO debía tocar el champion → es un -cambio numérico disfrazado de refactor: revertir y reportar. diff --git a/docs/refactor/drift_report_mapie_2026-06.md b/docs/refactor/drift_report_mapie_2026-06.md deleted file mode 100644 index f603258..0000000 --- a/docs/refactor/drift_report_mapie_2026-06.md +++ /dev/null @@ -1,136 +0,0 @@ -# Drift report — revalidación conformal MAPIE 1.x (Track B, gate B1) - -**Fecha**: 2026-06-09 (gate rojo) / **2026-06-10 (RESUELTO: gate VERDE)** -**Harness**: `tests/test_models/test_conformal_mapie_drift.py` (opt-in vía -`CRPTO_RUN_CHAMPION_DRIFT=1`) - -## RESOLUCIÓN 2026-06-10 — gate VERDE con drift CERO - -El binario de abril (`models/search_pd/pd-hpo-local-2026-04-03-1325/`, -modelo + calibrador + baselines) fue recuperado y el harness, apuntando a la -receta literal del pkl congelado, dio **drift 0.000e+00 en todas las -columnas** (y_pred, endpoints 90/95, edges de bandas, cobertura por celda) y -reprodujo exactamente los multiplicadores de piso `{score_q01: 1.05, -score_q04: 1.02}`. La cadena conformal congelada es **bit-exact -reproducible bajo el stack actual** (MAPIE 1.4 runtime, numpy/catboost/ -sklearn de hoy); la migración MAPIE queda revalidada de facto. - -Con aprobación explícita de Carlos se ejecutó la **unificación de linaje** -(camino 1 + corrección de identidad): `models/pd_canonical.cbm` y -`models/pd_canonical_calibrator.pkl` son ahora copias byte a byte del -candidato de abril, y `data/processed/test_predictions.parquet` se -reconstruyó desde ese bundle con -`scripts/rebuild_test_predictions_from_frozen.py` (assert duro: pd_calibrated -== y_pred congelado, diff máx 0.0). Métricas PD del paper actualizadas: -AUC 0.7127→0.7139, Brier 0.1546→0.1544, ECE 0.0062→0.0070; el certificado -($170,464.54, V=0.028875, Γ_CP=0.187987, 45/45) **no cambia**. Manifest: -bloque `april_lineage_unification` + 14 hashes re-freezados. - -Hallazgo adicional documentado durante la unificación: la "identidad -candidato↔canónico" **nunca existió** (el pd_canonical de abril en el -proyecto de origen ya era un tercer re-entrenamiento, AUC 0.7124; CatBoost -con el mismo config no es bit-reproducible entre corridas). Además, las -tablas A7/A8 (funded set por préstamo) provienen de un re-solve LP -degenerado y **no se regeneran**: quedan congeladas como la vista oficial -del certificado (341 préstamos, Tabla 7 del paper). A5/A9/A10 (experimentos -derivados de re-solve, independientes del linaje PD) se re-freezaron bajo el -stack lockeado actual para que un revisor pueda regenerarlas. - ---- - -## Registro histórico del gate rojo (2026-06-09) - -**Resultado del gate**: **ROJO — STOP Track B** según el criterio de -`MAPIE_MIGRATION_PLAN.md` ("si el drift excede tolerancia, es un cambio de -modelo, no un refactor"). - -## Qué se midió - -El harness recomputa en memoria los intervalos conformal del champion desde -la receta congelada en `models/conformal_results_mondrian.pkl` (partición -`score_decile_mondrian`, prob source raw, `n_score_bins=5`, fallback -`grade_then_global`, `alpha_90=0.1`, `alpha_95=0.05`, escala -`bernoulli_sqrt`, `min_group_size=100`, split de calibración 80/20 temporal -con seed 42, multiplicadores de piso `{score_q01: 1.05, score_q04: 1.02}`) y -compara contra `data/processed/conformal_intervals_mondrian.parquet`. - -Tolerancias del plan: `1e-6` por préstamo en endpoints; `5e-4` de cobertura -por celda Mondrian. - -## Resultado - -| Columna | max abs diff | Tolerancia | Estado | -| --- | --- | --- | --- | -| `y_pred` | 2.107e-01 | 1e-6 | FALLA | -| `pd_low_90` | 4.007e-01 | 1e-6 | FALLA | -| `pd_high_90` | 8.244e-01 | 1e-6 | FALLA | -| `pd_low_95` | 4.043e-01 | 1e-6 | FALLA | -| `pd_high_95` | 2.921e-01 | 1e-6 | FALLA | -| edges de bandas de score | 5.805e-03 | 1e-6 | FALLA | -| cobertura por celda (peor: `score_q02`) | 2.728e-02 | 5e-4 | FALLA | -| etiquetas de partición drifteadas | 30,961 / 276,869 | 0 | FALLA | -| multiplicadores de piso re-aprendidos | `{q00:1.05, q01:1.05, q02:1.08, q03:1.05, q04:1.02}` vs congelado `{q01:1.05, q04:1.02}` | igualdad | FALLA | - -Checks estructurales que SÍ pasan: tamaños del split de calibración -fit/holdout exactos vs el pkl; 5 grupos sin fallback; `id` y `y_true` -posicionalmente idénticos al parquet congelado (misma data, mismo orden). - -## Atribución de causa raíz - -El drift **no proviene de MAPIE ni de la capa conformal** (la ruta Mondrian -del champion usa cuantiles numpy puros, sin clases MAPIE). La causa está -aguas arriba, en la identidad del modelo: - -- El pkl congelado registra `model_path = - models/search_pd/pd-hpo-local-2026-04-03-1325/pd_candidate_model.cbm`: los - intervalos congelados se generaron en abril con el candidato del search - (no presente en el checkout local). -- `models/pd_canonical.cbm` es el re-entrenamiento de la rebaseline - `ijds-rebaseline-2026-06-07` con el mismo config (`crpto_pd_model.yaml`, - trial 56). El re-entrenamiento no fue bit-exact respecto del candidato de - abril: correlación de `y_pred` 0.9917, mediana de |diff| 9.5e-3, p99 - 5.9e-2, max 2.1e-1. -- Con predicciones distintas, los edges de deciles, las etiquetas de - partición (11.2 % de filas cambian de celda), los cuantiles por celda y - los multiplicadores de piso re-aprendidos difieren en cascada. - -## Implicaciones - -1. **La cadena computacional end-to-end no es reproducible** desde los - artefactos canónicos: `pd_canonical.cbm` (junio) no regenera - `conformal_intervals_mondrian.parquet` (abril). El diseño replay-mode de - `crpto.conformal.intervals` es lo que mantiene la consistencia del - pipeline: restaura bytes, no recomputa. -2. **El claim de reproducibilidad del paper se sostiene tal como está - redactado** ("paper-facing reruns consume frozen artifacts"), pero un - revisor que intente recomputar los intervalos desde el modelo canónico - obtendrá números distintos. Conviene que el reproducibility package sea - explícito en que la regeneración es por replay de artefactos congelados. -3. **Misma familia de causa raíz que el drift de hardening** detectado el - 2026-06-09 en `crpto_tableA5/A7–A10` (re-solves con el stack y modelo - actuales vs tablas committeadas pre-rebaseline): la rebaseline de junio - re-entrenó el PD canónico sin regenerar todas las superficies derivadas. - -## Decisiones según el plan - -- **B2 (migración MAPIE + split de `conformal.py`)**: NO se ejecuta. Sin un - gate verde no hay forma de demostrar que el split/migración preserva la - cadena congelada. -- **B3 (feature_config → Parquet)**: NO se ejecuta en esta ventana (el - track completo queda detenido; es aditivo pero comparte la precondición - de un gate verde para validar consumidores). -- El harness queda en el repo como gate permanente (opt-in por variable de - entorno) para la próxima ventana de re-promoción. - -## Caminos de resolución (decisión de Carlos, post-submission o run-tag nuevo) - -1. **Restaurar el binario de abril**: `dvc pull` (o recuperar del remote) de - `models/search_pd/pd-hpo-local-2026-04-03-1325/` y apuntar el harness al - `model_path` del pkl. Si el gate pasa con el candidato de abril, la - receta es íntegra y solo la identidad canonical↔candidato queda como - nota de gobernanza. -2. **Re-promoción completa**: nuevo run-tag que re-entrene PD, regenere - intervalos, re-valide policy y re-freezee manifest + tablas derivadas - (incluye resolver el drift de hardening A5/A7–A10 de una vez). -3. **Documentar y seguir**: mantener replay-mode como única vía oficial de - reproducción y reflejarlo en el data/code disclosure del journal. diff --git a/docs/refactor/ijds_tooling_decisions_2026-07-09.md b/docs/refactor/ijds_tooling_decisions_2026-07-09.md deleted file mode 100644 index c703407..0000000 --- a/docs/refactor/ijds_tooling_decisions_2026-07-09.md +++ /dev/null @@ -1,127 +0,0 @@ -# IJDS Tooling and Refactor Decisions - 2026-07-09 - -This is the final decision record for the IJDS code and manuscript workflow. -It replaces the iterative tooling lab from 2026-07-08, whose useful decisions -have been implemented. Scientific history remains in `docs/research/`; this -file describes only the live engineering contract. - -## Objective - -Keep one auditable route from frozen upstream artifacts to the submitted -claim. Prefer small, explicit modules and named commands over additional -frameworks, parallel implementations, or hidden manuscript-time computation. - -## Adopted tools - -| Tool | Role | Decision | -|---|---|---| -| `uv` | Python environment, lockfile, commands | Sole Python package/runtime interface. | -| Ruff | lint and formatting | Sole linter and formatter. | -| mypy | stable gradual type gate | Required by `just type-check`. | -| ty | fast independent type audit | Active and full scopes; full scope blocks submission closeout. | -| pytest | behavioral and claim-sync tests | Required for focused, smoke, and full suites. | -| just | named local workflow | Sole human-facing command menu. | -| DVC | frozen artifact lineage | Keep for scientific provenance; never use it as a general task runner. | -| pre-commit + prek | hook compatibility and fast config validation | Keep both checks; do not create a second hook policy. | -| pdoc | optional local API browsing | Ephemeral via `uv run --with pdoc`; no project dependency. | - -`ty` complements rather than replaces mypy. The active scope includes the -exact-alpha replay and calibration-selected policy modules. Its full clean -scope is useful as an independent submission check, while mypy remains the -stable repository contract. - -## Rejected additions - -| Tool | Decision | Reason | -|---|---|---| -| Pyrefly | Do not adopt | It duplicated type checking and produced substantially more migration noise than actionable signal. | -| Commitizen | Do not adopt | Commit-message automation does not improve scientific validity or the one-author release flow. | -| Permanent pdoc dependency | Do not adopt | Generated API pages are useful locally but are not a publication artifact. | -| A second task runner | Do not adopt | `just` already exposes the complete Windows-first workflow. | -| Automatic semantic versioning | Do not adopt | Run tags, Git commits, and evidence hashes are the relevant scientific identifiers. | - -## Live methodology path - -1. `src/models/conformal_alpha_grid.py` exactly replays the frozen 90% - intervals and reports the alpha sensitivity. -2. `src/optimization/policy_evaluation.py` uses point PD in the economic - objective and an effective PD only in the risk constraint. -3. `src/optimization/policy_selection.py` defines the nine-cell round-number - grid, deterministic endpoint screen, and cap-stability interval. -4. `scripts/experiments/ijds_policy_support.py` aligns candidate and exact-alpha - rows directly by ID and owns shared solving/evaluation. -5. `scripts/build_ijds_calibration_selected_evidence.py` materializes A35-A40 - from versioned experiment outputs. Manuscript rendering does not solve or - retune portfolios. - -The active run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. -The exact-alpha run is -`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. - -## Deliberate simplifications - -- One active policy family: `q=(p+u)/2`, `tau=0.17`, `gamma=0.50`. -- Point PD remains the economic objective; uncertainty is a feasibility - guardrail. -- One deterministic 3x3 November selector, with outcomes isolated from a - 12-column input frame, `B_u<=0.28`, and an outcome-free December replay. -- One independent December decision audit that records the funded-set coverage - miss instead of converting it into an unsupported guarantee. -- Month-cluster bootstrap is primary; loan-level resampling is a sensitivity. -- One A35-A40 active evidence bundle. -- One body, one supplement, and one official submission TeX source. -- No nested temporal selector, effective-PD objective branch, active cap/tail - variants, or manuscript-time optimizer. -- Historical A1-A34 tables remain diagnostics and provenance, not competing - active claims. - -## Named commands - -```powershell -just ijds-evidence -just ijds-active-replay -just lint -just type-check -just type-advisory-full -just smoke -just validate-champion -just drift-gate -just test -just submission-check -``` - -`just submission-check` is the ordinary release gate and includes the full -pytest suite. `just -ijds-active-replay` is intentionally separate because it recomputes exact -interval grids and solves experiment portfolios. - -## Compilation contract - -The official build uses `latexmk`. On Windows it resolves TinyTeX's -`latexmk.pl` and launches it with Perl, bypassing the defective -`runscript.tlu` executable wrapper. If that payload is unavailable or fails, -the robust fallback is: - -```text -pdflatex -> bibtex -> pdflatex -> pdflatex -``` - -The first pass writes citation keys to `.aux`, BibTeX writes `.bbl`, and the -last two passes stabilize citations, cross-references, and pagination. This is -one compilation workflow, not three independent builds. - -## Drift policy - -Experimental drift is allowed only under a new run tag. Promotion requires: - -- no writes to manifest-listed or protected champion artifacts; -- an explicit scientific reason and comparator; -- claim-sync and publication-integrity tests; -- `just validate-champion`; -- `just drift-gate` when PD or conformal paths change; -- regenerated tables and a visually inspected PDF. - -Light drift is not itself a benefit. A challenger is promoted only when it -improves the submitted method or its defensibility enough to justify the extra -surface area. Otherwise it is removed from the live path. diff --git a/docs/research/README.md b/docs/research/README.md index 567aa29..6e94b54 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -1,97 +1,53 @@ -# paper-crpto — Research Dossier - -Notas de investigación y registros que aún tienen valor de consulta para el -paper y la tesis. Las auditorías, backlogs y checklists puntuales fechados de -mayo-junio 2026 se retiraron el 2026-06-13 (su conocimiento ya está aplicado -en el paper, el código y el manifest; ver `CHANGELOG.md`). Lo que queda es lo -perenne y lo que el código lee/escribe. - -## Registros activos (referenciados por el código o el paper) - -- `active_claims_2026-07-04.md` — source-of-truth operativo del claim IJDS - activo: replay conformal exacto al 90%, selector de calibración 3x3, - política lineal 50/50, evidencia A35--A40 y stop rules. -- `ijds_exact_alpha_calibration_selection_2026-07-09.md` — closeout que - documenta por qué se retiró el alpha-0.01 aproximado y cómo se eligió la - política simple sin outcomes OOT en el selector final. -- `crpto_p1_evidence_2026-05-04.md` — evidencia P1 alrededor del champion - congelado (escrito por `scripts/analyze_crpto_evidence.py`). -- `crpto_journal_package_2026-05-04.md` — tablas A12–A34 y figuras journal - (escrito por `scripts/build_crpto_journal_package.py`). -- `crpto_extended_evidence_cards_2026-06-06.md` — fichas de evidencia extendida - (PyEPO/DFL, FICO proxy, IFRS9/SICR, CRC/CROMS-lite); superficie canónica en - `reports/crpto/extended/`. -- `crpto_conditional_tightening_appendix_2026-05-04.md` — lemma condicional - Hoeffding/Bernstein y caveats de dependencia (material del supplement). -- `crpto_bound_tightening_experiment_2026-06-11.md` — registro del menú de - bounds A21 (Cantelli/Bennett/Bernstein + la fila agnóstica que prueba Markov - óptimo). Generado por `scripts/build_bound_tightening_audit.py`. -- `pool93_tail_risk_closeout_2026-07-02.md` — cierre A37--A39 para la - asignacion pool93 seleccionada: tail-risk repricing, cluster-bound y - bootstrap fijo. -- `archive/paper4_crpto_crosswalk_2026-07-02.md` — decision autocontenida sobre - que evidencia extendida entra al paper IJDS, que queda en supplement y que - queda en tesis/future work (archivada tras aplicarse). -- `ijds_rebaseline_2026-06-07.md` — registro historico del rebaseline previo - (`ijds-rebaseline-2026-06-07`), retenido como provenance despues del cierre - pool93. -- `literature_reference_audit_2026-06-14.md` — auditoría de evidencia local de - lectura para las referencias citadas en el cuerpo IJDS. -- `ijds_simplification_cleanup_audit_2026-07-06.md` — decisión reader-facing - sobre qué lenguaje técnico queda en el body, qué pasa al supplement/package, - limpieza local de peso y frontera entre refactor estricto y nueva corrida - tolerante. -- `ijds_scientific_upgrade_audit_2026-07-07.md` — separación entre mejoras de - paper que pueden entrar con evidencia congelada y extensiones que quedan - fuera del claim enviado salvo un nuevo protocolo etiquetado. - -- `ijds_corpus_claims_improvement_plan_2026-07-07.md` - analisis con - `academic-pdf-intake` del paper, supplement, submission PDF y corpus - `Papers_tesis`; sus recomendaciones editoriales IJDS quedaron aplicadas y se - conserva como trazabilidad, no como backlog abierto. -- `ijds_literature_expansion_scan_2026-07-08.md` - scan de literatura externa y - local para nuevas referencias IJDS: contextual optimization, incertidumbre de - credit scoring, conformal no-exchangeable, post-selection y comparadores - decision-calibrated 2026. -- `pool93_certificate_semantics_v2_2026-07-09.md` - auditoría histórica de la - frontera pool93 y su baseline; conserva procedencia, no claims activos. - -## Registros de gobernanza (decisiones; no se re-ejecutan sin permiso) - -- `crpto_champion_reopen_plan_2026-05-21.md` — secuencia gobernada de reopen - del champion y sus gates innegociables (documentación, no acción). -- `crpto_bound_improvement_intake_2026-05-21.md` — paquete de challengers - PD/conformal del sandbox externo y sus gates. -- `crpto_champion_tournament_protocol_2026-05-25.md` — contrato anti-cherry-pick - y gates del torneo de champion; histórico después de la promoción pool93. -- `crpto_publication_strategy_2026-05-12.md` — decisión de venue, plantilla, - anonimato y salida IJDS/EJOR; algunas referencias a `45/45` son históricas. -- `crpto_pyepo_dfl_intake_2026-05-26.md` — PyEPO 1.3.7 y cierre del comparador - DFL/SPO+ aislado. - -## `literature/` — notas de lectura versionadas - -Notas curadas sobre fuentes específicas. Los PDFs de literatura se mantienen en -`Papers_tesis/`, que está ignorado por Git, para no commitear material con -copyright. En `docs/research/literature/` deben quedar solo notas, hashes, -decisiones editoriales y trazabilidad de uso. - -## `foundations/` — referencia técnica perenne - -Material de fundamento que puede volver a consultarse para la tesis: conformal -prediction (research, quick reference, comparación de librerías), calibración, -MRM, literatura de portfolio selection, state-of-the-art, decisiones de -arquitectura (`crpto_decision_changes_and_learnings.md`), topología del -pipeline, runbook e integraciones. - -## `future_work/` — backlog del segundo paper - -Memos de extensión (incertidumbre temporal vNext, conformal de clasificación, -validation hardening) que no forman parte de los claims del manuscrito IJDS. - -> Para el **estado de refactors**, ver [`../refactor/README.md`](../refactor/README.md); -> el backlog consolidado 2026-06 quedó ejecutado y archivado en -> [`../refactor/archive/NEXT_WORK_PLAN_2026-06.md`](../refactor/archive/NEXT_WORK_PLAN_2026-06.md). -> Las fases F1–F5 de la auditoría 2026-07-05 están cerradas; el registro de -> ejecución y el backlog post-submission viven en -> [`crpto_full_audit_2026-07-05.md`](crpto_full_audit_2026-07-05.md). +# Active Research Dossier + +This directory contains only records needed to understand or reproduce the +current CRPTO IJDS paper. A dated document is evidence only when the active +claim or source registry names it. + +## Reading Order + +1. `active_claims_2026-07-14.md` +2. `ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md` +3. `ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md` +4. `ijds_normalized_objective_frontier_protocol_2026-07-12.md` +5. `ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md` +6. `ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md` +7. `ijds_raw_data_contract_protocol_2026-07-13.md` +8. `ijds_credit_risk_controls_protocol_2026-07-13.md` +9. `ijds_label_lag_sensitivity_protocol_2026-07-14.md` +10. `ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md` +11. `ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md` +12. `../../configs/ijds_active_evidence_sources.yaml` +13. `../../configs/ijds_claim_ledger.yaml` +14. `../../reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` + +## Active Interpretation + +The paper audits one frozen ML--conformal--optimization pipeline. Under the +declared six-month outcome-availability rule, all five learner specifications +have all eight all-candidate coverage upper bounds below 0.90. Binary residual +geometry changes sharply near a prevalence threshold. Portfolio contrast signs +depend on the outcome-blind ruler, coordinate, and comparator support; exact +broad-support envelopes all cross zero. No learner, missingness encoding, +gamma, ruler, coordinate, scenario, or policy is selected. + +## Literature + +Bibliographic metadata belongs in `paper/references.bib`. The local +`Papers_tesis/` directory contains the working PDF corpus and is ignored by +Git. Do not commit copyrighted PDFs. Literature supports positioning and +assumptions; it does not override the registered empirical evidence. + +## Historical Boundary + +Earlier paper versions, search results, policy promotions, and development +memos are archived at `D:\crpto_legacy` and in Git history. A small number of +old paths remain in the repository only because immutable DVC or extraction +hashes require them. They are not active scientific records and should not be +summarized in the manuscript. + +## Maintenance Rule + +Do not add routine progress memos. Update the relevant protocol, registry, +test, builder, or manuscript surface directly. Create a new protocol only when +a scientific object changes before execution. diff --git a/docs/research/active_claims_2026-07-04.md b/docs/research/active_claims_2026-07-04.md deleted file mode 100644 index f765e9f..0000000 --- a/docs/research/active_claims_2026-07-04.md +++ /dev/null @@ -1,168 +0,0 @@ -# CRPTO Active Claim Registry - 2026-07-09 - -This file is the source of truth for IJDS-facing claims. The active result is a -simple, calibration-selected 90% conformal guardrail. Older pool93 frontier -files remain immutable provenance under `EXTRACTION_MANIFEST.json`; they are no -longer evidence for the manuscript's main claim. - -## Active Decision - -- Run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` -- Conformal target: `alpha = 0.10`; frozen conservative alpha used by the - recipe: `0.095`. -- Partition: five score-quantile Mondrian cells on calibrated PD (the frozen - artifact retains the historical label `score_decile_mondrian`). -- Decision score: `q_i = p_i + 0.50 (u_i - p_i) = (p_i + u_i) / 2`. -- Portfolio risk tolerance: `tau = 0.17`. -- Economic objective: expected point-PD net return, `c_i - p_i L`, with - `L = 0.45`. Conformal `q_i` enters the risk constraint, not the objective. -- Budget: `$1,000,000`; maximum concentration: `0.25`. - -The policy is selected from nine round-number candidates: -`tau in {0.15, 0.17, 0.19}` crossed with -`gamma in {0.25, 0.50, 0.75}`. The final tagged selector uses November 2017, -requires full budget use, enforces the effective-PD cap and deterministic -`B_u <= 0.28`, and maximizes expected point-PD objective. Five of nine -candidates are eligible; the selected policy is `tau = 0.17, gamma = 0.50`. -The selected row remains optimal for endpoint caps in -`[0.259036, 0.290491)`. - -The conformal recipe uses `142,550` calibration-fit rows. Policy selection is -performed on `14,943` November rows. Outcomes are stored separately from the -12-column selector frame, which contains no defaults, realized returns, -miscoverage, or assumption-conditional fields. An outcome-free replay on -`20,695` December rows independently selects the same policy. Outcomes joined -afterward give weighted default `0.145650`, weighted -miscoverage `0.124925`, endpoint budget `0.262082`, and accounting bound -`0.387007`. This audit is diagnostic evidence that stable policy selection is -not selected-set coverage validity. Conformal endpoints themselves use -calibration labels, as required. - -## Full OOT Result - -The fixed policy is evaluated on `276,869` loans from January 2018 through -September 2020: - -| Quantity | Value | -|---|---:| -| Funded loans | `308` | -| Allocated budget | `$1,000,000` | -| Expected point-PD objective | `$168,271.56` | -| Realized return | `$179,327.59` | -| Weighted default rate | `0.039375` | -| Weighted miscoverage `V` | `0.036875` | -| Weighted point PD | `0.081949` | -| Weighted decision score | `0.170000` | -| `Gamma_CP` | `0.176102` | -| `Gamma_internalized` | `0.088051` | -| `Gamma_residual` | `0.088051` | -| Endpoint budget `B_u` | `0.258051` | -| Observed accounting bound `B_u + V` | `0.294926` | -| Markov event threshold `B_u + sqrt(alpha)` | `0.574279` | - -The primary fixed-allocation bootstrap return interval is -`$163,421.14`--`$193,551.65` (`5,000` draws over `31` origination-month -clusters). A funded-loan sensitivity gives -`$162,706.17`--`$193,924.74`. Neither resamples the model, conformal recipe, -selector, or optimizer. - -## Matched Baseline - -The point-PD comparator uses the same `276,869` candidates, budget, -concentration cap, `tau = 0.17`, LGD, solver, and point-PD economic objective. -It earns `$196,369.14`, funds `225` loans, and has weighted default `0.118400`, -miscoverage `0.041900`, endpoint budget `0.921317`, and Markov threshold -`1.237545`. - -Relative to that comparator, selected CRPTO gives up `$17,041.55` (`8.678%`) -of realized return, reduces weighted default by `7.9025` percentage points, -reduces weighted miscoverage by `0.5025` percentage points, and lowers the -endpoint-plus-Markov threshold by `66.3266` percentage points. These are -retrospective OOT contrasts, not causal effects or universal dominance. - -A more conservative `gamma = 0.75` comparator earns `$172,939.50`, with -weighted default `0.035875` and threshold `0.516624`. It shows the remaining -within-CRPTO trade-off: the selected 50/50 policy earns `$6,388.08` more at -`0.35` percentage points more weighted default and a `0.057655` higher -threshold. - -## Exact Alpha Evidence - -The exact replay reproduces the stored 90% reference intervals to numerical -precision (`max abs error <= 6.67e-16`). Other alpha levels are sensitivity -rows under the same frozen widening recipe, not separately selected policies. -The declared sensitivity grid is -`A = {0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. - -Do not reinstate the former `alpha = 0.01` headline. Its exact intervals have -average width `0.9882`, and `93.54%` of OOT upper endpoints equal one. That -setting is nearly vacuous for portfolio discrimination. The active method uses -the conventional 90% level because it is the recipe's selected and exactly -replayed reference level. - -## Theory Boundary - -For a fixed funded set, the deterministic accounting identity - -`weighted outcome <= B_u + V` - -holds without a statistical assumption. In the active OOT allocation its -right-hand side is `0.294926`, while the observed weighted outcome is -`0.039375`. - -The Markov statement is secondary. If one additionally assumes weighted -funded-set validity, `E[V] <= alpha`, then - -`P(weighted outcome >= B_u + sqrt(alpha)) <= sqrt(alpha)`. - -At `alpha = 0.10`, this gives threshold `0.574279` and probability bound -`0.316228`. It is deliberately reported as a weak, assumption-conditional -sensitivity, not as a deterministic risk cap, nominal selected-set coverage, -or the paper's primary novelty. - -## Evidence Contract - -The active evidence bundle is: - -- `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` -- A35: exact alpha replay and saturation audit. -- A36: November selector, endpoint-cap stability interval, December replay, - and independent decision audit. -- A37: full-OOT and temporal fixed-policy evaluation. -- A38: selected funded-set grade composition. -- A39: fixed-allocation month-cluster bootstrap plus funded-loan sensitivity. -- A40: selected, more-conservative, and matched point-PD comparison. - -The manuscript must say explicitly that earlier project development inspected -this static OOT corpus. The final tagged rule is outcome-free with respect to -its policy-ranking code path, but the evaluation is a transparent retrospective -lockbox replay, not a pristine prospective trial. - -## Retired Headline Claims - -The following remain provenance only and must not appear as active results: - -- alpha-0.01 endpoints obtained by cross-family average-width scaling; -- the `8/8` approximate alpha-grid pass; -- the 50,010-policy frontier as the active selector; -- the `0.345084` Markov threshold; -- capped/tail-focused policy families as the selected method; -- the exploratory 25-policy `gamma = 0.35`, threshold-cap `0.65` challenger; -- policy hyperparameters chosen from OOT realized outcomes. - -The frozen upstream model, calibrator, interval artifacts, historical pool93 -tables, and `EXTRACTION_MANIFEST.json` remain untouched. - -## Reopen Gate - -Reopen the active method only for one of four reasons: - -1. a calibration-only rule materially improves return at the same `B_u<=0.28` - screen; -2. a simpler rule matches the selected policy within prespecified tolerances; -3. a valid selected-set or prospective protocol materially strengthens the - statistical claim; -4. an IJDS reviewer requests a specific additional test. - -Otherwise, keep one method, one policy, and one manuscript narrative. diff --git a/docs/research/active_claims_2026-07-14.md b/docs/research/active_claims_2026-07-14.md new file mode 100644 index 0000000..07564f0 --- /dev/null +++ b/docs/research/active_claims_2026-07-14.md @@ -0,0 +1,438 @@ +# CRPTO Active IJDS Claim Registry - 2026-07-15 + +This is the sole claim registry for the active IJDS manuscript. Numerical +statements must be traceable to +`reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`, whose inputs are +hash-locked by `configs/ijds_active_evidence_sources.yaml`. Earlier +fixed-taxonomy V1--V3, pool93, compact-v7, selected-policy, and all +pre-endpoint-correction evaluations are provenance only. + +## Editorial Decision + +- **GO:** one retrospective paper about identification at the interface of + credit-risk prediction, binary conformal intervals, and constrained monthly + allocation. +- **NO-GO:** model or policy superiority, selected-set conformal validity, + causal effects, a universal guardrail direction, prospective confirmation, + or deployment claims. +- The strongest result is negative but constructive: candidate coverage does + not transport under five reported learner specifications, binary residual + geometry changes near a prevalence threshold, and status-indexed portfolio + contrasts depend on the outcome-free ruler and coordinate used to define + comparable decision stringency. +- No OOT outcome selects a learner, window, taxonomy, gamma, ruler, coordinate, + cap, comparator, or policy. + +## Active Lineages + +### Binary geometry and exact point-cap audit + +- Outcome-free freeze: `ijds-binary-geometry-frontier-v4-2026-07-12-v1`, tag + `protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1`, commit + `2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd`. +- Active reason-audited evaluation: + `ijds-binary-geometry-frontier-v4-2026-07-15-v5`, tag + `protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5`, commit + `e2bba580a0b07c145bd64ff61440973d6e31349b`. +- V5 imports and verifies the V1 outcome-free freeze. Relative to the preceding + endpoint-corrected evaluation, every shared scientific column reconciles at + machine precision; V5 adds an exhaustive endpoint-reason taxonomy and exact + identification-width columns. + +### Two-ruler diagnostic + +- Outcome-free freeze: `ijds-normalized-objective-frontier-2026-07-13-v1c`, tag + `protocol/ijds-normalized-objective-frontier-2026-07-13-v1c`. +- Active reason-audited evaluation: + `ijds-normalized-objective-frontier-2026-07-15-v5`, tag + `protocol/ijds-normalized-objective-frontier-2026-07-15-v5`, commit + `e2bba580a0b07c145bd64ff61440973d6e31349b`. +- The freeze contains 6,240 solves and 622,455 funded rows. V5 reports 720 + monthly contrasts, 48 window contrasts, 144 metric-direction cells, and eight + endpoint-audit rows. + +### Credit-risk controls and data audits + +- Frozen five-model scores: `ijds-credit-risk-controls-2026-07-13-v1b`, tag + `protocol/ijds-credit-risk-controls-2026-07-13-v1b`. +- Active reason-audited evaluation: + `ijds-credit-risk-controls-2026-07-15-v5`, tag + `protocol/ijds-credit-risk-controls-2026-07-15-v5`, commit + `e2bba580a0b07c145bd64ff61440973d6e31349b`. +- Full archive audit: `ijds-raw-data-contract-2026-07-14-v2`. +- Label-lag sensitivity: `ijds-label-lag-sensitivity-2026-07-14-v1`. +- Evaluated-cap solver audit: `ijds-policy-support-tie-audit-2026-07-12-v1`. + +### Evaluation-endpoint availability sensitivity + +- Complete nonselective run: + `ijds-endpoint-availability-sensitivity-2026-07-14-v1`, tag + `protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1`, commit + `8865f1cfbd387576bdf805f3e52f030261e4b717`. +- It changes the Charged Off administrative availability lag over + `{0,3,6,8,12}` months while holding scores, fitted residual recipes, + supports, rulers, coordinates, and allocations fixed. +- The archive and exploratory lag results had already been inspected. This is + retrospectively protocol-locked complete reporting, not preregistration, + confirmation, or a basis for choosing an endpoint. + +### Portfolio-structure sensitivity + +- Complete nonselective run: + `ijds-portfolio-structure-sensitivity-2026-07-15-v6`, tag + `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6`, commit + `490c653a43e2003d83184f47e1277bd2d4390c43`. +- It crosses budgets `{0.5,1,2}` million, purpose caps + `{0.20,0.25,0.30,1.00}`, and LGD `{0.25,0.45,0.65}` while preserving both + rulers, all three coordinates, eight windows, fifteen monthly menus, and the + active six-month endpoint. +- The outcome-free freeze contains all 36 scenarios and 252 hash-described + shard artifacts. The separately invoked evaluator reports every 1,728 + window contrast and 5,184 metric direction; it selects no scenario. +- V1--V5 are stopped or checkpoint provenance. V6 changes only the + outcome-free ID-order exposure tolerance after a documented numerical + diagnosis; it is retrospective discipline, not preregistration or + confirmation. + +### Missingness-encoding sensitivity + +- Complete feature-semantics-preserving run: + `ijds-missingness-sensitivity-2026-07-15-v3`, tag + `protocol/ijds-missingness-sensitivity-2026-07-15-v3`, commit + `199afb083da37af6a51d5ba9e3c4d6280b952fe9`. +- It compares the active sentinel convention, explicit missing indicators, and + native nullable CatBoost features. All three specifications and all eight + windows are reported; no encoding or learner is selected and no missingness + mechanism or portfolio result is identified. + +### Second temporal origin + +- Complete later-origin run: + `ijds-rolling-origin-2017-2026-07-15-v4`, tag + `protocol/ijds-rolling-origin-stability-2026-07-15-v4`, commit + `e2bba580a0b07c145bd64ff61440973d6e31349b`. +- The later origin repeats the declared CatBoost coverage audit over eight + shifted residual windows. It is a two-origin retrospective recurrence, not an + independent replication, prospective validation, or invariance result. + +### Fit-label completion sensitivity + +- Complete four-scenario run: + `ijds-fit-label-completion-sensitivity-2026-07-16-v2`, tag + `protocol/ijds-fit-label-completion-sensitivity-2026-07-16-v2`, commit + `fbcafcf84645024b9753aba2f04a4263b8e76236`. +- It refits CatBoost, Platt scaling, and all residual recipes under the + observed-only fit and three declared stress rules for the 215 labels + unavailable at their fitting cutoffs. Evaluation outcomes remain isolated + until the score-and-recipe freeze. +- All 32 overall scenario-window coverage upper bounds remain below 0.90. The + W7--W8 stratum-2 crossing remains in three scenarios but disappears when all + unavailable labels are completed as default. These are nonlinear stress + scenarios, not sharp bounds over all label assignments. + +### Allocation-granularity sensitivity + +- Complete deterministic run: + `ijds-allocation-granularity-sensitivity-2026-07-16-v3`, tag + `protocol/ijds-allocation-granularity-sensitivity-2026-07-16-v3`, commit + `fb1a7b1837d1f8ab2b81239533f51c996f41671c`. +- It floors every positive continuous exposure to a USD 25 lot, holds residual + capital as cash, and evaluates all 96 baseline endpoint tracks without + reoptimization or selection. +- Rate perturbations are at most 0.000345 percentage points for payoff, 0.001284 + for default, and 0.001190 for miscoverage. This diagnoses the continuous + relaxation; it does not authorize an integer policy. + +## Research Object + + + + + +- Raw archive: 2,925,493 rows, 2,925,492 valid dated loans, 142 columns, + 2,060,077 36-month contracts, and 865,415 60-month contracts. +- Active status-independent design: all 640,543 eligible 36-month loans under + the declared dates, maturity, schema, and origination-observability rules. + This is not a computational sample. +- PD development: 17,433 rows. Probability calibration/taxonomy: 14,101 rows. + Residual pool: 49,007 rows. Outcome-free policy development: 94,885 rows. +- Primary OOT: 376,890 candidates in 15 monthly menus from April 2016 through + June 2017. Censored extension: 88,227 candidates from July--September 2017. +- Primary endpoint census: 364,814 resolved and 12,076 unresolved candidates. +- The endpoint-reason taxonomy partitions every candidate: 307,842 are Fully + Paid by the cutoff, 56,972 are Charged Off by the cutoff, 11,551 remain + nonterminal or unresolved, 47 terminate after the reconstructed cutoff, and + 478 have no reconstructible terminal availability date. +- The distributed archive is not a verified September 2020 point-in-time + snapshot. Terminal status is conservatively reconstructed as observable by + September 30, 2020; terminal statuses whose reconstructed availability is + later remain unresolved. +- The raw file contains 36,485 last-payment dates and 40,214 last-credit-pull + dates after the evaluation cutoff. These facts prohibit calling the file a + contemporaneous administrative snapshot. +- Candidate membership never uses status. Allocations are frozen before the + endpoint panel is joined, and partial ID joins fail. + +## Prediction and Conformal Object + +- Primary score: CatBoost followed by a separate 2011 Platt map. Only this + score enters portfolio optimization. +- Coverage-only controls: numeric logistic, domain-constrained monotonic + CatBoost, platform-signal WOE/IV, and a pricing-excluded application WOE/IV + scorecard. Each has a separately fitted Platt map and fixed 2011 taxonomy. +- The two original V4 specifications and three later credit-risk controls form + a complete reported coverage audit, not a model contest. Every retained + evaluation was protocol-locked before its corresponding outcome join, but + the inspected archive means this is not preregistration or an untouched + holdout. OOT AUC, Brier, calibration, IV, and PSI are descriptive. +- Five score strata are fixed separately from 2011 scores. All eight eligible + consecutive six-month residual windows beginning January--August 2012 are + reported. The windows overlap and are not independent replications. +- At alpha 0.10, the interval + `[max(0,p-c_g), min(1,p+c_g)]` predicts the observed binary endpoint. It is + neither a confidence interval for individual latent PD nor the convex hull of + a discrete prediction set. +- Exact all-candidate sharp coverage bounds are + `1 - mean(miss_high)` and `1 - mean(miss_low)`. Singleton, empty, and + `{0,1}` prediction sets are handled loan by loan. + +## Active Exact Statements + + + + + +1. **Comparator map.** Positive-affine scores admit translated caps; + normalized coordinates preserve positive-affine units; same-cap nesting and + C2 funded-moment matching mechanically order only the plug-in objective. + None of these facts orders status-indexed payoff, default, or miscoverage. +2. **Binary threshold geometry.** For fixed `0 <= p < 1/2`, the population + residual quantile changes from `p` to `1-p` when prevalence crosses alpha, + while miscoverage is exactly `1{Y=0,l>0} + 1{Y=1,u<1}`. Varying empirical + scores make this a mechanism rather than a finite-sample explanation. +3. **Sharp common-outcome identification.** Candidate and fixed-allocation + bounds use binary loan-wise extrema; paired-policy bounds assign each + unresolved endpoint once over the funded union. + + + For an additive paired contrast, + the sharp interval width is the sum, over unresolved loans, of the absolute + difference between that loan's two attainable contrast contributions. This + is finite-archive partial identification, not sampling uncertainty or a + missingness model. Rate metrics use declared capital normalizers; the + floor-with-cash diagnostic retains committed capital rather than + renormalizing away residual cash. +4. **Exact declared support.** On each LP basis, allocations are affine in the + cap and sharp endpoints are concave/convex, so basis endpoints suffice for + extrema over the declared support. Its envelope is deterministic design + sensitivity, not a confidence interval or universal comparator claim. + +## Coverage and Geometry Evidence + + + + + + + + +Under the declared six-month endpoint contract, all eight all-candidate +coverage upper bounds are below 0.90 for every model: + +| Specification | Lowest lower bound | Highest upper bound | OOT AUC | OOT Brier | +|---|---:|---:|---:|---:| +| CatBoost | 0.842485 | 0.882597 | 0.640605 | 0.129878 | +| Numeric logistic | 0.850031 | 0.896222 | 0.642045 | 0.128846 | +| Monotonic CatBoost | 0.848396 | 0.886489 | 0.651954 | 0.128613 | +| Platform-signal WOE/IV | 0.848908 | 0.894908 | 0.633066 | 0.129485 | +| Pricing-excluded application WOE/IV | 0.852013 | 0.897726 | 0.612939 | 0.130190 | + +- Mean calibration error is negative for all five models (-0.047109 to + -0.028923), and every calibration slope is below one (0.543210--0.918655). +- All 45 OptBinning fits are optimal. The WOE/IV and monotonic specifications + challenge model-class and platform-pricing explanations; they are not the + paper's methodological novelty and cannot promote a model. +- Two active raw features have declared coverage exceptions: + `mths_since_last_delinq` is structurally nullable and + `pub_rec_bankruptcies` has partial legacy support. Their deterministic + missing-value conventions are disclosed. The dedicated encoding sensitivity + preserves feature semantics and gives 8/8 below-nominal upper bounds for the + active convention (maximum 0.882597), explicit indicators (0.884332), and + native nullable features (0.880037). It neither identifies the missingness + mechanism nor authorizes an encoding winner or portfolio claim. +- At the later temporal origin, all eight CatBoost upper bounds are also below + 0.90 (maximum 0.874768), versus 0.882597 at the primary origin. These 16 cells + are two retrospective origin-window families, not independent replications. +- Four declared fit-label scenarios cover 41 unavailable PD-development labels, + 24 Platt labels, and 150 conformal-fit labels. Every one of the 32 overall + coverage upper bounds is below 0.90; scenario maxima range from 0.882594 to + 0.884669. The four scenarios are declared stresses, not a sharp nonlinear + identification region. +- In CatBoost stratum 2, prevalence changes from 0.101703 in W7 to 0.097147 in + W8; the fitted residual quantile changes from 0.888435 to 0.111801 and mean + OOT width from 0.984263 to 0.207631. +- The W7--W8 threshold crossing persists for the retrospectively + protocol-locked 0-, 3-, and 6-month fit-label lags, each retaining more than 99% in every + fitting month. It disappears at 8 and 12 months, which fail the locked + retention rule. This is sensitivity evidence, not a causal attribution. +- The W7--W8 crossing persists in the observed-only, all-nondefault, and + hindsight-terminal completion scenarios. It disappears under all-default: + W7/W8 prevalence is 0.105974/0.100287 and both quantiles remain near 0.889. + The observed phase path is therefore not scenario-invariant. +- W8 stratum-2 coverage remains bounded by [0.822536, 0.854707]. Narrower + intervals do not restore transport. + +## Decision and Comparator Evidence + + + + + + + + +The frozen contrast is gamma 1 minus gamma 0. The objective-matched ruler holds +the common plug-in objective floor fixed; the normalized-score ruler holds a +positive-affine-invariant relative score relaxation fixed but does not equalize +opportunity cost. + +| Ruler / coordinate | Payoff hull (USD) | Default hull (pp) | Miscoverage hull (pp) | +|---|---:|---:|---:| +| Objective matched .25 | [-9,134.34, 5,603.66] | [-0.0068, 0.1265] | [-0.0068, 0.1265] | +| Objective matched .50 | [-82,616.17, -27,958.37] | [0.4572, 1.0973] | [1.0154, 1.9321] | +| Objective matched .75 | [-179,484.66, 92,558.18] | [-0.4352, 2.4948] | [1.3252, 4.1848] | +| Normalized score .25 | [-626,374.61, -195,967.63] | [8.4829, 13.4246] | [8.3910, 13.7536] | +| Normalized score .50 | [-259,658.18, -54,025.82] | [3.2214, 6.5637] | [2.1070, 5.2142] | +| Normalized score .75 | [-135,781.22, 9,812.59] | [1.4392, 2.3807] | [0.3447, 1.6991] | + +The active reason-audited evaluation keeps 12,076 outcomes unrestricted. Of +these, 11,551 are nonterminal or unresolved, 47 have reconstructed terminal +availability after the cutoff, and 478 lack a reconstructible terminal +availability date. The latter two categories together explain the 525-row +difference from the pre-correction endpoint, but they encode distinct sources +of nonobservability and are never presented as one timing mechanism. + +The complete endpoint-availability sensitivity is: + +| Charged Off lag (months) | Resolved / unresolved | Coverage upper below 0.90 | Maximum upper | Payoff lower / cross | Default higher / cross | Miscoverage higher / cross | +|---:|---:|---:|---:|---:|---:|---:| +| 0 | 364,861 / 12,029 | 40/40 | 0.897641 | 32/16 | 33/15 | 40/8 | +| 3 | 364,861 / 12,029 | 40/40 | 0.897641 | 32/16 | 33/15 | 40/8 | +| 6 | 364,814 / 12,076 | 40/40 | 0.897726 | 32/16 | 33/15 | 40/8 | +| 8 | 364,570 / 12,320 | 40/40 | 0.898151 | 32/16 | 33/15 | 40/8 | +| 12 | 363,288 / 13,602 | 39/40 | 0.900411 | 31/17 | 32/16 | 40/8 | + +- The sole lag-12 coverage exception is the pricing-excluded application + WOE/IV scorecard in W2. Therefore the 40/40 statement is valid for the active + six-month contract and lags 0, 3, and 8, but not universally over all tested + endpoint lags. +- After removing the lag column, the six-month row is value-identical to the + active evaluation for 120 coverage cells, 48 two-ruler contrasts, and 648 exact-support + envelopes. +- No opposite one-sided decision direction appears at 12 months. Payoff and + default each move one cell from adverse to crossing zero; miscoverage is + unchanged. +- Broad stress places zero in all 216 exact envelopes at every endpoint lag. + Development-support payoff is 6 lower/66 crossing at lags 0, 3, 6, and 8 and 72 + crossing at lag 12; default is 72 crossing at every lag; miscoverage changes + from 27 higher/45 crossing at lags 0, 3, 6, and 8 to 26/46 at lag 12. +- Fit-label timing and evaluation-endpoint timing are separate one-factor + sensitivities. The former refits residual recipes; the latter holds recipes + and allocations fixed. They were not crossed factorially, so no joint lag + robustness claim is active. + +- The complete portfolio-structure grid contains 36 scenarios and 48 + ruler-coordinate-window cells per metric in each scenario. Every scenario + retains at least 17 cells with higher default and at least 21 with higher + miscoverage. +- Across all 1,728 cells per metric, payoff is lower/higher/crossing/exact-zero + in 740/223/709/56; default is higher/lower/crossing/exact-zero in + 1,072/133/467/56; and miscoverage is higher/lower/crossing/exact-zero in + 1,205/131/336/56. +- Zero scenarios are favorable on all three metrics in all 48 cells, and zero + are adverse on all three metrics in all 48 cells. At least one favorable cell + appears in 26 scenarios for payoff and 20 each for default and miscoverage. +- Purpose caps 0.20--0.30 bind in every portfolio; the 1.00 cap never binds. + The common-frontier constraint binds throughout, and maximum single-loan + weight rounds to 0.08, 0.04, and 0.02 at the three increasing budgets. +- The baseline scenario reconciles exactly to the active evaluation. The grid supports + structural conditionality, not a scenario winner, universal adversity, + structural invariance, or deployment guidance. +- The USD 25 floor-with-cash diagnostic covers 1,440 portfolios, 143,175 + positive source exposures, and 96 endpoint tracks. It changes 2,985 + exposures, leaves at most USD 75 cash per month, and perturbs any evaluated + rate by at most 0.001284 percentage points. It is not an integer + reoptimization or a discrete-policy claim. + +- Objective-matched .25 crosses zero for all three metrics in all eight + windows. Its repeated allocation remains identical across windows, but the + 12,076 unresolved endpoints eliminate the earlier favorable point claim. +- Objective-matched .50 is adverse in all eight windows. At .75, payoff and + default cross zero in seven windows and are adverse in one; miscoverage is + adverse in all eight. +- Normalized .25 and .50 are adverse in all eight windows. At normalized .75, + default and miscoverage are adverse in all eight; payoff is adverse in seven + and crosses zero in one. +- Across all 48 cells, payoff is lower in 32 and crosses zero in 16; default is + higher in 33 and crosses zero in 15; miscoverage is higher in 40 and crosses + zero in 8. No opposite one-sided direction survives. +- Broad stress `[0.05,0.12]` places zero in all 216 exact envelopes. +- Over development-admissible support, terminal default crosses zero in 72/72 + cells, payoff is lower in 6 and crosses in 66, and miscoverage is higher in + 27 and crosses in 45. All 27 W8 envelopes cross zero. +- The evaluated-cap audit contains 7,297 point-cap rows. It finds no near-zero + nonbasic reduced costs and no allocation sensitivity in 2,941 reversed-order + reruns; maximum allocation distance is `1.45e-14`. This supports numerical + stability at evaluated caps, not continuous-frontier uniqueness. + +## Permitted Claims + +- Under the declared six-month endpoint contract, candidate-level binary + coverage fails after temporal transport before optimization for all five + reported score specifications. +- The same CatBoost coverage finding recurs under three feature-semantics- + preserving missingness encodings and at one later retrospective origin, + without selecting an encoding or claiming independent replication. +- The coverage finding also survives all four declared fit-label scenarios; + the W7--W8 geometric crossing survives only three of them. +- Binary absolute-residual geometry is prevalence-sensitive but not invariant + to every fit-label completion. +- The score, ruler, and coordinate jointly define the portfolio comparison. +- Within the finite protocol-locked grid, direction is not invariant to ruler or + coordinate and no endpoint has a universal status-indexed outcome ordering. +- Across the complete budget--concentration--LGD grid, every scenario retains + adverse default and miscoverage cells, while neither universal favorability + nor universal adversity holds. +- Exact support envelopes quantify partial identification over declared + outcome-free comparator supports. +- USD 25 floor-with-cash rounding negligibly perturbs rates in the evaluated + baseline family; this is a continuous-relaxation diagnostic only. +- CRPTO remains one integrated ML--conformal--optimization object; its result is + an audit of the handoff, not abandonment of any component. + +## Forbidden Claims + + +- Policy, gamma, ruler, coordinate, learner, or window winner. +- Selected-set or funded-set conformal guarantee. +- Universal economic, default, or miscoverage dominance. +- A verified September 2020 archive snapshot. +- Causal, prospective, confirmatory, preregistered, deployment, or fair-lending + conclusions. +- Status-indexed standardized payoff proxy as cash-flow return, IRR, NPV, or welfare. +- Independent replication counts from overlapping windows or repeated + allocations. +- Continuous-frontier uniqueness or a universal comparator support. +- Sharp fit-label completion bounds or a phase transition invariant to all + fit-label assignments. +- Integer-policy optimality or robustness to untested allocation lot rules. + +## Pre-Freeze Boundary + +Submission freeze is not active. Further work may improve code, sensitivity +analysis, exposition, and reproducibility, but it may not select results from +2016--2017 outcomes or rewrite protected historical artifacts. Any new +paper-facing run requires a written protocol and immutable tag before the +corresponding run, explicit stop rules, complete reporting, and a new versioned +evidence source. Because the archive has already been inspected, such locking +is retrospective discipline rather than preregistration. diff --git a/docs/research/archive/paper4_crpto_crosswalk_2026-07-02.md b/docs/research/archive/paper4_crpto_crosswalk_2026-07-02.md deleted file mode 100644 index 351a672..0000000 --- a/docs/research/archive/paper4_crpto_crosswalk_2026-07-02.md +++ /dev/null @@ -1,51 +0,0 @@ -# Paper 4 / CRPTO Crosswalk - 2026-07-02 - -## Decision - -Keep CRPTO paper-facing and self-contained. The IJDS paper should not import a -second sequential-decision laboratory narrative. The useful material is already -absorbed as bounded comparators, robustness diagnostics and future-work gates. - -## What Enters The IJDS Paper - -| source material | IJDS destination | reason | -|---|---|---| -| Regret-auditability / SPO+ comparator | Body discussion and A19 supplement table | Shows the trade-off: SPO+ lowers decision regret, while CRPTO supplies the funded-set certificate. | -| Pool93 finite-grid search | Body and A35 | Central promoted result: return-bound frontier, exact alpha checks and selected body point. | -| Pool93 selected-allocation diagnostics | A36--A39 supplement | Grade composition, tail-risk repricing, cluster-bound sensitivity and fixed-allocation bootstrap. | -| Multi-source / localized / weighted coverage ideas | A23 and limitations | Explains where stronger coverage notions would require a new calibration protocol. | -| Tail-risk/OCE/CVaR diagnostics | A20--A22 and A37 | Useful risk-profile evidence, but not the optimized objective. | - -## What Stays Out Of The IJDS Body - -| material | status | boundary | -|---|---|---| -| Full PyEPO / DFL suite | thesis or extended evidence | DFL is a regret laboratory, not a conformal funded-set certificate. | -| FICO proxy comparison | reviewer-response or thesis governance | Score-governance diagnostic only; no fair-lending legal claim. | -| IFRS9/SICR proxy | thesis/future work | IFRS9-inspired only; no contractual allowance model. | -| CATE / causal policy value | parked | Observational diagnostics only; no policy-value claim. | -| Online conformal deployment | parked | Retrospective/source diagnostics only; no live deployment claim. | -| Fair-lending certification | out of scope | Requires protected attributes or an approved proxy protocol. | - -## Paper-Facing Stop Rule - -The current paper should stop at A35--A39 unless a new result changes one of -these claims: - -1. the pool93 body/default return-bound frontier; -2. the exact funded-set certificate; -3. the selected-allocation tail, concentration or bootstrap profile; -4. the A19 regret-auditability contrast; or -5. a reviewer-requested limitation or reproducibility disclosure. - -Everything else is useful research context, but not a reason to reopen the -champion or expand the IJDS body. - -## Current Read - -The strongest IJDS claim is not "best classifier" or "best regret learner." It -is a governed predict-then-optimize certificate: a calibrated PD model feeds -conformal uncertainty, a finite policy surface is searched exactly, and the -selected funded set is audited through a distribution-free Markov cap under the -stated weighted-validity assumption. Pool93 A35--A39 are the evidence package -for that claim. diff --git a/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.csv b/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.csv deleted file mode 100644 index 7c3685b..0000000 --- a/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.csv +++ /dev/null @@ -1,45 +0,0 @@ -alpha,delta,bound,variance_mode,threshold_t,empirical_V_alpha01,margin_vs_empirical_V,tighter_than_markov,empirical_V_below_threshold,paper_role -0.01,0.1,cantelli_one_sided,strong_individual_validity,0.03671911841700631,0.028875000000007218,0.007844118416999096,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.01,0.1,bennett,strong_individual_validity,0.04894523515948999,0.028875000000007218,0.02007023515948277,True,True,conditional independence/variance tightening; appendix-only -0.01,0.1,cantelli_one_sided,weak_weighted_validity,0.06612486080161614,0.028875000000007218,0.03724986080160893,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.01,0.1,bernstein,strong_individual_validity,0.06983234702482426,0.028875000000007218,0.04095734702481704,True,True,conditional independence/variance tightening; appendix-only -0.01,0.1,freedman_martingale,strong_individual_validity,0.06983234702482426,0.028875000000007218,0.04095734702481704,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.01,0.1,bennett,weak_weighted_validity,0.07223113225071706,0.028875000000007218,0.04335613225070985,True,True,conditional independence/variance tightening; appendix-only -0.01,0.1,bernstein,weak_weighted_validity,0.08516935997197075,0.028875000000007218,0.05629435997196354,True,True,conditional independence/variance tightening; appendix-only -0.01,0.1,freedman_martingale,weak_weighted_validity,0.08516935997197075,0.028875000000007218,0.05629435997196354,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.01,0.1,markov,none,0.09999999999999999,0.028875000000007218,0.07112499999999278,False,True,main distribution-free claim; only first moment needed -0.01,0.1,hoeffding,loan_independence,0.10556386729216258,0.028875000000007218,0.07668886729215536,False,True,conditional bounded-difference diagnostic -0.01,0.1,cantelli_one_sided,agnostic_theorem_assumption_only,0.30849623113198604,0.028875000000007218,0.27962123113197884,False,True,"sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions" -0.03,0.17320508075688773,cantelli_one_sided,strong_individual_validity,0.06370389423459714,0.028875000000007218,0.03482889423458993,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.03,0.17320508075688773,bennett,strong_individual_validity,0.07558083265225993,0.028875000000007218,0.04670583265225271,True,True,conditional independence/variance tightening; appendix-only -0.03,0.17320508075688773,bernstein,strong_individual_validity,0.08585069603225817,0.028875000000007218,0.05697569603225096,True,True,conditional independence/variance tightening; appendix-only -0.03,0.17320508075688773,freedman_martingale,strong_individual_validity,0.08585069603225817,0.028875000000007218,0.05697569603225096,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.03,0.17320508075688773,cantelli_one_sided,weak_weighted_validity,0.10079673598756035,0.028875000000007218,0.07192173598755314,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.03,0.17320508075688773,bennett,weak_weighted_validity,0.10879705436939043,0.028875000000007218,0.07992205436938321,True,True,conditional independence/variance tightening; appendix-only -0.03,0.17320508075688773,hoeffding,loan_independence,0.11338950220977291,0.028875000000007218,0.0845145022097657,True,True,conditional bounded-difference diagnostic -0.03,0.17320508075688773,bernstein,weak_weighted_validity,0.11448842470504272,0.028875000000007218,0.08561342470503551,True,True,conditional independence/variance tightening; appendix-only -0.03,0.17320508075688773,freedman_martingale,weak_weighted_validity,0.11448842470504272,0.028875000000007218,0.08561342470503551,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.03,0.17320508075688773,markov,none,0.17320508075688773,0.028875000000007218,0.1443300807568805,False,True,main distribution-free claim; only first moment needed -0.03,0.17320508075688773,cantelli_one_sided,agnostic_theorem_assumption_only,0.40270488101738233,0.028875000000007218,0.37382988101737513,False,True,"sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions" -0.05,0.22360679774997896,cantelli_one_sided,strong_individual_validity,0.08710942569059568,0.028875000000007218,0.05823442569058847,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.05,0.22360679774997896,bennett,strong_individual_validity,0.09932002901426772,0.028875000000007218,0.0704450290142605,True,True,conditional independence/variance tightening; appendix-only -0.05,0.22360679774997896,bernstein,strong_individual_validity,0.10612143445843009,0.028875000000007218,0.07724643445842287,True,True,conditional independence/variance tightening; appendix-only -0.05,0.22360679774997896,freedman_martingale,strong_individual_validity,0.10612143445843009,0.028875000000007218,0.07724643445842287,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.05,0.22360679774997896,hoeffding,loan_independence,0.12707656887180302,0.028875000000007218,0.0982015688717958,True,True,conditional bounded-difference diagnostic -0.05,0.22360679774997896,cantelli_one_sided,weak_weighted_validity,0.12795022720461297,0.028875000000007218,0.09907522720460575,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.05,0.22360679774997896,bennett,weak_weighted_validity,0.13831826683965617,0.028875000000007218,0.10944326683964896,True,True,conditional independence/variance tightening; appendix-only -0.05,0.22360679774997896,bernstein,weak_weighted_validity,0.14195940375105448,0.028875000000007218,0.11308440375104727,True,True,conditional independence/variance tightening; appendix-only -0.05,0.22360679774997896,freedman_martingale,weak_weighted_validity,0.14195940375105448,0.028875000000007218,0.11308440375104727,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.05,0.22360679774997896,markov,none,0.223606797749979,0.028875000000007218,0.19473179774997176,False,True,main distribution-free claim; only first moment needed -0.05,0.22360679774997896,cantelli_one_sided,agnostic_theorem_assumption_only,0.4561113860291041,0.028875000000007218,0.4272363860290969,False,True,"sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions" -0.1,0.31622776601683794,cantelli_one_sided,strong_individual_validity,0.14141489843989055,0.028875000000007218,0.11253989843988334,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.1,0.31622776601683794,bennett,strong_individual_validity,0.15470968466559049,0.028875000000007218,0.12583468466558326,True,True,conditional independence/variance tightening; appendix-only -0.1,0.31622776601683794,bernstein,strong_individual_validity,0.15823021015605543,0.028875000000007218,0.1293552101560482,True,True,conditional independence/variance tightening; appendix-only -0.1,0.31622776601683794,freedman_martingale,strong_individual_validity,0.15823021015605543,0.028875000000007218,0.1293552101560482,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.1,0.31622776601683794,hoeffding,loan_independence,0.16757385859869947,0.028875000000007218,0.13869885859869224,True,True,conditional bounded-difference diagnostic -0.1,0.31622776601683794,cantelli_one_sided,weak_weighted_validity,0.18699409066477574,0.028875000000007218,0.1581190906647685,True,True,conditional variance diagnostic; sharper one-sided Chebyshev -0.1,0.31622776601683794,bennett,weak_weighted_validity,0.20238284767581538,0.028875000000007218,0.17350784767580815,True,True,conditional independence/variance tightening; appendix-only -0.1,0.31622776601683794,bernstein,weak_weighted_validity,0.2042032156609526,0.028875000000007218,0.17532821566094536,True,True,conditional independence/variance tightening; appendix-only -0.1,0.31622776601683794,freedman_martingale,weak_weighted_validity,0.2042032156609526,0.028875000000007218,0.17532821566094536,True,True,martingale analogue of Bernstein; needs a sealed sequential protocol -0.1,0.31622776601683794,markov,none,0.31622776601683794,0.028875000000007218,0.28735276601683074,False,True,main distribution-free claim; only first moment needed -0.1,0.31622776601683794,cantelli_one_sided,agnostic_theorem_assumption_only,0.5411405551693861,0.028875000000007218,0.5122655551693789,False,True,"sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions" diff --git a/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.tex b/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.tex deleted file mode 100644 index aaf9d80..0000000 --- a/docs/research/bound_tightening_audit/crpto_tableA21c_bound_comparison_experimental.tex +++ /dev/null @@ -1,50 +0,0 @@ -\begin{tabular}{rrllrrrrrl} -\toprule -alpha & delta & bound & variance\_mode & threshold\_t & empirical\_V\_alpha01 & margin\_vs\_empirical\_V & tighter\_than\_markov & empirical\_V\_below\_threshold & paper\_role \\ -\midrule -0.010000 & 0.100000 & cantelli\_one\_sided & strong\_individual\_validity & 0.036719 & 0.028875 & 0.007844 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.010000 & 0.100000 & bennett & strong\_individual\_validity & 0.048945 & 0.028875 & 0.020070 & True & True & conditional independence/variance tightening; appendix-only \\ -0.010000 & 0.100000 & cantelli\_one\_sided & weak\_weighted\_validity & 0.066125 & 0.028875 & 0.037250 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.010000 & 0.100000 & bernstein & strong\_individual\_validity & 0.069832 & 0.028875 & 0.040957 & True & True & conditional independence/variance tightening; appendix-only \\ -0.010000 & 0.100000 & freedman\_martingale & strong\_individual\_validity & 0.069832 & 0.028875 & 0.040957 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.010000 & 0.100000 & bennett & weak\_weighted\_validity & 0.072231 & 0.028875 & 0.043356 & True & True & conditional independence/variance tightening; appendix-only \\ -0.010000 & 0.100000 & bernstein & weak\_weighted\_validity & 0.085169 & 0.028875 & 0.056294 & True & True & conditional independence/variance tightening; appendix-only \\ -0.010000 & 0.100000 & freedman\_martingale & weak\_weighted\_validity & 0.085169 & 0.028875 & 0.056294 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.010000 & 0.100000 & markov & none & 0.100000 & 0.028875 & 0.071125 & False & True & main distribution-free claim; only first moment needed \\ -0.010000 & 0.100000 & hoeffding & loan\_independence & 0.105564 & 0.028875 & 0.076689 & False & True & conditional bounded-difference diagnostic \\ -0.010000 & 0.100000 & cantelli\_one\_sided & agnostic\_theorem\_assumption\_only & 0.308496 & 0.028875 & 0.279621 & False & True & sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions \\ -0.030000 & 0.173205 & cantelli\_one\_sided & strong\_individual\_validity & 0.063704 & 0.028875 & 0.034829 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.030000 & 0.173205 & bennett & strong\_individual\_validity & 0.075581 & 0.028875 & 0.046706 & True & True & conditional independence/variance tightening; appendix-only \\ -0.030000 & 0.173205 & bernstein & strong\_individual\_validity & 0.085851 & 0.028875 & 0.056976 & True & True & conditional independence/variance tightening; appendix-only \\ -0.030000 & 0.173205 & freedman\_martingale & strong\_individual\_validity & 0.085851 & 0.028875 & 0.056976 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.030000 & 0.173205 & cantelli\_one\_sided & weak\_weighted\_validity & 0.100797 & 0.028875 & 0.071922 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.030000 & 0.173205 & bennett & weak\_weighted\_validity & 0.108797 & 0.028875 & 0.079922 & True & True & conditional independence/variance tightening; appendix-only \\ -0.030000 & 0.173205 & hoeffding & loan\_independence & 0.113390 & 0.028875 & 0.084515 & True & True & conditional bounded-difference diagnostic \\ -0.030000 & 0.173205 & bernstein & weak\_weighted\_validity & 0.114488 & 0.028875 & 0.085613 & True & True & conditional independence/variance tightening; appendix-only \\ -0.030000 & 0.173205 & freedman\_martingale & weak\_weighted\_validity & 0.114488 & 0.028875 & 0.085613 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.030000 & 0.173205 & markov & none & 0.173205 & 0.028875 & 0.144330 & False & True & main distribution-free claim; only first moment needed \\ -0.030000 & 0.173205 & cantelli\_one\_sided & agnostic\_theorem\_assumption\_only & 0.402705 & 0.028875 & 0.373830 & False & True & sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions \\ -0.050000 & 0.223607 & cantelli\_one\_sided & strong\_individual\_validity & 0.087109 & 0.028875 & 0.058234 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.050000 & 0.223607 & bennett & strong\_individual\_validity & 0.099320 & 0.028875 & 0.070445 & True & True & conditional independence/variance tightening; appendix-only \\ -0.050000 & 0.223607 & bernstein & strong\_individual\_validity & 0.106121 & 0.028875 & 0.077246 & True & True & conditional independence/variance tightening; appendix-only \\ -0.050000 & 0.223607 & freedman\_martingale & strong\_individual\_validity & 0.106121 & 0.028875 & 0.077246 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.050000 & 0.223607 & hoeffding & loan\_independence & 0.127077 & 0.028875 & 0.098202 & True & True & conditional bounded-difference diagnostic \\ -0.050000 & 0.223607 & cantelli\_one\_sided & weak\_weighted\_validity & 0.127950 & 0.028875 & 0.099075 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.050000 & 0.223607 & bennett & weak\_weighted\_validity & 0.138318 & 0.028875 & 0.109443 & True & True & conditional independence/variance tightening; appendix-only \\ -0.050000 & 0.223607 & bernstein & weak\_weighted\_validity & 0.141959 & 0.028875 & 0.113084 & True & True & conditional independence/variance tightening; appendix-only \\ -0.050000 & 0.223607 & freedman\_martingale & weak\_weighted\_validity & 0.141959 & 0.028875 & 0.113084 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.050000 & 0.223607 & markov & none & 0.223607 & 0.028875 & 0.194732 & False & True & main distribution-free claim; only first moment needed \\ -0.050000 & 0.223607 & cantelli\_one\_sided & agnostic\_theorem\_assumption\_only & 0.456111 & 0.028875 & 0.427236 & False & True & sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions \\ -0.100000 & 0.316228 & cantelli\_one\_sided & strong\_individual\_validity & 0.141415 & 0.028875 & 0.112540 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.100000 & 0.316228 & bennett & strong\_individual\_validity & 0.154710 & 0.028875 & 0.125835 & True & True & conditional independence/variance tightening; appendix-only \\ -0.100000 & 0.316228 & bernstein & strong\_individual\_validity & 0.158230 & 0.028875 & 0.129355 & True & True & conditional independence/variance tightening; appendix-only \\ -0.100000 & 0.316228 & freedman\_martingale & strong\_individual\_validity & 0.158230 & 0.028875 & 0.129355 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.100000 & 0.316228 & hoeffding & loan\_independence & 0.167574 & 0.028875 & 0.138699 & True & True & conditional bounded-difference diagnostic \\ -0.100000 & 0.316228 & cantelli\_one\_sided & weak\_weighted\_validity & 0.186994 & 0.028875 & 0.158119 & True & True & conditional variance diagnostic; sharper one-sided Chebyshev \\ -0.100000 & 0.316228 & bennett & weak\_weighted\_validity & 0.202383 & 0.028875 & 0.173508 & True & True & conditional independence/variance tightening; appendix-only \\ -0.100000 & 0.316228 & bernstein & weak\_weighted\_validity & 0.204203 & 0.028875 & 0.175328 & True & True & conditional independence/variance tightening; appendix-only \\ -0.100000 & 0.316228 & freedman\_martingale & weak\_weighted\_validity & 0.204203 & 0.028875 & 0.175328 & True & True & martingale analogue of Bernstein; needs a sealed sequential protocol \\ -0.100000 & 0.316228 & markov & none & 0.316228 & 0.028875 & 0.287353 & False & True & main distribution-free claim; only first moment needed \\ -0.100000 & 0.316228 & cantelli\_one\_sided & agnostic\_theorem\_assumption\_only & 0.541141 & 0.028875 & 0.512266 & False & True & sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions \\ -\bottomrule -\end{tabular} diff --git a/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.csv b/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.csv deleted file mode 100644 index 2a0684b..0000000 --- a/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.csv +++ /dev/null @@ -1,15 +0,0 @@ -assumption,status,diagnostic_value,interpretation -nonnegative_normalized_weights,pass,1.0,Funded-set weights are non-negative and normalized. -bounded_miss_indicators,pass,1.0,"miscovered_alpha01 is binary, so V is a bounded weighted sum." -effective_sample_size,concentrated,126.06608326411892,"n_eff is far below the funded loan count, so iid-style concentration does not get to use the headline OOT sample size." -loan_independence,not_verified,,Defaults and conformal misses can share calibration history and macro period shocks; loan-level concentration bounds are appendix-only. -post_selection_uniformity,not_supported_by_markov,45.0,A naive union Markov statement over the 45 final policies is vacuous at alpha01; the exact robust-region audit remains empirical evidence. -sequential_martingale_protocol,not_available,,"Azuma/Freedman need a prospective filtration or online validation design; the current A24 replay is diagnostic, not a live guarantee." -chebyshev_two_sided,drop_from_table,,Two-sided Chebyshev is dominated by Cantelli for the one-sided exceedance probability used in A21. -azuma_hoeffding_martingale,drop_from_table,,Azuma gives the same numerical threshold as Hoeffding here while adding a sequential validation protocol assumption. -chernoff_mgf,drop_from_table,,"Chernoff is sharp, but it requires independent misses with each individual miss probability bounded by alpha." -union_markov_45_policy_region,drop_from_table,45.0,A naive union Markov statement over the 45 final policies is vacuous at the paper alphas. -empirical_bernstein_or_bootstrap,diagnostic_only,,"Empirical-Bernstein or bootstrap intervals would use observed OOT labels; useful for sensitivity, not for the distribution-free theorem." -cluster_independence_period,conditional_loose,0.5364639575384164,"Cluster Hoeffding threshold at delta=0.10 is 0.5365; max cluster exposure is 0.2913, so this is not tighter than Markov's 0.1000 threshold." -cluster_independence_grade,conditional_loose,0.6512384189214475,"Cluster Hoeffding threshold at delta=0.10 is 0.6512; max cluster exposure is 0.4235, so this is not tighter than Markov's 0.1000 threshold." -cluster_independence_period_grade,conditional_loose,0.33441555802895656,"Cluster Hoeffding threshold at delta=0.10 is 0.3344; max cluster exposure is 0.1480, so this is not tighter than Markov's 0.1000 threshold." diff --git a/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.tex b/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.tex deleted file mode 100644 index 6ec3b2e..0000000 --- a/docs/research/bound_tightening_audit/crpto_tableA21d_bound_assumption_audit_experimental.tex +++ /dev/null @@ -1,20 +0,0 @@ -\begin{tabular}{llrl} -\toprule -assumption & status & diagnostic\_value & interpretation \\ -\midrule -nonnegative\_normalized\_weights & pass & 1.000000 & Funded-set weights are non-negative and normalized. \\ -bounded\_miss\_indicators & pass & 1.000000 & miscovered\_alpha01 is binary, so V is a bounded weighted sum. \\ -effective\_sample\_size & concentrated & 126.066083 & n\_eff is far below the funded loan count, so iid-style concentration does not get to use the headline OOT sample size. \\ -loan\_independence & not\_verified & NaN & Defaults and conformal misses can share calibration history and macro period shocks; loan-level concentration bounds are appendix-only. \\ -post\_selection\_uniformity & not\_supported\_by\_markov & 45.000000 & A naive union Markov statement over the 45 final policies is vacuous at alpha01; the exact robust-region audit remains empirical evidence. \\ -sequential\_martingale\_protocol & not\_available & NaN & Azuma/Freedman need a prospective filtration or online validation design; the current A24 replay is diagnostic, not a live guarantee. \\ -chebyshev\_two\_sided & drop\_from\_table & NaN & Two-sided Chebyshev is dominated by Cantelli for the one-sided exceedance probability used in A21. \\ -azuma\_hoeffding\_martingale & drop\_from\_table & NaN & Azuma gives the same numerical threshold as Hoeffding here while adding a sequential validation protocol assumption. \\ -chernoff\_mgf & drop\_from\_table & NaN & Chernoff is sharp, but it requires independent misses with each individual miss probability bounded by alpha. \\ -union\_markov\_45\_policy\_region & drop\_from\_table & 45.000000 & A naive union Markov statement over the 45 final policies is vacuous at the paper alphas. \\ -empirical\_bernstein\_or\_bootstrap & diagnostic\_only & NaN & Empirical-Bernstein or bootstrap intervals would use observed OOT labels; useful for sensitivity, not for the distribution-free theorem. \\ -cluster\_independence\_period & conditional\_loose & 0.536464 & Cluster Hoeffding threshold at delta=0.10 is 0.5365; max cluster exposure is 0.2913, so this is not tighter than Markov's 0.1000 threshold. \\ -cluster\_independence\_grade & conditional\_loose & 0.651238 & Cluster Hoeffding threshold at delta=0.10 is 0.6512; max cluster exposure is 0.4235, so this is not tighter than Markov's 0.1000 threshold. \\ -cluster\_independence\_period\_grade & conditional\_loose & 0.334416 & Cluster Hoeffding threshold at delta=0.10 is 0.3344; max cluster exposure is 0.1480, so this is not tighter than Markov's 0.1000 threshold. \\ -\bottomrule -\end{tabular} diff --git a/docs/research/champion_reopen_execution_manifest_2026-06-19.md b/docs/research/champion_reopen_execution_manifest_2026-06-19.md deleted file mode 100644 index 380d01a..0000000 --- a/docs/research/champion_reopen_execution_manifest_2026-06-19.md +++ /dev/null @@ -1,82 +0,0 @@ -# Champion Reopen Execution Manifest - 2026-06-19 - -## Decision - -Continue the champion-reopen thread as exploratory evidence only until a -candidate clears the full CRPTO chain: PD, calibration, conformal validity, -portfolio exact audit, and robust-region checks. - -## Active Claim Target - -Find a defensible challenger that improves the current decision certificate, -not only the classifier AUC. The current promotion floor remains: - -- PD: mean AUC improvement at least `+0.005` without worse Brier/ECE behavior. -- Portfolio: beat `$170,464.54` or keep return similar with clearly better - `V(alpha=0.01)` / `C_CP`. -- Robustness: exact alpha pass, zero violation, and robust region at least - `45/45`. - -## Retained Infrastructure - -- `configs/experiments/champion_reopen.yaml` defines the isolated search grid, - promotion gates, output roots, calibrators, and seed policy. -- `scripts/experiments/run_champion_reopen.py` writes command manifests and - executes resumable smoke, feature-search, seed-replay, and calibration waves. -- `scripts/experiments/monitor_champion_reopen.py` prints the live leaderboard - from experiment-only status JSON files. -- `scripts/experiments/run_tabprep_feature_selection_catboost.py` now ranks - pool features even when the selector model omits them, preventing silent - empty `pooltopX_*` cases. - -## Guardrails - -All outputs must stay under: - -- `data/processed/experiments/champion_reopen/` -- `models/experiments/champion_reopen/` -- `reports/crpto/experiments/champion_reopen/` -- `reports/run_logs/champion_reopen/` - -No champion artifact, extraction manifest, canonical conformal interval, or -portfolio bound-aware promotion output is overwritten by this thread. - -## Runtime Commands - -Create or refresh the dedicated environment: - -```bash -uv venv .venv-champion-search --python 3.11 -uv pip install --python .venv-champion-search/bin/python -r configs/experiments/champion_reopen_requirements.txt -``` - -Run a smoke wave: - -```bash -.venv-champion-search/bin/python scripts/experiments/run_champion_reopen.py \ - --stage smoke \ - --run-tag champion-reopen-2026-06-19 \ - --execute \ - --resume -``` - -Monitor: - -```bash -tail -f reports/run_logs/champion_reopen/champion-reopen-2026-06-19*/*.log -.venv-champion-search/bin/python scripts/experiments/monitor_champion_reopen.py \ - --run-tag champion-reopen-2026-06-19 -``` - -## Stop / Advance Rule - -Advance from smoke to full feature search only if: - -- `pool_ranking_count > 0` and `generated_ranking_count > 0`; -- every smoke case writes status and predictions; -- no protected path is touched; -- at least one non-core subset completes without calibration failure. - -If those pass, launch `--stage feature_search`, then `--stage seed_replay`. -Downstream conformal and portfolio waves remain gated on the top three -seed-stable PD+calibration finalists. diff --git a/docs/research/conformal_claim_parallelization_memo_2026-06-22.md b/docs/research/conformal_claim_parallelization_memo_2026-06-22.md deleted file mode 100644 index a4a5a44..0000000 --- a/docs/research/conformal_claim_parallelization_memo_2026-06-22.md +++ /dev/null @@ -1,81 +0,0 @@ -# Conformal Claim And Parallelization Memo - 2026-06-22 - -## Decision - -Continue the conformal reopen search as a paper-facing claim search, not as a -classification leaderboard. The live claim target is a decision certificate: -a calibrated PD model plus conformal uncertainty should support a robust -portfolio region with auditable coverage, width, bound, return, and zero -violation evidence. - -## Local Paper Synthesis - -The local sources in `Papers_tesis/paper` point to four roles for this phase: - -- Conformal prediction and risk control: Angelopoulos and Bates, Angelopoulos - et al., Bates et al., Barber et al., and Learn-Then-Test support finite-sample - marginal/risk-control language, while warning against overclaiming exact - individual conditional coverage. -- Conformal robust optimization: Sun et al., Patel et al., Hu et al., - Johnstone and Cox, and Zhao et al. motivate calibrated uncertainty sets as - inputs to robust or satisficing decisions, especially when downstream loss is - the object of interest. -- Predict-then-optimize and robust optimization: Bertsimas and Kallus, - Elmachtoub and Grigas, Donti et al., Bertsimas and Sim, Delage and Ye, and - Goldfarb and Iyengar support the paper framing that a useful predictor is one - whose uncertainty improves a decision, not only a score. -- Credit/P2P context: Serrano-Cinca and Gutierrez-Nieto, Guo et al., Zhao et - al., Jagtiani and Lemieux, Albanesi and Vamossy, Chi et al., Torkian et al., - and Das et al. support the business meaning of credit risk, profit scoring, - portfolio allocation, fairness/equity, and tabular/alternative-data credit - modeling. - -## Method Implications - -- Prefer clean partitions with paper meaning: `grade`, score bands, grade by - score band, and later vintage/risk bands if the support is large enough. -- Report Mondrian/group results as approximate group-conditional diagnostics, - not as impossible exact distribution-free conditional guarantees. -- Select conformal candidates by a multi-criterion gate: global coverage, - minimum group coverage, temporal stability, width/Winkler, sidecar set - behavior, and downstream portfolio usefulness. -- Keep calibration alternatives as a tournament, but only promote them when - they do not trade better ECE for wider or less stable conformal intervals. -- Treat MAPIE and crepes as sidecar validators or alternative implementations - for coverage/partition sanity checks unless they beat the native artifact - path without weakening traceability. - -## Implementation Change - -`scripts/search/run_conformal_reopen_search.py` now supports: - -- `--phase1-workers N` for parallel independent phase-1 inner conformal runs. -- Automatic resume of completed phase-1 namespaces when both tuning and result - artifacts exist. -- `conformal_reopen_phase1_progress.json` with total, completed, running, failed - and reused-existing inner runs. - -`configs/profiles/search_conformal_claim_max.yaml` sets `parallel_workers: 3`. - -`scripts/experiments/launch_conformal_parallel_after_current_inner.sh` waits for -the current serial block to publish its first checkpoint, stops the old serial -tmux session, and relaunches the same conformal run tag with parallel phase-1 -workers. This preserves the work already spent on the first block while -allowing the remaining inner blocks to run concurrently. - -## Evidence Gate - -A conformal candidate can move to portfolio only if it provides: - -- accepted global coverage and group coverage under the profile gates; -- no large width inflation versus the current canonical conformal artifact; -- stable temporal diagnostics; -- a clear partition story suitable for the paper; -- a final status artifact with reproducible namespaces and inputs. - -## Stop Rule - -Stop extending conformal variants when the best candidate is either accepted for -portfolio search or all accepted variants are dominated in width, temporal -stability, or downstream bound/return. Extra variants should be parked in -experiment artifacts, not promoted into the manuscript. diff --git a/docs/research/crpto_bound_improvement_intake_2026-05-21.md b/docs/research/crpto_bound_improvement_intake_2026-05-21.md deleted file mode 100644 index bb8141c..0000000 --- a/docs/research/crpto_bound_improvement_intake_2026-05-21.md +++ /dev/null @@ -1,76 +0,0 @@ -# CRPTO Bound-Improvement Intake - 2026-05-21 - -> Ported from the CRPTO research archive -> (`paper1_bound_improvement_intake_2026-05-21`). **Documentation only.** Records -> the PD/conformal challenger package from the external regret-auditability -> sandbox and the gates a bound-improvement lane must pass. The frozen champion is -> not replaced; the regret-auditability lane was closed with append/park. - -External artifact root: an external experiments volume outside this repository. - -## Decision - -This intake does not replace the frozen champion. It records a credible -PD/conformal challenger package and defines the gated runs for the -bound-improvement lane. - -## PD signal - -Main challenger `full_challenger_woe__bureau_behavior_15`: AUC `0.720679`, Brier -`0.153161`, ECE `0.007689`. Delta AUC vs incumbent replay `+0.008001`. - -## Conformal signal - -The sandbox-selected conformal configuration is usable, but not final: grade E -fails the strict 90% group gate and E/G are weak at 95%. The next action is a -focused conformal follow-up before any full cuOpt portfolio promotion. - -## Portfolio quick signal - -The quick CPU run produced an alpha01 pass candidate (return `75602.19`, -`V=0.098750`, `Gamma_CP=0.206650`, zero violation) using only 25k candidates — a -quick signal, not comparable to the 276k champion. - -## Compatibility smoke - -A HiGHS smoke run confirmed local compatibility with the external intervals and -produced an alpha01 pass candidate (mode `blended_uncertainty`, risk `0.175`, -gamma `0.425`, return `59767.36`, `V=0.092925`, `Gamma_CP=0.216429`, zero -violation). Compatibility smoke, not a replacement for the full cuOpt search. - -## Gates - -- Do not compare the quick 25k return directly with the frozen 276k champion. -- Run final portfolio only after focused conformal follow-up. -- Use cuOpt/proxy-first broad search plus exact rerank; CPU exact-all is out of - scope. -- Promote only if the challenger improves a declared metric without breaking - coverage, min-group coverage, exact alpha01 pass, zero violation, and - source/temporal caveats. - -## Bound fronts (agenda) - -- `nested_prospective_confirmation`: run only after PD/conformal/portfolio - selection is frozen. Gate: strict temporal or prospectively sealed split keeps - alpha01 pass with zero violation. -- `direct_crc_ltt_decision_loss`: calibrate monotone loss - `L=max(0, sum w_i Y_i - tau)` or `V` directly. Gate: decision-loss gate passes - without weakening return or coverage. -- `dependency_aware_concentration`: cluster by `issue_month`, `grade`, - source/state and compare cluster-robust tail bounds. Gate: cluster-aware bound - is less vacuous than Markov and more credible than iid. -- `mondrian_funded_set_refinement`: compute `sum_g W_g alpha_g` for selected and - challenger policies. Gate: weighted group bound improves over nominal alpha - without hidden subgroup failure. -- `decision_aware_conformal_selector`: select by coverage, width, robust_return, - `V`, `gamma_cp`, violation and group gates. Gate: selector changes or confirms - the conformal choice under decision metrics. -- `less_conservative_uncertainty_sets`: compare grade x scoreband or - polyhedral/contextual conformal candidates. Gate: `Gamma_CP` falls while - coverage and min-group gates hold. -- `online_shift_aware_bound`: temporal replay or online/weighted conformal only - as a declared retrospective gate. Gate: coverage and `V` stable under sealed - temporal slices. -- `richer_financial_target`: prototype `LGD*default` or ECL proxy loss if data - quality is sufficient. Gate: financial target improves interpretability without - adding unsupported IFRS9 claims. diff --git a/docs/research/crpto_bound_tightening_experiment_2026-06-11.md b/docs/research/crpto_bound_tightening_experiment_2026-06-11.md deleted file mode 100644 index ee5ba37..0000000 --- a/docs/research/crpto_bound_tightening_experiment_2026-06-11.md +++ /dev/null @@ -1,53 +0,0 @@ -# CRPTO Bound Tightening Experiment - 2026-06-11 - -Merged into `main` (2026-06-11) and cited by Online Supplement Appendix A. This audit reads frozen funded-set weights only; it does not re-run DVC stages, does not search policies, and does not promote a new Lending Club champion. The A21c/A21d tables live under `docs/research/bound_tightening_audit/`, deliberately outside the `EXTRACTION_MANIFEST` sweep area: git versioning plus `tests/test_scripts/test_build_bound_tightening_audit.py` guarantee their integrity by re-deriving them deterministically from the frozen A7 weights. - -## Fixed Funded-Set Diagnostics - -- funded loans: `341` -- effective sample size: `126.1` -- sum of squared weights: `0.007932` -- max loan weight: `0.0350` -- observed `V(alpha=0.01)`: `0.028875` - -## Alpha 0.01 Bound Menu - -| Bound | Mode | threshold t | margin vs V | Role | -|---|---|---:|---:|---| -| `cantelli_one_sided` | `strong_individual_validity` | `0.036719` | `0.007844` | conditional variance diagnostic; sharper one-sided Chebyshev | -| `bennett` | `strong_individual_validity` | `0.048945` | `0.020070` | conditional independence/variance tightening; appendix-only | -| `cantelli_one_sided` | `weak_weighted_validity` | `0.066125` | `0.037250` | conditional variance diagnostic; sharper one-sided Chebyshev | -| `bernstein` | `strong_individual_validity` | `0.069832` | `0.040957` | conditional independence/variance tightening; appendix-only | -| `freedman_martingale` | `strong_individual_validity` | `0.069832` | `0.040957` | martingale analogue of Bernstein; needs a sealed sequential protocol | -| `bennett` | `weak_weighted_validity` | `0.072231` | `0.043356` | conditional independence/variance tightening; appendix-only | -| `bernstein` | `weak_weighted_validity` | `0.085169` | `0.056294` | conditional independence/variance tightening; appendix-only | -| `freedman_martingale` | `weak_weighted_validity` | `0.085169` | `0.056294` | martingale analogue of Bernstein; needs a sealed sequential protocol | -| `markov` | `none` | `0.100000` | `0.071125` | main distribution-free claim; only first moment needed | -| `hoeffding` | `loan_independence` | `0.105564` | `0.076689` | conditional bounded-difference diagnostic | -| `cantelli_one_sided` | `agnostic_theorem_assumption_only` | `0.308496` | `0.279621` | sharp variance under E[V]<=alpha alone; worse than Markov, so no second-moment tightening exists without extra assumptions | - -## Recommendation - -- Keep Markov as the body theorem: it is the only first-moment, distribution-free statement compatible with the current post-selection caveat. -- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: cluster exposure is too concentrated. -- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett and Freedman show how much tightness is available if a reviewer accepts stronger independence, variance, or martingale assumptions. -- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. They are respectively dominated, duplicative, too strong for the current individual-alpha evidence, or vacuous after policy-region correction. - -## Assumption Audit - -| assumption | status | diagnostic_value | interpretation | -|:----------------------------------|:------------------------|-------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------| -| nonnegative_normalized_weights | pass | 1 | Funded-set weights are non-negative and normalized. | -| bounded_miss_indicators | pass | 1 | miscovered_alpha01 is binary, so V is a bounded weighted sum. | -| effective_sample_size | concentrated | 126.066 | n_eff is far below the funded loan count, so iid-style concentration does not get to use the headline OOT sample size. | -| loan_independence | not_verified | nan | Defaults and conformal misses can share calibration history and macro period shocks; loan-level concentration bounds are appendix-only. | -| post_selection_uniformity | not_supported_by_markov | 45 | A naive union Markov statement over the 45 final policies is vacuous at alpha01; the exact robust-region audit remains empirical evidence. | -| sequential_martingale_protocol | not_available | nan | Azuma/Freedman need a prospective filtration or online validation design; the current A24 replay is diagnostic, not a live guarantee. | -| chebyshev_two_sided | drop_from_table | nan | Two-sided Chebyshev is dominated by Cantelli for the one-sided exceedance probability used in A21. | -| azuma_hoeffding_martingale | drop_from_table | nan | Azuma gives the same numerical threshold as Hoeffding here while adding a sequential validation protocol assumption. | -| chernoff_mgf | drop_from_table | nan | Chernoff is sharp, but it requires independent misses with each individual miss probability bounded by alpha. | -| union_markov_45_policy_region | drop_from_table | 45 | A naive union Markov statement over the 45 final policies is vacuous at the paper alphas. | -| empirical_bernstein_or_bootstrap | diagnostic_only | nan | Empirical-Bernstein or bootstrap intervals would use observed OOT labels; useful for sensitivity, not for the distribution-free theorem. | -| cluster_independence_period | conditional_loose | 0.536464 | Cluster Hoeffding threshold at delta=0.10 is 0.5365; max cluster exposure is 0.2913, so this is not tighter than Markov's 0.1000 threshold. | -| cluster_independence_grade | conditional_loose | 0.651238 | Cluster Hoeffding threshold at delta=0.10 is 0.6512; max cluster exposure is 0.4235, so this is not tighter than Markov's 0.1000 threshold. | -| cluster_independence_period_grade | conditional_loose | 0.334416 | Cluster Hoeffding threshold at delta=0.10 is 0.3344; max cluster exposure is 0.1480, so this is not tighter than Markov's 0.1000 threshold. | diff --git a/docs/research/crpto_champion_reopen_plan_2026-05-21.md b/docs/research/crpto_champion_reopen_plan_2026-05-21.md deleted file mode 100644 index b4a9b08..0000000 --- a/docs/research/crpto_champion_reopen_plan_2026-05-21.md +++ /dev/null @@ -1,100 +0,0 @@ -# CRPTO Champion Reopen Plan - 2026-05-21 - -> Ported from the CRPTO research archive -> (`paper1_champion_reopen_plan_2026-05-21`). **Documentation only.** Records the -> governed reopen sequence and non-negotiable gates. The frozen champion is not -> reopened by porting this plan; the regret-auditability lane was closed with -> append/park (no replacement). Any real reopen must not contradict -> `models/final_project_promotion.json`. - -## Objective - -Reopen the champion search only where new evidence can plausibly replace the -frozen champion and improve the theoretical bound. The target is one governed -challenger package with a promote / append / park decision, not another broad -artifact loop. - -## Current Evidence - -External PD search produced a credible replacement candidate: - -- main challenger `full_challenger_woe__bureau_behavior_15`: AUC `0.720679` vs - incumbent replay `0.712678` (`+0.008001`); Brier `0.153161` vs `0.154591` - (`-0.001430`); ECE `0.007689` vs `0.006152` (`+0.001537`). -- sensitivity: `full_challenger__canonical_4` (AUC `0.720624`, Brier `0.153182`, - ECE `0.005917`); `full_challenger_woe__affordability_rate_5` (AUC `0.720052`, - Brier `0.153276`, ECE `0.007502`). - -The external PD model expands the feature contract from 42 to 106 features -(adding 64 bureau/behavior/WOE features while retaining current features): -predictive upside with governance cost, so promotion requires downstream proof, -not AUC alone. The external conformal package is usable but not final (grade E -misses the strict 90% group gate; E/G weak at 95%), so the next search focuses -conformal before portfolio promotion. - -## Non-Negotiable Gates - -1. **PD gate.** AUC improvement at least `+0.005` vs frozen incumbent replay; - Brier improvement or non-inferiority; ECE not worse by more than `+0.0025` (or - a sensitivity candidate with better ECE stays competitive downstream); no - monotonicity/threshold/governance regression that invalidates the MRM story. -2. **Conformal gate.** Global 90/95 coverage passes; minimum grade coverage at - 90% passes the strict floor with grade E explicitly resolved or caveated; grade - E/F/G 95% weakness improves or is proven irrelevant to funded-set risk; - width/Winkler does not inflate enough to destroy portfolio value. -3. **Portfolio gate.** Full-universe run (not 25k quick signal); exact `alpha01` - and `alpha03` pass; `violation = 0`; `V <= sqrt(alpha)` at alpha01; robust - return beats the frozen economic champion on the same universe, or improves - bound metrics enough to become an appendix challenger rather than a - replacement. -4. **Bound-improvement gate.** `Gamma_CP` falls, or `V` falls, or the same return - is achieved with a cleaner proof surface; funded-set group weights do not hide - subgroup failure; a sealed/nested confirmation run remains possible after - candidate selection. - -## Reopen Sequence - -- **Phase 0 - Stage PD challengers.** Stage only the three external PD finalists - as candidate artifacts (do not overwrite canonical PD artifacts). This makes - existing conformal code work via an upstream-run-tag indirection without - changing canonical champion files. -- **Phase 1 - Focused Conformal Reopen.** Run conformal reopen using the staged - PD candidate as upstream. Keep Venn-Abers/calibrated and raw sources; compare - `grade`, `score_decile_mondrian`, `grade_x_scoreband_mondrian`; force attention - to grade E/F/G and temporal slices; prefer configurations that reduce - `Gamma_CP` and keep funded-set coverage, not only global coverage. Output: one - winning conformal namespace, one comparison table, one memo on whether grade - E/G got resolved. -- **Phase 2 - Portfolio Reopen (cuOpt frontier + exact rerank).** RAPIDS env for - frontier generation (`cuOpt`), exact HiGHS rerank delegated to `.venv`. Waves: - smoke (`max_candidates=25000`, alpha grid `0.01,0.03,0.10`); medium - (`100000`/`150000`, wider grids); full universe (`max_candidates=0`, full alpha - grid, promotion decision vs `bound_aware_276k_economic_champion`). -- **Phase 3 - Bound Hardening.** Only after Phase 2 selects a serious candidate: - funded-set Mondrian bound `sum_g W_g alpha_g`; decision-aware selector audit; - dependency-aware diagnostic (cluster by `issue_month`, `grade`, state/source - proxy); nested/sealed confirmation on a predeclared holdout; direct CRC/LTT loss - only if it can change the main theorem or appendix bound. - -## Stop Rules - -Stop and park the challenger if any of these happen: - -- PD improvement fails to survive conformal/portfolio downstream. -- Conformal fixes E/G only by making intervals so wide that `Gamma_CP` or return - becomes unusable. -- Full-universe portfolio cannot beat the frozen champion and does not improve - `V`/`Gamma_CP` meaningfully. -- The only remaining improvement requires unsupported IFRS9/ECL, live deployment, - or legal fair-lending claims. - -## Promotion Outcomes - -- **Promote**: replace champion only if full-universe return and bound metrics - beat the current champion under exact validation. -- **Append**: keep as stronger challenger appendix if PD/conformal improves but - portfolio does not replace champion. -- **Park**: keep for agenda extendida CRPTO/tesis if it teaches a method lesson but does not change - CRPTO. -- **Delete/archive**: discard scratch runs and repeated variants that do not - change the decision. diff --git a/docs/research/crpto_champion_tournament_protocol_2026-05-25.md b/docs/research/crpto_champion_tournament_protocol_2026-05-25.md deleted file mode 100644 index 067333f..0000000 --- a/docs/research/crpto_champion_tournament_protocol_2026-05-25.md +++ /dev/null @@ -1,151 +0,0 @@ -# CRPTO / IJDS Champion Tournament Protocol - 2026-05-25 - -> Historical note, 2026-07-04: this protocol is retained as the anti-cherry-pick -> design that motivated the pool93 reopen. The active body claim is now the -> pool93 finite-grid frontier recorded in -> `docs/research/active_claims_2026-07-04.md`. The frozen champion numbers below -> are historical gates, not the current IJDS headline. - -> Ported from the CRPTO research archive -> (`paper1_crpto_ijds_champion_tournament_protocol_2026-05-25`). **Documentation -> only.** Defines the governed, anti-cherry-pick tournament that *would* govern -> any future champion reopen. The frozen champion -> (`paper-thesis-final-economic-2026-04-06`) is **not** reopened by porting this -> protocol; it is recorded as the reviewer-facing selection contract. - -## Purpose - -Reopens the CRPTO champion search as a governed tournament, not another artifact -loop. Target claim: - -`calibrated PD -> Mondrian conformal interval -> uncertainty set -> robust LP -> auditable policy` - -The search may replace the frozen economic champion only if a predeclared -candidate wins the complete downstream trade-off. Better AUC alone, better return -alone, or better bound alone is not sufficient. - -Historical frozen champion gate: return `170464.5429284627`, `V` `0.03645`, -`Gamma_CP` `0.18591`, violation `0`, funded coverage `0.9433`, policy -`blended_uncertainty` (risk `0.175`, gamma `0.45`, uncertainty aversion `0.1`), -region `45/45`. - -## What This Is Testing - -Not whether a new scorer beats the old scorer, but whether the full CRPTO chain -can find a policy that is more valuable and at least as defensible under IJDS -standards: - -1. PD is calibrated and auditable. -2. Conformal coverage is stable enough to become a decision input. -3. The robust LP converts uncertainty into a funded set with zero violation. -4. Exact alpha-grid validation confirms the apparent frontier is real. -5. Nested or prospective confirmation shows the winner was not selected by - peeking across waves. -6. The paper can publish a claim-artifact-test map, negative-results registry, - and final selection rule. - -## Anti-Cherry-Pick Contract - -Before any serious medium/full run, the run root must contain: - -- `predeclared_candidate_registry.json`: every PD/conformal/portfolio lane that - may compete for champion replacement. -- `phase_gate_status.json`: uniform gates for every lane. -- `selection_rule.json`: the final ranking rule and tie-breakers. -- `negative_results_registry.csv`: every failed, skipped, parked or appendix - lane, with reason. - -A late idea may be added only by opening a new protocol version **before** running -its downstream portfolio stage. Late ideas cannot enter the same tournament as -champion candidates after seeing portfolio results; they go to agenda extendida CRPTO/tesis, appendix, -or a future protocol. - -## Candidate Lanes - -- **Lane A - Frozen Incumbent Replay.** Anchor every comparison to the official - claim using `models/final_project_promotion.json` artifacts. Comparator and - sanity gate, not a new challenger. -- **Lane B - External PD Finalists.** `canonical_4`, `bureau_behavior_15`, - `affordability_rate_5`. Gates: AUC/Brier/ECE/reliability/Gini reported - separately; calibration deterioration bounded; monotonicity and - feature-governance auditable; downstream conformal/portfolio proxy improves or - preserves the IJDS claim. -- **Lane C - Governance-Aware PD HPO.** CatBoost `monotone_constraints`, - `feature_weights`, `first_feature_use_penalties`/`penalties_coefficient`; - `posterior_sampling`/Langevin only as diagnostic unless a CPU-reproducible path - is shown. Optuna with persistent per-lane storage, retained failed trials, - constrained/multi-objective (calibration + downstream proxies, not AUC alone), - seed replay for top trials. Any PD that wins only AUC but fails calibration is - appended or parked. -- **Lane D - Calibration & Conformal Tournament.** Primary partitions - `score_decile_mondrian`, `grade`, `grade_x_scoreband_mondrian`, - `vintage_x_scoreband` (only if cell size sufficient). Venn-Abers main; isotonic - /Platt/beta controlled alternatives; MAPIE risk-control only as sidecar. Phase - champion is the best row passing coverage/group/width/temporal/feasibility - gates, not the highest coverage row. The `score_decile_mondrian` lesson stays - active: a regulatory-looking grouping that fails group coverage cannot feed the - robust LP. -- **Lane E - Portfolio Frontier Tournament.** Cascade 25k smoke -> 50k/75k medium - -> 100k/150k frontier -> full universe -> exact alpha-grid rerank + sealed - confirmation. Solver: cuOpt for the broad frontier (RAPIDS env), exact rerank - in `.venv` with HiGHS/highspy; HiGHS fallback rows labeled. Grid: risk - `0.165-0.200`, gamma `0.275-0.600`, uncertainty aversion `0-0.25`; policy - families `blended_uncertainty`, `capped_blended_uncertainty`, - `tail_blended_uncertainty`, segment-tail only if they preserve the main claim. -- **Lane F - Theory & Bound Hardening.** Implement now: funded-set Mondrian - refinement; decision-aware conformal selector audit; nested/prospective - confirmation; regret and price-of-robustness table; bootstrap funded-set - diagnostics. Prototype or park: direct CRC/LTT on decision loss; - dependence-aware concentration by cluster; online/shift-aware conformal; richer - LGD/ECL targets; OCE/CVaR as a new objective. - -## Phase Gates - -- **PD gate.** AUC improves or within tolerance; Brier and ECE non-inferior; - calibration diagrams reveal no new material failure; monotonicity/governance - defensible; downstream conformal feasibility proxy not degraded. -- **Conformal gate.** Global 90/95 coverage passes; min-group coverage passes the - declared floor; rare grades visible in diagnostics; width not inflated enough to - destroy portfolio value; temporal warnings labeled, not hidden. -- **Portfolio gate.** alpha01 exact pass true; violation zero; realized return - competitive; `V`, `Gamma_CP`, funded coverage and composition defensible; - survives medium/full confirmation, not only 25k/50k probes. -- **Historical champion replacement gate.** This was the pre-pool93 gate: - full-universe exact alpha01 pass true; violation `0`; return at least - `170464.5429284627` (preferably with margin); `V <= 0.03645` or a clearly - superior `Gamma_CP`/coverage trade-off without materially worsening `V`; - `Gamma_CP <= 0.18591` or a clearly superior bound trade-off; funded coverage - comparable or better; nested/prospective confirmation passes; the - negative-results registry proves the winner was not cherry-picked. After the - pool93 promotion, new work should use `active_claims_2026-07-04.md` as the - live reopen gate. - -## IJDS Evidence Packet If A New Champion Wins - -Promote only paper-facing artifacts (child convention): a tournament final -summary CSV under `reports/crpto/tables/`, a negative-results registry CSV under -`reports/crpto/tables/`, and a `docs/research/crpto_champion_decision_memo_.md`. -The manuscript body changes only after the decision memo says `promote`. Appendix -receives high-value negative or theorem-tight evidence. agenda extendida CRPTO/tesis receives ideas -that teach a method lesson but fail replacement gates. - -## What Not To Re-run - -- exact-all CPU grid; AUC-only HPO; -- `score8_raw_sqrt`, `grade_cal_sqrt`, `score8_cal_none` as champion lanes after - prior downstream failures (diagnostic controls only); -- online conformal without a serious temporal split; -- OCE/CVaR as replacement objective unless a new protocol explicitly targets a - tail-risk paper, not IJDS CRPTO replacement. - -## Immediate Execution Order (if reopened) - -1. Freeze this protocol and run root. -2. Emit candidate registry and dirty-state audit. -3. Run tournament smoke across all declared PD/conformal lanes, not canonical - alone. -4. Promote only conformal finalists with decision-aware evidence. -5. Run cuOpt frontier 25k/50k uniformly for finalists. -6. Run exact rerank only for finalists selected before seeing full-universe - results. -7. Seal final selection, then run nested/prospective confirmation. diff --git a/docs/research/crpto_conditional_tightening_appendix_2026-05-04.md b/docs/research/crpto_conditional_tightening_appendix_2026-05-04.md deleted file mode 100644 index c905e39..0000000 --- a/docs/research/crpto_conditional_tightening_appendix_2026-05-04.md +++ /dev/null @@ -1,108 +0,0 @@ -# paper-crpto Conditional Tightening Appendix - 2026-05-04 - -This note records the dependency audit behind the conditional Hoeffding/Bernstein -tightening used in the CRPTO theory section. It is an appendix to -`book/chapters/02-marco-teorico.qmd`, not a new -empirical claim and not a replacement for the main Markov theorem. - -Standalone note: this appendix is retained because it is useful for the journal -version and reviewer response package. It must not be upgraded to the main claim -unless a dependence-aware proof or prospective validation design is added. - -## Main Distribution-Free Claim - -The main theorem controls the funded-set weighted noncoverage - -```text -V = sum_i w_i 1{Y_i > u_i(alpha)} -``` - -for a bounded realized target `Y_i in [0, 1]` and an allocation fixed before the -evaluated labels are observed. Under exchangeability between calibration and -test observations, conformal validity gives `E[V] <= alpha`. Markov then gives -the finite-sample, distribution-free probability statement used by the paper. - -This claim does not require independence among the evaluated loans. It also does -not require a parametric default model, a latent-PD assumption, or stable -covariates across all possible future regimes. That is why it remains the -primary theory claim. - -## Conditional Tightening Claim - -The Hoeffding/Bernstein tightening is narrower and conditional. It may be stated -only after conditioning on: - -- the calibration sample used to form conformal radii; -- a funded-set allocation fixed before the evaluated labels are observed; -- additional independence, or conditional independence, of the indicators - `Z_i = 1{Y_i > u_i(alpha)}`. - -Under those extra assumptions, the weighted sum `V = sum_i w_i Z_i` becomes a -sum of bounded independent terms. Hoeffding gives a tail bound based on -`sum_i w_i^2`; Bernstein additionally uses conditional variance and `w_max`. - -## Cluster-Aware Journal Proposition - -The journal-strengthening version can state a weaker dependence-aware -proposition without pretending that loan-level indicators are independent. -Partition the funded set into clusters `g = 1, ..., G` such as period, grade, or -period-grade. Define - -```text -Z_g = sum_{i in g} w_i 1{Y_i > u_i(alpha)} -W_g = sum_{i in g} w_i. -``` - -Within each cluster, the miss indicators may be arbitrarily dependent. After -conditioning on the calibration sample and the fixed funded allocation, assume -only that the cluster aggregates `Z_g` are independent or conditionally -independent across `g`. Then `0 <= Z_g <= W_g` and Hoeffding gives - -```text -P(V - E[V] >= t) <= exp(-2 t^2 / sum_g W_g^2). -``` - -This is not the main theorem. It is a transparent supplement result: it moves -the extra assumption from loan-level independence to cross-cluster independence -and makes the exposure concentration term explicit. Table A14 supplies the -period, grade and period-grade weights that determine whether this tightening -is practically useful. - -The numerical add-on is now Table A21: - -| Cluster | 90% threshold | 95% threshold | Interpretation | -|---|---:|---:|---| -| period | 0.538 | 0.612 | Looser than the main `sqrt(alpha) = 0.100` Markov reporting threshold. | -| grade | 0.630 | 0.717 | Looser than Markov because exposure is concentrated by grade. | -| period-grade | 0.332 | 0.377 | Best of the cluster partitions, but still not tighter than Markov. | - -So the dependence-aware result is mathematically transparent, but empirically -not more tight than Markov for this funded-set exposure concentration. It is -therefore useful as a journal caveat and reviewer-defense calculation, not as a -replacement main theorem. - -## Dependency Caveat - -Split conformal itself does not automatically make the test indicators -independent after calibration. They share a common calibration sample and may -also share unmodeled macro shocks, underwriting regimes, or borrower -correlation. Therefore the tightening should be presented as a conditional -journal appendix: useful, transparent, and mathematically standard, but not part -of the distribution-free core guarantee. - -## Paper-Safe Wording - -Use this distinction in the manuscript: - -- Markov bound: main theorem; finite-sample; distribution-free; conservative. -- Hoeffding/Bernstein tightening: conditional lemma; sharper under additional - independence or dependence-control assumptions. -- Exact 276K evidence: empirical validation of the promoted frozen policy and - robust region; not a post-selection conformal guarantee by itself. - -## Future Strengthening - -A stronger journal version could replace the conditional independence assumption -with a dependence-aware concentration result, a cluster-level bound by period or -grade, or a prospective nested design where policy selection and final -confirmation are separated before labels are inspected. diff --git a/docs/research/crpto_editorial_claims_references.qmd b/docs/research/crpto_editorial_claims_references.qmd deleted file mode 100644 index a69c736..0000000 --- a/docs/research/crpto_editorial_claims_references.qmd +++ /dev/null @@ -1,258 +0,0 @@ -## Guía Editorial, Claims y Referencias {#sec-crpto-editorial-guide} - -> Nota histórica, actualizada 2026-07-09: esta guía fue escrita antes del cierre pool93. -> La fuente activa de métricas y límites de claim es -> `docs/research/active_claims_2026-07-04.md`. Las referencias a -> `paper-thesis-final-economic-2026-04-06`, `45/45` o A7--A11 se mantienen como -> provenance editorial; el cuerpo IJDS actual usa el punto pool93, la frontera -> policy-aware A35, las auditorías A36--A39 y el baseline emparejado A40. - -Esta página existe porque el libro cumple una función distinta al paper. El -paper final debe ser breve, selectivo y persuasivo; el libro puede ser más -generoso: explica por qué se tomó cada decisión, qué evidencia sostiene cada -claim, qué preguntas de reviewer ya están respondidas y qué partes todavía son -agenda futura. En otras palabras, esta sección es el puente entre el proyecto -vivo y el manuscrito publicable. - -Para facilitar la escritura posterior, esta página usa referencias numeradas -locales del tipo `[1]`. La bibliografía global del libro sigue viviendo en -`book/references.bib`; los números de esta página son una guía editorial para -el CRPTO, no un reemplazo del sistema BibTeX del libro. - -### Tesis defendible en una frase - -CRPTO demuestra que una banda conformal calibrada puede dejar de ser un -diagnóstico posterior y convertirse en una restricción operativa para asignar -capital crediticio bajo incertidumbre, con una policy económica promovida, -artefactos reproducibles y un bound que separa cuidadosamente garantía -distribution-free de evidencia empírica post-selección. - -La frase anterior tiene cuatro piezas: - -1. **banda conformal calibrada**: se apoya en la teoría conformal y de control de - riesgo [1]--[5]; -2. **restricción operativa**: conecta esos intervalos con robust optimization y - price of robustness [6]--[9]; -3. **policy económica promovida**: usa el cierre pool93 documentado en A35--A40 - y no reabre la cadena upstream congelada; -4. **cautela post-selección**: separa el teorema Markov del tightening - condicional y de las validaciones empíricas A7--A11. - -### Escalera de claims {#sec-crpto-claim-ladder} - -El paper se vuelve más fuerte cuando los claims se ordenan por nivel de -ambición. No todos tienen el mismo estatus. Algunos son teóricos, otros son -empíricos, otros son de ingeniería reproducible. - -| Nivel | Claim | Evidencia actual | Lo que no debe prometer | -|---|---|---|---| -| 1 | La PD base es suficientemente calibrada para alimentar incertidumbre | AUC `0.7124`, Brier `0.1546`, ECE `0.0064`, Venn-Abers | no afirmar que AUC es el aporte principal | -| 2 | Los intervalos conformales tienen cobertura útil para decisión | coverage 90 `92.97%`, min group coverage `91.90%` | no afirmar cobertura condicional perfecta | -| 3 | El intervalo puede convertirse en conjunto de incertidumbre | definición de `u_i(alpha)` y `Gamma_CP` | no tratar `Gamma_CP` como presupuesto ad hoc | -| 4 | El bound controla no-cobertura ponderada del funded set | `thm-conformal-feasibility` y `V` | no decir que controla directamente una PD latente sin supuesto adicional | -| 5 | Existe una frontera robusta finita en OOT | A35: `50,010` políticas semánticas deduplicadas y `27,508` elegibles all-alpha sobre piso; terminal `37,068/37,068` all-alpha passers | no decir que eso certifica una región continua ni un óptimo global | -| 6 | El body point pool93 es económico, no theorem-tight | retorno `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral exacto de pérdida `0.345084` | no mezclarlo con el endpoint de menor umbral ni con el max-return endpoint | -| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A35--A40, apéndice condicional, A25--A34 | no venderlo como validación live, causal o universal | - -: Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} - -La regla de escritura es simple: el cuerpo principal del paper debe vivir en los -niveles 1--6; el nivel 7 alimenta appendix, respuesta a reviewers y versión -journal. Si una frase no cabe en esta escalera, probablemente pertenece a la -agenda futura o a otro paper. - -### Aportes por audiencia - -El mismo resultado se lee distinto según quién revise el manuscrito. - -| Audiencia | Qué le importa | Qué debe ver rápido | Artefacto o sección | -|---|---|---|---| -| OR / Management Science | decisión bajo incertidumbre y trade-off robustez-retorno | CRPTO como puente CP -> RO -> portfolio | @sec-alpha-gamma-bound, @sec-crpto-results | -| ML / conformal prediction | garantía distribution-free y límites post-selección | Markov como claim principal, tightening condicional separado | `thm-conformal-feasibility`, `lem-conditional-tightening` | -| Crédito / riesgo | auditabilidad, funded set, governance | policy congelada, composición del funded set, segment checks | @sec-crpto-p1-evidence | -| Reviewer escéptico de DFL | por qué no basta SPO+ | SPO+ reduce regret, CRPTO compra cobertura auditable | @fig-p1-spo-regret, @sec-crpto-comparison | -| Reproducibilidad | trazabilidad y no cherry-picking | mapa claim -> artifact -> test | @tbl-crpto-claim-artifact-test | - -: Traducción del aporte por tipo de reviewer {#tbl-crpto-reviewer-audiences} - -Esta tabla es útil para escribir la introducción del paper. El paper no debe -presentarse como "ganamos todo"; debe presentarse como "resolvemos una dimensión -que otros enfoques dejan débil: decisión auditable bajo incertidumbre". - -### Qué aprendimos de la literatura - -La literatura conformal fundacional nos da el permiso teórico para construir -intervalos con cobertura finita bajo intercambiabilidad [1], [2]. RCPS, LTT y -CRC enseñan una lección más general: la predicción conformal puede controlar -riesgos definidos por el usuario, no solamente coverage marginal promedio [3]-- -[5]. Esa es la puerta conceptual para nuestro bound: el riesgo relevante no es -solo "falló el intervalo", sino "falló el intervalo en préstamos que realmente -financié". - -La robust optimization clásica enseña que la robustez tiene precio y que ese -precio puede medirse [6]. Los trabajos conformal robust optimization muestran -que los conjuntos conformales pueden alimentar problemas de optimización [7], -[8]. Predict-then-Calibrate es especialmente cercano porque separa predicción y -calibración para LP contextuales robustos [9]. Nuestra diferencia práctica es el -aterrizaje: crédito minorista, funded set ponderado, grilla finita exacta y -artefactos reproducibles. - -La literatura más reciente marca hacia dónde crecería una versión futura: -CROMS sugiere selección de modelos conformales con criterio de robust -optimization [10]; end-to-end conformal calibration y conformal risk training -apuntan a entrenar la calibración con la decisión aguas abajo [11], [12]; -multi-distribution robust CP y online CP via universal portfolios apuntan a -shift y streaming [13], [14]. Online DFL abre el comparador dinámico para -decisiones repetidas [16]. Ninguna de esas líneas invalida el paper actual; más -bien confirma que la dirección del paper es contemporánea y fértil. - -### Tabla de traducción conceptual - -| Concepto estadístico | Traducción en CRPTO | Lectura de negocio | -|---|---|---| -| `alpha` | nivel de tolerancia a no-cobertura | cuánto riesgo de cola acepta el comité | -| `PD_low`, `PD_high` | intervalo conformal por préstamo | rango defendible de riesgo | -| `u_i(alpha)` | cota superior usada por el LP | PD prudente para decisión | -| `Gamma_CP` | ancho conformal ponderado total | incertidumbre disponible antes de fijar la policy | -| `Gamma_int` | intervención prudencial usada por la policy | robustez efectivamente comprada | -| `Gamma_res` | ancho conformal residual | holgura que completa el certificado policy-aware | -| `V` | no-cobertura ponderada del funded set | cuánto default realizado quedó fuera de la cota | -| `violation` | exceso de riesgo ponderado sobre `tau` | incumplimiento del límite de portafolio | -| `alpha01_exact_pass` | check exacto de bound en OOT | semáforo para promoción paper-facing | -| `price_of_robustness` | retorno sacrificado por protegerse | costo económico de auditabilidad | - -: Diccionario operativo del CRPTO {#tbl-crpto-operational-dictionary} - -Esta traducción es una de las fortalezas del libro. En el paper final, varios de -estos términos pueden aparecer en una sola figura o tabla; aquí conviene -mantenerlos explícitos porque ayudan a que el lector no confunda métricas -predictivas, conformales y de portafolio. - -### Preguntas de reviewer y respuesta corta - -**¿Por qué no basta con reportar AUC?** -Porque el aporte está downstream. La PD es un insumo; la contribución ocurre -cuando la incertidumbre entra al optimizador y cambia el funded set. - -**¿SPO+ no es mejor porque reduce regret?** -SPO+ es un comparador fuerte y reduce regret en el benchmark. CRPTO no pretende -dominarlo en regret; compra auditabilidad, cobertura verificable y una región -finita exacta sobre la grilla de políticas declarada. Es otro eje de valor [15], -[16]. - -**¿El bound controla PD latente o default observado?** -El bound principal controla un target acotado observado `Y_i in [0,1]` y su -no-cobertura ponderada. Interpretarlo como PD latente requiere un supuesto -adicional. Esa cautela está explícita en @sec-alpha-gamma-bound. - -**¿Hay riesgo de post-selección?** -Sí, por eso el paper no vende el cierre `276k` como una garantía conformal nueva. -Lo presenta como evidencia empírica exacta de la policy promovida y lo refuerza -con A3, A9 y el apéndice condicional. - -**¿Por qué rank 2 y rank 3 no son winners si pasan exact eval?** -Porque el selector no solo mira bound de portfolio. Ranks 2 y 3 pasan la eval -exacta de portfolio, pero fallan el gate conformal por cobertura mínima de -grupo. Esa es precisamente la utilidad de A10: muestra que el selector no es una -excusa post-hoc. - -**¿Los stress tests reemplazan un dataset externo?** -No. A6 y A11 fortalecen la evidencia bajo reponderación y flips adversariales, -pero no reemplazan la réplica externa. Esa capa ya queda cubierta por A25--A34 -en Prosper y Freddie/Mendeley; lo que sigue fuera es una validación live o -prospectiva con protocolo sellado. - -### Qué puede entrar al paper final - -| Material del libro | Ubicación probable en paper | Motivo | -|---|---|---| -| Diagrama CRPTO | Figura 1 | explica el método en una sola vista | -| Escalera alpha -> `Gamma_CP` -> funded set | Teoría / método | une conformal y robust optimization | -| A35 pool93 finite-grid frontier | Resultados principales | demuestra cierre exacto del claim sobre la grilla finita declarada | -| P1 strict temporal holdout | Appendix o robustness checks | responde post-selección | -| Funded-set composition | Appendix | útil para reviewers de crédito y riesgo | -| A10 exact finalists | Appendix | defiende el selector conformal | -| A11 enhanced shift | Appendix | fortalece robustness interna; A25--A34 cubren réplica externa estática | -| A25--A34 external replication | cuerpo corto y appendix | responde la crítica de un solo dataset sin reabrir el champion | -| Referencias numeradas | Related work del paper | ayuda a pasar de libro a manuscrito | - -: Destino editorial probable del material del libro {#tbl-crpto-paper-placement} - -### Paquete journal-ready agregado - -Las paginas @sec-p1-manuscript-blueprint y @sec-p1-journal-appendix convierten -esta guia editorial en un paquete de escritura mas cercano al manuscrito. El -blueprint define venue target, abstract operativo, claims C1--C7, notacion unica -y plan de figuras/tablas. El appendix journal agrega A12--A34 y las figuras -paper-grade, incluida la nueva Figura 1 IJDS, generadas o publicadas desde artefactos congelados. - -| Nuevo material | Papel editorial | Artefacto | -|---|---|---| -| Blueprint del manuscrito | estructura paper-ready y mapa claim -> artifact -> test -> location | `14g-manuscript-blueprint.qmd` | -| Figura CRPTO limpia | Figura 1 IJDS | `crpto_fig1_journal_pipeline.png` | -| Figura alpha -> `Gamma_CP` -> funded set | puente teoria-metodo | `crpto_fig13_alpha_gamma_funded_set.png` | -| A35 frontier, A36--A39 pool93 y A40 point-PD | evidencia visual/tabular del cierre pool93 y costo comparable de robustez | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`; `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`; `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`; `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`; `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`; `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` | -| A12--A34 | robustness appendix journal | `scripts/build_crpto_journal_package.py`; `scripts/build_tail_satisficing_challenger_audit.py`; `scripts/build_tail_constrained_reoptimization.py`; `scripts/build_distribution_robustness_diagnostics.py`; `scripts/build_multidataset_external_replication.py`; `scripts/build_price_of_robustness_cross_dataset.py` | -| A25--A34 | réplica externa Prosper/Freddie | `models/crpto_multidataset_external_status.json` | -| Status del paquete | trazabilidad y fuente canonica | `models/crpto_journal_package_status.json`; `models/crpto_multidataset_external_status.json` | - -: Material journal-ready agregado al CRPTO {#tbl-crpto-journal-ready-package} - -La regla de jerarquia sigue intacta: si una tabla diagnostica nueva entra en -tension con `models/final_project_promotion.json`, gana la promocion oficial. -A12--A34 fortalecen robustez y escritura journal, pero no re-promueven el -champion. - -### Referencias numeradas locales - -[1] Vovk, Gammerman and Shafer (2005), *Algorithmic Learning in a Random -World*. Fundamento original de conformal prediction. - -[2] Angelopoulos and Bates (2023), *Conformal Prediction: A Gentle Introduction*. -Entrada pedagógica moderna para explicar cobertura distribution-free. - -[3] Bates et al. (2021), *Distribution-Free, Risk-Controlling Prediction Sets*. -Base RCPS para hablar de control de riesgo más allá de intervalos clásicos. - -[4] Angelopoulos et al. (2025), *Learn then Test*. Marco para calibrar -algoritmos predictivos con control de riesgo. - -[5] Angelopoulos et al. (2024), *Conformal Risk Control*. Generaliza control de -riesgo conformal para pérdidas monotónicas acotadas. - -[6] Bertsimas and Sim (2004), *The Price of Robustness*. Lenguaje clásico de -presupuesto de robustez y costo de protección. - -[7] Johnstone and Cox (2021), *Conformal Uncertainty Sets for Robust -Optimization*. Puente temprano entre CP y robust optimization. - -[8] Patel, Rayan and Tewari (2024), *Conformal Contextual Robust Optimization*. -Extiende la conexión CP-RO en problemas contextuales. - -[9] Sun, Liu and Li (2024), *Predict-then-Calibrate*. Comparador cercano para LP -contextuales robustos con calibración posterior. - -[10] Bao et al. (2025), *Optimal Model Selection for Conformalized Robust -Optimization (CROMS)*. Motiva el selector conformal decision-aware. - -[11] Yeh et al. (2025/2026), *End-to-End Conformal Calibration for Optimization -Under Uncertainty*. Dirección futura para calibración conformal integrada a -decisión. - -[12] Yeh et al. (2025), *Conformal Risk Training*. Motiva OCE/CVaR y control de -cola como extensión P2. - -[13] Yang and Jin (2026), *Multi-Distribution Robust Conformal Prediction*. -Motiva validación robusta ante múltiples fuentes o distribuciones. - -[14] Liu, Dobriban and Orabona (2026), *Online Conformal Prediction via -Universal Portfolio Algorithms*. Motiva recalibración online bajo streaming. - -[15] Elmachtoub and Grigas (2022), *Smart Predict, then Optimize*. Base de SPO+ -y decision-focused learning. - -[16] Capitaine et al. (2026), *Online Decision-Focused Learning*. Extiende DFL a -decisiones online y drift. - -[17] Powell (2026), *Sequential Decision Analytics and Modeling*. Da el lenguaje -CFA/VFA/DLA para clasificar CRPTO como política parametrizada auditable. diff --git a/docs/research/crpto_extended_evidence_cards_2026-06-06.md b/docs/research/crpto_extended_evidence_cards_2026-06-06.md deleted file mode 100644 index 8fdc0c8..0000000 --- a/docs/research/crpto_extended_evidence_cards_2026-06-06.md +++ /dev/null @@ -1,201 +0,0 @@ -# CRPTO Extended Evidence Cards - 2026-06-06 - -## Decision - -This memo consolidates the remaining high-value material for the CRPTO book, -master's thesis and IJDS appendix surface. The material is imported as -**evidence cards**, not as a second manuscript, not as a dependency on another -workspace, and not as a promotion protocol. - -The rule is deliberately conservative: - -1. CRPTO keeps the promoted pool93 IJDS body point and the current funded-set - claim. -2. Evidence cards can strengthen thesis chapters, appendix language, reviewer - defenses and future-work gates. -3. Evidence cards do not reopen the champion, add heavy solver dependencies, or - convert exploratory lanes into production claims. -4. Any future promotion requires a pre-declared claim target, evidence gate, - artifact sink and stop rule inside this repository. - -Canonical tables live in `reports/crpto/extended/`. - -## Post-Pool93 IJDS Closure - -After the pool93 closure, the IJDS manuscript should not import the full -sequential-decision laboratory as a second center of gravity. The paper-facing -surface is now: - -- **Body:** pool93 A35 return-bound frontier, exact funded-set certificate and - Theorem 1 under weighted funded-set validity. -- **Supplement:** A19 regret-auditability comparator, A20--A24 robustness and - source diagnostics, and A35--A39 selected-allocation pool93 audits. -- **Submission package:** claim matrix and reproduction notes that preserve the - A19 SPO+ numbering rule and the A35--A39 pool93 closure. -- **Thesis / extended evidence:** the full DFL, FICO-proxy, IFRS9/SICR, - CRC/CROMS-lite and source-governance material below. - -This means the extended cards remain useful, but none of them replaces A35--A39 -or adds a hidden promotion criterion. - -## Card Register - -| card_id | card | thesis use | IJDS use | status | -|---|---|---|---|---| -| EC01 | PyEPO 1.3.7 / DFL formal suite | DFL comparator appendix | SPO+ comparator context only | append for thesis | -| EC02 | FICO proxy versus calibrated champion | model-risk and score-governance appendix | context only if reviewer asks | append for thesis | -| EC03 | IFRS9/SICR prudential absorption | prudential uncertainty appendix | implication/limitation only | append for thesis | -| EC04 | CRC/LTT and CROMS-lite decision-risk governance | decision-risk governance appendix | reviewer-defense only | append or future gate | - -The machine-readable register is -`reports/crpto/extended/crpto_extended_evidence_cards_2026-06-06.csv`. - -## EC01 - PyEPO 1.3.7 / DFL Formal Suite - -**Claim target.** Show that modern decision-focused learning is a serious -regret comparator, while CRPTO remains the auditable conformal-risk method. - -**Evidence.** The curated PyEPO suite uses `pyepo==1.3.7`, -`gurobipy==13.0.2`, an exact top-k oracle for standard methods and a -Gurobi-backed binary oracle for CaVE. All smoke and nonnegative-regret checks -passed. - -| surface | best result | interpretation | -|---|---|---| -| Full DFL suite | SPO+ mean regret `0.122379`; RFYL `0.125405`; CaVE `0.128109` | SPO+ remains the regret leader; RFYL and CaVE are credible thesis comparators. | -| Temporal DFL suite | SPO+ mean regret `0.061835`; RFYL `0.071448`; CaVE `0.072284` | The DFL story survives a temporal surface. | -| Paired CRPTO comparator | SPO+ improvement `48.51%` versus two-stage | Supports the appendix claim already used by the IJDS supplement. | - -**Destination.** Thesis appendix and DFL comparator discussion. IJDS only keeps -the existing SPO+ comparator framing. - -**Boundary.** DFL can reduce regret, but it does not provide conformal coverage, -the exact funded-set bound, source-governance guarantees, or a replacement for -the CRPTO champion. - -**Stop rule.** Do not import solver logs, Gurobi artifacts, PyTorch/PyEPO -dependencies or heavy run directories. Reopen only if a reviewer asks for a -comparator table or the thesis needs a standalone DFL appendix. - -Canonical tables: - -- `reports/crpto/extended/crpto_extended_pyepo_dfl_execution_ledger_20260528.csv` -- `reports/crpto/extended/crpto_extended_pyepo_dfl_full_summary_20260528.csv` -- `reports/crpto/extended/crpto_extended_pyepo_dfl_temporal_summary_20260528.csv` - -## EC02 - FICO Proxy Versus Calibrated Champion Governance - -**Claim target.** Support the governance claim that the calibrated champion is -not merely a repackaged traditional score signal. - -**Evidence.** On the latest 40% of the OOT split (`n=103,865`), the calibrated -champion improves over the origin-time FICO proxy on both ranking and -calibration metrics: - -| metric | delta champion - FICO proxy | interpretation | -|---|---:|---| -| AUC | `+0.10757` | stronger discrimination | -| Gini / Somers' D | `+0.21514` | stronger rank ordering | -| Brier | `-0.010815` | better probabilistic loss | -| ECE 10-bin | `-0.022137` | better calibration | -| Decile-band MAE | `-0.020532` | lower decile calibration error | -| Rank shift >= 20pp | `0.486526` | nearly half the loans move materially | - -**Destination.** Model-risk, score-governance and thesis appendix. In IJDS this -is optional context only if a reviewer asks whether CRPTO depends on FICO-style -ranking. - -**Boundary.** This is not a legal fair-lending claim, protected-attribute -inference, or champion replacement claim. It is a score-governance diagnostic. - -**Stop rule.** Do not rerun unless a reviewer asks for score-governance -replication or a new external score dataset appears. - -Canonical table: - -- `reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv` - -## EC03 - IFRS9/SICR Prudential Absorption - -**Claim target.** Keep the prudential value of the IFRS9 lane inside the thesis -without turning IJDS into a second IFRS9 paper. - -**Evidence.** - -| prudential piece | retained value | boundary | -|---|---|---| -| ECL scenario propagation | baseline ECL `USD 870.3M`; severe ECL `USD 1.479B`; severe uplift `69.9%` | diagnostic scenario stress, not production allowance | -| Conformal ECL range | baseline point `USD 432.1M`; high conformal `USD 1.408B`; high/point ratio `3.26x` | uncertainty diagnostic, not audited regulatory capital | -| SICR conformal trigger | `t*=0.30`; recall of missed defaults `75.8%`; added ECL `USD 56.6M` | complementary signal, not contractual staging policy | -| Competing-risks correction | KM baseline about `USD 1.003B`; CIF-adjusted `USD 870.3M`; over-reserve about `USD 125.8M` | prepayment caveat, not a full IFRS9 engine | -| Stage threshold governance | minimum cost near `pd_threshold=0.15`; robust band `0.13--0.18` | thresholds are governance decisions | - -**Destination.** Thesis prudential appendix and CRPTO implications/future-work -section. IJDS can mention it as applied implication only. - -**Boundary.** IFRS9-inspired evidence is not contractual IFRS9. The repository -does not claim a production allowance model. - -**Stop rule.** Keep contractual IFRS9 claims false without monthly DPD history, -cure logic, EAD paths, recovery/prepayment timing and governed macro scenarios. - -Canonical table: - -- `reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv` - -## EC04 - CRC/LTT And CROMS-Lite Governance - -**Claim target.** Show that the project did not select a conformal method only -by visual preference or single-metric ranking; it also audited decision-risk -gates and selector tradeoffs. - -**Evidence.** - -- CRC/LTT-style gates over retained policies find `22` operational gate passes. -- One source-hardened candidate passes the stricter source-defense gate. -- The official CRPTO champion remains protected because the source-defense - screen is a governance diagnostic, not a promotion protocol. -- CROMS-lite exposes return versus source-defense tradeoffs: a source-defended - policy improves worst-source coverage by `+0.3636` but gives up - `USD 3,842.90` of realized return versus the official champion. - -**Destination.** Thesis decision-risk governance appendix and reviewer defense. -IJDS keeps A5/A10 as the official compact selector evidence. - -**Boundary.** These are screens over retained artifacts, not end-to-end CROMS -training, not a new conformal selector, and not a champion promotion protocol. - -**Stop rule.** Open a direct CRC/LTT decision-loss gate only with a declared -split and fixed monotone loss before outcomes. - -Canonical tables: - -- `reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv` -- `reports/crpto/extended/crpto_extended_decision_risk_governance_croms_lite_2026-05-18.csv` -- `reports/crpto/extended/crpto_extended_future_experiment_gate_register_2026-05-19.csv` -- `reports/crpto/extended/crpto_extended_strong_appendix_register_2026-06-06.csv` - -## Editorial Status - -| surface | action | -|---|---| -| IJDS body | No change. The body remains CRPTO: calibrated PD, conformal uncertainty, robust policy and funded-set certificate. | -| IJDS supplement | Already carries the compact comparator/limitation language through A19--A39. | -| Quarto CRPTO book | Add these cards to the controlled appendix and thesis/future-work map. | -| Master's thesis | Use the full evidence-card register to show breadth, maturity, negative results and stop rules. | -| Future experiments | Reopen only from `reports/crpto/extended/crpto_extended_future_experiment_gate_register_2026-05-19.csv`. | - -## Closed Or Parked Material - -The following remain outside the CRPTO/IJDS claim: - -- legal fair-lending certification; -- contractual IFRS9 allowance modeling; -- CATE policy value; -- online/live conformal deployment; -- exact Bellman optimality; -- DFL or PyEPO as champion replacement; -- GPU and quantum papers. - -This keeps the thesis broad without making the IJDS paper look like several -papers competing for the same center. diff --git a/docs/research/crpto_full_audit_2026-07-05.md b/docs/research/crpto_full_audit_2026-07-05.md deleted file mode 100644 index e041918..0000000 --- a/docs/research/crpto_full_audit_2026-07-05.md +++ /dev/null @@ -1,404 +0,0 @@ -# Auditoría integral CRPTO — 2026-07-05 - -> **Cierre semántico posterior (2026-07-09).** Las secciones de proceso y -> refactor de este memo conservan valor histórico. Para métricas del -> certificado, frontera A35 y baseline A40, prevalece -> `pool93_certificate_semantics_v2_2026-07-09.md`; ese cierre reemplaza el -> atajo lineal que este memo llamaba "Markov cap". - -Auditoría de consistencia de claims, parsimonia de código, skills, docs y plan -editorial del paper, de cara a la submission IJDS (target interno 2026-08-10). -Método: tres pasadas de exploración paralelas (código, claims, tooling/docs) -más verificación directa de fuentes. Corrección post-auditoría: al retomar, -F1/F2 estaban en el working tree pero no committeadas. Codex movió ese trabajo -a `codex/crpto-full-audit-closeout` y cerró F1–F5 con commits separados; P3 se -auditó como no-op por falta de una rama muerta segura en -`scripts/generate_conformal_intervals.py`. - -Alcance acordado con Carlos: el refactor incluye la capa protegida validando -con `just drift-gate`; la fuente editorial del paper es `paper/CRPTO_ijds.qmd` -(se porta a mano al `.tex`); el libro Quarto queda fuera de esta ronda. - -## 1. Resumen ejecutivo - -| Área | Veredicto | Estado | -|---|---|---| -| Claims paper/supplement/tex/tablas/JSONs | Limpio: 0 números stale en 15 claims verificados | Sin acción necesaria | -| `AGENTS.md` | Estaba stale (rebaseline como champion activo) | RESUELTO en F1: reducido a puntero | -| PDFs de submission | `supplement_ijds.pdf` stale (15-jun); PDF oficial sin garantía de frescura | RESUELTO en F1: recompilados y verificados | -| Skills `.claude/skills/` | No cargaban (formato plano) | RESUELTO en F1: migradas + 2 nuevas | -| Código | Sano; sin AI slop; F4 redujo helpers duplicados y F5 cerró el lane protegido seguro | Resuelto en F4/F5 | -| Docs | Memos cerrados mezclados con vigentes; la mayoría pinneados por referencias | Parcial en F2 (ver 7.2) | -| justfile / DVC / CI / pre-commit / configs | Coherentes, sin huérfanos | Sin acción necesaria | -| Escritura del paper | Sin slop de buzzwords; el riesgo real es densidad/repetición | F3 ejecutada | - -## 2. Consistencia de claims - -Los 15 claims cuantitativos del cuerpo se cotejaron entre `CRPTO_ijds.qmd`, -`supplement_ijds.qmd`, `submission/CRPTO_ijds_submission.tex`, las tablas -A35–A39, los JSONs de gobernanza pool93 y (spot-check) el libro. Referencia -canónica: `docs/research/active_claims_2026-07-04.md`. - -| Claim | Valor canónico | Estado | -|---|---|---| -| Retorno body point pool93 | $184,832.48 | OK en qmd, tex (`\$184{,}832.48`), A35, gobernanza | -| V(alpha=0.01) | 0.035350 | OK en todas las fuentes | -| Gamma_CP(alpha=0.01) | 0.162616 | OK; notación Gamma_CP vs gamma de política bien separada | -| Umbral exacto de pérdida (alpha=0.01) | 0.345084 | OK en la A35 policy-aware corregida | -| Endpoint budget upper B_u | 0.245084 = sum(w q) + Gamma_res | OK; en el body point la cota por fila está inactiva y el atajo lineal coincide numéricamente | -| Alpha grid | 8/8 screen pass; exceso realizado sobre tau 0.0 | OK | -| Return floor declarado | $170,464.54 | OK; en gobernanza y supplement, no como headline | -| Frontera consolidada | 50,010 semánticas dedup; 27,508 all-alpha sobre floor | OK | -| Búsqueda terminal | 37,068/37,068 passers; 296,544 checks | OK | -| Panel OOT | 276,869 préstamos | OK | -| Policy mode body | capped_blended_uncertainty (tau 0.1715, gamma 0.5475) | OK | -| Endpoint conservador | $170,467.27, cap 0.273036 | OK | -| Endpoint económico | $223,458.14 ("above $223K" en abstract) | OK | -| Replicaciones externas | Prosper/Freddie como transferencia, no certificados nuevos | OK | -| Métricas PD | AUC 0.7139, Brier 0.1544, ECE 0.0070 | OK | - -`tests/test_pool93_body_claim_sync.py` (242 líneas, 9 anclas) enforza la -sincronía manuscrito–gobernanza–tablas. La skill `/crpto-claim-sync` (creada -en F1) empaqueta ese gate más el cotejo por grep de esta tabla. - -## 3. Backlog de refactor (parsimonia) - -Cadena protegida (todo cambio pasa `just drift-gate` bit-exacto): -`scripts/train_pd_model.py` (2,341 líneas), `scripts/generate_conformal_intervals.py` -(1,870), `scripts/optimize_portfolio.py`, `scripts/validate_conformal_policy.py`, -`scripts/search/run_portfolio_bound_exact_eval.py`, `src/models/conformal/*`, -`src/models/optuna_tuning.py`, `src/models/conformal_tuning.py`, -`src/models/pd_model.py`, `src/models/calibration.py`. Por la regla 12 de -`CLAUDE.md`, cualquier archivo que matchee `src/models/conformal*.py` entra al -lane protegido aunque el cambio parezca trivial. - -No se encontró AI slop en código (sin emojis en logs, sin bloques comentados, -sin retry/defensive code de producción). SPO/PyEPO es comparador externo real, -no código vestigial. El detalle ejecutado de S1–S5 y P0–P3 está en la sección 8. - -## 4. Skills - -Hallazgo principal (resuelto en F1): las 6 skills vivían como -`.claude/skills/.md` plano y no cargaban en Claude Code actual, que -requiere `.claude/skills//SKILL.md`. Tras la migración, las 8 skills -cargan (verificado en vivo en la sesión del 2026-07-05). - -Inventario post-F1: - -| Skill | Estado | -|---|---| -| `/crpto-render` | Migrada; ahora referencia `just book-clean` para recovery | -| `/crpto-stage` | Migrada; ahora incluye nota de costo/tiempo de stages pesados | -| `/crpto-smoke` | Migrada sin cambios | -| `/crpto-paper-export` | Migrada sin cambios | -| `/crpto-validate-champion` | Migrada sin cambios | -| `/crpto-mrm-card` | Migrada; pre-check explícito de `skops` | -| `/crpto-claim-sync` | NUEVA: gate de sincronía de claims (3 pytest + grep de números canónicos en qmd/tex) | -| `/crpto-submission-freeze` | NUEVA: checklist ejecutable del freeze (gates, PDFs, anonimato, 25 páginas, ScholarOne) | - -`.codex/skills/crpto/SKILL.md` ya estaba al día (pool93) y no se tocó. - -## 5. Disposición de docs - -Hallazgo clave al ejecutar F2: **la mayoría de los memos "archivables" están -pinneados por referencias externas** — capítulos del libro los citan -(07, 09, 13, 14, 19, 23), `EXTRACTION_MANIFEST.json` hashea tres de ellos, -y tests/scripts/configs citan otros. Archivar un memo exige editar la -superficie que lo cita, y el libro está fuera de alcance en esta ronda. - -Movidos en F2 (cero referencias externas; `git mv` + índices actualizados): - -- `docs/refactor/NEXT_WORK_PLAN_2026-06.md` -> `docs/refactor/archive/` -- `docs/refactor/FEATURE_CONFIG_PARQUET_PLAN.md` -> `docs/refactor/archive/` -- `docs/research/paper4_crpto_crosswalk_2026-07-02.md` -> `docs/research/archive/` - -KEEP forzado por referencias (candidatos originales que NO se movieron): - -| Memo | Pinneado por | -|---|---| -| `crpto_conditional_tightening_appendix_2026-05-04.md` | `EXTRACTION_MANIFEST.json`, `configs/crpto_publication_targets.yaml`, `models/crpto_evidence_status.json`, `scripts/analyze_crpto_evidence.py`, libro 02/03/05 | -| `crpto_journal_package_2026-05-04.md` | `EXTRACTION_MANIFEST.json`, `scripts/build_crpto_journal_package.py`, test del journal package, libro 05/07 | -| `crpto_p1_evidence_2026-05-04.md` | `EXTRACTION_MANIFEST.json`, `tests/test_scripts/test_export_crpto_tables.py`, `scripts/analyze_crpto_evidence.py` | -| `crpto_publication_strategy_2026-05-12.md` | `README.md`, `paper/README.md`, `tests/test_publication_targets.py`, libro 06 | -| `SENSITIVITY_RUN_DESIGN_2026-06.md` | `paper/supplement_ijds.qmd:586` | -| `crpto_bound_tightening_experiment_2026-06-11.md` | `scripts/build_bound_tightening_audit.py` | -| `CONFORMAL_REFACTOR_PLAN.md` | comentarios en `src/models/conformal_diagnostics.py` y `tests/test_models/test_calibrator_pickle_compat.py` | -| reopen plan / tournament protocol / regret closure / bound intake / pyepo intake / extended evidence cards | capítulos del libro 07/09/13/14/19/23 | - -Si en la ronda post-submission se edita el libro, ese es el momento de mover -estos memos junto con sus citas. Mientras tanto son la superficie de -provenance que el libro enlaza — moverlos rompería más de lo que limpia. - -## 6. Plan editorial del paper (F3 cerrada) - -La prosa no tiene slop de buzzwords y los boundaries de claims son -disciplinados. El problema editorial real era densidad y repetición. Los cinco -puntos se cerraron en F3; el detalle de ejecución está en la sección 8.3: - -1. Abstract: ~230 palabras con 10+ números; dejar los canónicos y mover - denominadores de frontera al cuerpo. -2. $184,832.48 aparece 11 veces en el body; reducir donde no aporta. -3. Frase temprana con la regla de selección del body point. -4. Nombrar `test_pool93_body_claim_sync.py` en el párrafo de reproducibilidad. -5. Recompilar PDFs y re-verificar límite de 25 páginas. - -### Narrativa y defensa (contexto para la sesión editorial) - -El arco de contribución tal como está escrito funciona y no hay que -reinventarlo: (1) problema — la incertidumbre predictiva rara vez cambia el -funded set; (2) diseño — PD congelado + intervalos Mondrian + LP robusto como -capas modulares post-hoc; (3) teoría — el Teorema 1 separa la identidad -determinista del paso Markov bajo la Assumption 1 (weighted funded-set -validity); (4) evidencia — frontera finita pool93 con certificado exacto; -(5) transferencia — Prosper/Freddie como replicación de receta, no -certificados nuevos; (6) gobernanza — cadena bit-exacta y guardrail tests. - -Los dos únicos riesgos de lectura que encontró la auditoría, y que F3 cerró sin -debilitar nada más: - -1. Un reviewer puede leer el body point como cherry-picking o confundirlo con - el endpoint económico de $223K — por eso el punto 3 (regla de selección - enunciada temprano: mayor retorno pasando las ocho alphas bajo el cap - declarado 0.345, elegido de una frontera finita declarada). -2. "Exact" puede leerse como validez universal en vez de contabilidad sobre el - funded set congelado — el boundary ya está bien puesto (Teorema 1, - supplement, CLAIM_AUDIT_MATRIX); al editar, no suavizarlo ni moverlo. - -Checklist editorial final: `paper/submission/CLAIM_AUDIT_MATRIX.md` (mapa -claim→evidencia→boundary + banco de objeciones de reviewers) y -`paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md`. No introducir claims -nuevos: el reopen gate de `active_claims_2026-07-04.md` define las cinco -únicas condiciones que justificarían reabrir la búsqueda. - -## 7. Estado de ejecución - -### 7.1 F1 — Consistencia rápida (EJECUTADA Y COMMITTEADA 2026-07-05) - -- `AGENTS.md`: reducido de 243 líneas stale a un puntero de ~30 líneas hacia - `CLAUDE.md`, `active_claims_2026-07-04.md` y `.codex/skills/crpto/SKILL.md`, - con las reglas innegociables inline. -- Skills: 6 migradas vía `git mv` a `.claude/skills//SKILL.md` con - frontmatter `name:`; 2 nuevas creadas. Las 8 cargan (verificado en vivo). -- `CLAUDE.md`: sección de skills actualizada con las 8. -- PDFs: `just paper-submission-pdf` regeneró `paper/CRPTO_ijds.pdf` y - `paper/supplement_ijds.pdf`; `latexmk -pdf -gg` regeneró - `paper/submission/CRPTO_ijds_submission.pdf` (26 páginas, 3 corridas de - pdflatex + bibtex). Verificación de contenido con pypdf: el PDF contiene las - frases del último commit del `.tex` (fba278a). Matices descubiertos: - los tres PDFs están **gitignored por diseño** ("regenerate locally"), así - que la frescura es local y no genera diffs; y `latexmk` sin `-gg` reporta - "up-to-date" sin recompilar — usar siempre `-gg` para el freeze. -- Verificación: `just lint` (0 errores, 186 archivos formateados), `just smoke` - (5 pass), `just validate-champion` (10 pass), más - `test_pool93_body_claim_sync` + `test_supplement_table_sync` + - `test_publication_targets` (12 pass). - -### 7.2 F2 — Docs archive (EJECUTADA Y COMMITTEADA 2026-07-05, alcance reducido) - -Ejecutado: los 3 movimientos listados en la sección 5, más actualización de -referencias en `docs/refactor/README.md`, `docs/research/README.md`, -`docs/SCOPE_AND_GOVERNANCE.md` (lane FEATURE_CONFIG como ejecutado/archivado) -y `docs/research/foundations/crpto_decision_changes_and_learnings.md`. - -Reducción de alcance documentada: de ~15 candidatos originales, 12 quedaron -KEEP por la red de referencias (tabla en sección 5). Esto no es deuda: es la -constatación de que el repo usa los memos como superficie de citas. - -## 8. Registro detallado de fases ejecutadas - -### 8.1 Cómo auditar o retomar trabajo futuro - -1. Leer `CLAUDE.md`, este memo (secciones 5–8) y - `docs/research/active_claims_2026-07-04.md`. Con eso alcanza; no hace falta - reconstruir la auditoría ni releer los reportes de exploración. -2. Para trabajo nuevo, no mezclar superficies en un commit: prosa, código no - protegido y código protegido siguen lanes separados. -3. Nada de este cierre requirió re-correr stages DVC protegidos. Si una tarea - futura parece requerirlo, está mal planteada: parar y preguntar. -4. Código/refactor va en rama + PR (regla 11 de `CLAUDE.md`). -5. Commitear con hooks activos (nunca `--no-verify`); los hooks de pre-push - corren smoke + validate-champion. - -### 8.2 Mapa de gates (qué verifica cada comando y cuánto cuesta) - -| Gate | Verifica | Costo | Obligatorio en | -|---|---|---|---| -| `just lint` | ruff check + format | segundos | todo commit | -| `just smoke` | sync final del paper + guardrails del libro (5 tests) | ~30 s | todo commit | -| `just validate-champion` | hashes SHA256 vs `EXTRACTION_MANIFEST.json` (10 tests) | segundos | tras cualquier cambio que produzca outputs | -| `/crpto-claim-sync` | 3 tests de sincronía + grep de números canónicos en qmd/tex | ~1 min | tras editar paper/supplement (F3) | -| `just test-fast` | suite completa sin marks `slow` | minutos | F4 | -| `just bound-audit` | menú de bounds A21 + búsquedas bound-aware | minutos | si se toca optimización (S5) | -| `just drift-gate` | recomputa la cadena del certificado y exige diff bit-exacto; re-puntúa ~514k filas dos veces | largo | baseline antes de F5 y cada commit de F5 | - -### 8.3 F3 — Editorial del paper (cerrada 2026-07-05) - -Fuente de edición: `paper/CRPTO_ijds.qmd` y `paper/supplement_ijds.qmd`; -cada cambio se porta a mano a `paper/submission/CRPTO_ijds_submission.tex` -(el comentario del `.tex` documenta ese contrato). Solo prosa — ningún número -congelado cambia. - -1. **Abstract** (qmd y tex): hoy incluye retorno, V, Gamma_CP, cap, violación, - 50,010, 27,508, endpoint 0.273036, ">$223K", premios externos +1.0/+9.5% y - panel 276,869. Dejar: panel, retorno, 8/8 con (V, Gamma_CP, cap) y una - frase de endpoints sin cifras finas. Mover los denominadores 50,010/27,508 - a introducción/resultados (ya están ahí; basta con quitarlos del abstract). -2. **Repetición del headline**: `\$184{,}832.48` aparece en tex líneas ~133, - 361, 632 (tabla certificado), 686, 705 (tabla frontera), 896, 911, 1058 - (conclusión) y en el abstract. Conservar abstract, tabla certificado, - tabla frontera y conclusión; en el resto usar "the selected body point" o - "$184.8K". Mismo criterio en el qmd (11 ocurrencias). -3. **Regla de selección del body point**: añadir una frase en la introducción - (cerca de la primera mención del pool93): el body point es la política de - mayor retorno que pasa las ocho alphas declaradas bajo el cap declarado - 0.345 — seleccionada de una frontera finita declarada, no un máximo global - ni el endpoint económico de $223K. Redacción base en - `active_claims_2026-07-04.md`, sección "How To Present The Denominators". -4. **Reproducibilidad**: en el párrafo de gobernanza del body (tex ~L560-573, - añadido en fba278a) ya se habla de "guardrail tests"; nombrar explícitamente - `tests/test_pool93_body_claim_sync.py` en qmd y tex. -5. **Cierre**: `just paper-submission-pdf`; en `paper/submission/`, - `latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex` - (exigir "Output written", no "up-to-date"); verificar <= 25 páginas de body - excluyendo referencias (última build: 26 páginas totales, referencias - arrancan en p. 23). - -Gate de salida: `/crpto-claim-sync` verde + `just smoke`. Si un número -divergió, la edición se revierte — F3 no puede cambiar valores. - -Trampas conocidas de F3: - -- `tests/test_pool93_body_claim_sync.py` ancla strings exactos en qmd/tex; si - una edición elimina la única ocurrencia de un ancla, el test falla — la - respuesta es reponer el número, nunca ajustar el test. -- Las tablas A35–A39 en `reports/crpto/tables/` son congeladas: el pipeline de - export no las regenera y ninguna edición editorial debe tocarlas. -- El abstract vive dos veces: `\ABSTRACT{...}` en el tex (~L66–90) y el bloque - inicial del qmd. Editar ambos o el claim-sync reporta divergencia. -- Formatos al portar qmd->tex: `$184,832.48` vs `\$184{,}832.48`; `Γ_CP` vs - `\Gamma_{\mathrm{CP}}`. El grep del claim-sync ya cubre ambas variantes. -- El supplement se edita en `paper/supplement_ijds.qmd`; su PDF es HTML-print - (`just paper-ijds-supplement-pdf`), no LaTeX. - -### 7.3 F3/F4/F5 — Cierre Codex 2026-07-05 - -Estado real de Git al retomar: F1/F2 estaban en el working tree, no -committeadas, pese a que este memo decía "committeadas". Codex movió el trabajo -a la rama `codex/crpto-full-audit-closeout` antes de tocar código. - -F3 ejecutada: - -- Abstract QMD/TEX comprimido: se quitaron denominadores de frontera, endpoint - fino y porcentajes externos; quedan panel, retorno redondeado, `8/8`, - `V`, `Gamma_CP`, cap y una frase de endpoints. -- Introducción QMD/TEX reforzada con la regla de selección del body point: - mayor retorno que pasa las ocho alphas bajo el cap Markov declarado `0.345` - dentro de una frontera finita, no óptimo global ni endpoint económico. -- Reproducibilidad QMD/TEX nombra `tests/test_pool93_body_claim_sync.py`. -- Se redujeron repeticiones exactas de `$184,832.48` fuera de certificado, - frontera y conclusión. -- PDFs regenerados: `just paper-submission-pdf` OK; `latexmk` falló por el - wrapper TinyTeX (`runscript.tlu` nil), así que se compiló el `.tex` oficial - con `pdflatex`, `bibtex`, `pdflatex`, `pdflatex`. Resultado: 26 páginas - totales; referencias arrancan en p. 22, body = 21 páginas. - -F4 ejecutada: - -- S1: migrados helpers duplicados en scripts paper-facing no protegidos hacia - `src.utils.script_helpers`. `write_table` ahora acepta `float_precision` para - preservar salidas `.tex` de 4 vs 6 decimales. `just tables` no dejó cambios - de contenido en tablas; `just figures` solo produjo ruido PDF/timestamp y se - descartó. -- S2 KEEP: `src/utils/pipeline_topology.py` no se archiva porque - `scripts/search/run_conformal_reopen_search.py` importa `load_profile_config`. -- S3 ejecutada: `src/evaluation/calibration_mapping.py` y - `tests/test_evaluation/test_calibration_mapping.py` retirados; los únicos - usos vivos restantes son referencias a los artefactos históricos en docs/book. -- S5 KEEP: `src/optimization/cuopt_adapter.py` no se archiva porque está - conectado a `src/optimization/portfolio_model.py`, perfiles `cuopt` y scripts - de búsqueda/experimentos. - -Gates F4: `just lint`, `just test-fast` y `just validate-champion` en verde. - -F5 ejecutada/auditada: - -- Baseline previo: `just validate-champion` y `just drift-gate` en verde. -- P0: eliminado `allow_legacy_fallback` de - `src/models/conformal_artifacts.py`, sus call-sites y test. -- P1: `scripts/optimize_portfolio.py` y `scripts/train_pd_model.py` adoptan el - `artifact_path` compartido. `generate_conformal_intervals.py` y - `validate_conformal_policy.py` no tenían helper local equivalente que migrar. - `_write_json` en `train_pd_model.py` se conservó porque envuelve - `atomic_write_json`, no el writer LF-idempotente de publicación. -- P2: retirados los dataclasses privados single-use de `train_pd_model.py` - (`ResolvedFeatureSets`, `TrainingSplits`, `PreparedTrainingInputs`) y - reemplazados por tuplas desempaquetadas en el mismo flujo. -- P3: auditado sin cambio. `fallback_modes`, `evaluation_scope`, - `shrinkback_enabled`, `global_rebalance_enabled`, fuentes de probabilidad y - familias de escala están vivos por CLI/perfiles/tests/reopen; no se encontró - rama muerta segura para remover sin cambiar superficie operacional. - -Gates F5: `just drift-gate` verde con diff cero tras P0, P1/portfolio, -P1/PD y P2; checks enfocados de Ruff/py_compile en verde. - -### 8.4 F4 — Refactor lane seguro (cerrada 2026-07-05) - -Estado: ejecutada por Codex con alcance conservador. No quedan acciones F4 -seguras pendientes; los hallazgos S2/S5 se convierten en KEEP por uso vivo. La -deduplicación protegida segura se cerró después en F5/P1. - -- **S1 — script_helpers en scripts no congelados**: ejecutado para scripts - paper-facing no protegidos. Validación: `just tables`, `just figures`, - `just lint`, `just test-fast`, `just validate-champion`. -- **S2 — archivar `src/utils/pipeline_topology.py`**: no ejecutado; el grep - encontró uso vivo en `scripts/search/run_conformal_reopen_search.py`. -- **S3 — archivar/inlinear `src/evaluation/calibration_mapping.py`**: - ejecutado; el grep por símbolos encontró solo su test. -- **S5 — evaluar `src/optimization/cuopt_adapter.py`**: no ejecutado; el grep - encontró uso vivo en `portfolio_model.py`, perfiles `cuopt` y scripts. -- **S4 movido al lane protegido**: quitar `allow_legacy_fallback` de - `src/models/conformal_artifacts.py` matchea `conformal*.py` (regla 12), así - que exigía drift-gate aunque fuera trivial. Cerrado después como P0 de F5. - -Decisión considerada y rechazada: deduplicar `book/_helpers/` contra -`script_helpers` — el aislamiento del libro vale más que ~40 líneas. - -Por qué importa la identidad byte a byte en S1: varios outputs de estos -scripts están hasheados en `EXTRACTION_MANIFEST.json` o comparados por tests -de sync, y los writers de `script_helpers` son LF-idempotentes en Windows -precisamente por eso. Si `git diff` muestra cambios de contenido tras -`just tables && just figures`, el refactor rompió algo: revertir y revisar, -no re-hashear. - -### 8.5 F5 — Refactor lane protegido (cerrada 2026-07-05) - -Estado: cerrada por Codex en `codex/crpto-full-audit-closeout` siguiendo el -protocolo **un cambio -> un `just drift-gate` verde -> un commit**. Cada corrida -de `drift-gate` reportó diff cero en `y_pred`, intervalos PD, score-band edges, -coberturas por celda y floor multipliers. - -- **P0**: ejecutado. Se eliminó `allow_legacy_fallback` (deprecado) de - `src/models/conformal_artifacts.py`, call-sites y test. -- **P1**: ejecutado donde había helper real que migrar. `optimize_portfolio.py` - y `train_pd_model.py` usan `src.utils.script_helpers.artifact_path`; - `generate_conformal_intervals.py` y `validate_conformal_policy.py` no tenían - `_artifact_path`/writer/loader local equivalente. El wrapper `_write_json` de - `train_pd_model.py` permanece por semántica atómica (`atomic_write_json`). -- **P2**: ejecutado. Los dataclasses privados single-use de `train_pd_model.py` - (`ResolvedFeatureSets`, `TrainingSplits`, `PreparedTrainingInputs`) fueron - sustituidos por tuplas privadas y desempaquetado inmediato. -- **P3**: auditado sin cambio. No se encontró rama muerta segura en - `generate_conformal_intervals.py`: los modos/fallbacks sospechosos están - conectados a CLI, perfiles, tests o reopen search; removerlos sería cambio de - superficie, no limpieza mecánica. - -Límites duros: no mover/renombrar clases que el calibrator pickle referencia -(compatibilidad documentada en `CONFORMAL_REFACTOR_PLAN.md` y -`tests/test_models/test_calibrator_pickle_compat.py`); no tocar -`EXTRACTION_MANIFEST.json`; si un cambio pide re-keying de DVC, es señal de -alcance excedido: parar y pedir permiso explícito. - -Nota operativa para futuros lanes protegidos: mantener el mismo protocolo de -drift-gate por commit. Si un cambio pide re-keying de DVC o toca -`EXTRACTION_MANIFEST.json`, es alcance excedido: parar y pedir permiso -explícito. diff --git a/docs/research/crpto_journal_package_2026-05-04.md b/docs/research/crpto_journal_package_2026-05-04.md deleted file mode 100644 index 272efac..0000000 --- a/docs/research/crpto_journal_package_2026-05-04.md +++ /dev/null @@ -1,100 +0,0 @@ -# paper-crpto Journal Package - 2026-05-04 - -This dossier records the journal-oriented tables and figures generated from -frozen CRPTO artifacts. It does not reopen the champion search. - -## Standalone Scope - 2026-05-12 - -This package is the journal/appendix layer for `Paper_CRPTO`. It is intentionally -larger than the short paper: A12--A34, Figures 1, 12--25 and the robustness notes -can be selected into a journal appendix, reviewer response or future thesis -chapter without changing the official champion. -A20--A21 are generated by `scripts/build_tail_satisficing_challenger_audit.py` -as a slower journal-only add-on. -A25--A34 and Figures 22--25 are generated by -`scripts/build_multidataset_external_replication.py` from curated local summaries. -A34 and Figure 25 are generated by `scripts/build_price_of_robustness_cross_dataset.py` -and `scripts/generate_crpto_figures.py`. - -## Generated artifacts - -- `reports/crpto/tables/crpto_tableA12_tail_risk_oce_cvar.csv` -- `reports/crpto/tables/crpto_tableA12_tail_risk_oce_cvar.tex` -- `reports/crpto/tables/crpto_tableA13_satisficing_margins.csv` -- `reports/crpto/tables/crpto_tableA13_satisficing_margins.tex` -- `reports/crpto/tables/crpto_tableA14_dependency_cluster_diagnostics.csv` -- `reports/crpto/tables/crpto_tableA14_dependency_cluster_diagnostics.tex` -- `reports/crpto/tables/crpto_tableA15_leave_one_period_stress.csv` -- `reports/crpto/tables/crpto_tableA15_leave_one_period_stress.tex` -- `reports/crpto/tables/crpto_tableA16_bootstrap_funded_set_metrics.csv` -- `reports/crpto/tables/crpto_tableA16_bootstrap_funded_set_metrics.tex` -- `reports/crpto/tables/crpto_tableA17_budget_cap_lgd_sensitivity.csv` -- `reports/crpto/tables/crpto_tableA17_budget_cap_lgd_sensitivity.tex` -- `reports/crpto/tables/crpto_tableA18_robust_region_policy_family.csv` -- `reports/crpto/tables/crpto_tableA18_robust_region_policy_family.tex` -- `reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv` -- `reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.tex` -- `reports/crpto/figures/crpto_fig1_journal_pipeline.png` -- `reports/crpto/figures/crpto_fig1_journal_pipeline.pdf` -- `reports/crpto/figures/crpto_fig1_journal_pipeline.svg` -- `reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png` -- `reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.pdf` -- `reports/crpto/figures/crpto_fig20_bound_claim_layers.png` -- `reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf` -- `reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.png` -- `reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf` -- `reports/crpto/figures/crpto_fig14_robust_region_heatmap.png` -- `reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf` -- `reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png` -- `reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf` - -## Scope notes - -- A12--A34 are diagnostic robustness, comparator, replication and packaging tables. -- A20--A21 are diagnostic challenger and cluster-bound audit tables from - the separate tail-satisficing audit script. -- A25--A34 are external economic replication diagnostics; they do not reopen - or replace the official Lending Club champion. -- Budget and segment-cap sensitivity are funded-set diagnostics, not - re-optimized portfolios. -- Tail-risk and bootstrap return columns are funded-set repricing diagnostics; - the official champion return remains sourced from `final_project_promotion.json`. -- `src/optimization/tail_satisficing_objective.py` exposes the same - OCE/CVaR/satisficing primitives as a future-experiment scaffold, - but this package still does not promote a new objective. -- The official champion remains `bound_aware_276k_economic_champion`. - -## Appendix map - -| Artifact | Purpose | Caveat | -|---|---|---| -| A12 tail risk OCE/CVaR | Adds funded-set tail-risk diagnostics under LGD alternatives. | Diagnostic repricing, not a new champion metric. | -| A13 satisficing margins | Converts champion evidence into OR-style pass/margin checks. | Editorial thresholds must be justified if moved to body. | -| A14 dependency clusters | Documents period/grade concentration for the tightening appendix. | Does not prove conditional independence. | -| A15 leave-one-period stress | Reweights the funded set by period. | Not a re-optimized portfolio. | -| A16 bootstrap funded-set metrics | Adds empirical intervals for realized funded-set quantities. | Not a conformal guarantee. | -| A17 budget/LGD/cap sensitivity | Checks practical sensitivity to budget, LGD and segment caps. | Cap checks are diagnostics, not solver constraints. | -| A18 robust region by family | Summarizes the `45/45` alpha01-safe region by `risk_tolerance x gamma`. | Bound-aware family only. | -| A19 regret-auditability frontier | Compares two-stage, SPO+ and CRPTO robust on regret versus verifiable risk controls. | Trade-off diagnostic, not a new champion selector. | -| A20 tail-risk robust-region audit | Re-solves the 45 alpha-safe policies and ranks CVaR/OCE/return trade-offs inside the robust region. | Generated separately; audit only, no champion promotion. | -| A21 cluster-bound tightening | Quantifies cluster-aware Hoeffding thresholds. | Transparent caveat, not tighter than Markov here. | -| A22 tail-constrained re-optimization | Turns CVaR/OCE into an active tail constraint over the 45-policy robust region. | Tail-constrained challenger, not a new champion. | -| A23 multi-distribution robustness | Reports worst-case coverage by grade and grade x vintage. | Diagnostic stress, not a new calibration protocol. | -| A24 online conformal stability | Replays OOT vintages with ACI-style alpha updates. | Static replay, not live deployment evidence. | -| A25 external replication gate | Reports Prosper and Freddie/Mendeley scoring, conformal and LP gates. | External replication evidence, not a new theorem. | -| A26 external candidate sensitivity | Audits robust LP stability as the candidate pool grows. | Candidate-pool stress only. | -| A27 Freddie horizon sensitivity | Audits the Freddie default-window choice before promoting FM48. | Dataset-level selection audit only. | -| A28 external LP exhaustiveness | Solves Prosper all-candidate and Freddie 500k/1M/all candidate LPs. | Exhaustiveness audit, not a new theorem. | -| A29 Freddie sparse Mondrian audit | Splits Freddie coverage by all, eligible and sparse groups. | Sparse-cell caveat only. | -| A30 external metric intervals | Adds intervals for external AUC, coverage, alpha coverage and robust objective. | Bootstrap uncertainty, not conformal validity. | -| A31 external OOT subperiod metrics | Breaks external metrics by OOT year or quarter. | Subperiod audit only. | -| A32 Prosper default-definition sensitivity | Repeats Prosper under alternate final-status default definitions. | Semantics audit only. | -| A33 Freddie segment sensitivity | Repeats Freddie FM48 for red, green and combined groups. | Segment sensitivity only. | -| A34 cross-dataset price of robustness | Reports how the external robustness premium scales with panel default rate. | Frozen application readout, not champion promotion. | - -## Quarto integration - -- `book/chapters/06-blueprint-manuscrito.qmd` uses - these artifacts to define the paper outline and final table/figure plan. -- `book/chapters/07-apendice-robustez.qmd` - renders A12--A34 and Figures 1, 12--25 plus the bound-claim stack figure. diff --git a/docs/research/crpto_p1_evidence_2026-05-04.md b/docs/research/crpto_p1_evidence_2026-05-04.md deleted file mode 100644 index 9493708..0000000 --- a/docs/research/crpto_p1_evidence_2026-05-04.md +++ /dev/null @@ -1,67 +0,0 @@ -# paper-crpto P1 Evidence - 2026-05-04 - -This dossier records the P1 evidence now materialized around the official -`ijds-rebaseline-2026-06-07` champion. It does not reopen the -champion search. - -## Standalone Scope - 2026-05-12 - -The evidence here is part of the independent paper-crpto dossier. It can be -rendered, audited and cited from the standalone Quarto book, but it should still -be read as evidence around the frozen champion rather than a new search. - -## Generated artifacts - -- `reports/crpto/tables/crpto_tableA3_nested_holdout.csv` -- `reports/crpto/tables/crpto_tableA3_nested_holdout.tex` -- `reports/crpto/tables/crpto_tableA4_segment_period_sensitivity.csv` -- `reports/crpto/tables/crpto_tableA4_segment_period_sensitivity.tex` -- `reports/crpto/tables/crpto_tableA5_decision_aware_selector.csv` -- `reports/crpto/tables/crpto_tableA5_decision_aware_selector.tex` -- `reports/crpto/tables/crpto_tableA6_synthetic_shift.csv` -- `reports/crpto/tables/crpto_tableA6_synthetic_shift.tex` -- `reports/crpto/tables/crpto_tableA7_funded_set_loans.csv` -- `reports/crpto/tables/crpto_tableA7_funded_set_loans.tex` -- `reports/crpto/tables/crpto_tableA8_funded_set_composition.csv` -- `reports/crpto/tables/crpto_tableA8_funded_set_composition.tex` -- `reports/crpto/tables/crpto_tableA9_strict_temporal_holdout.csv` -- `reports/crpto/tables/crpto_tableA9_strict_temporal_holdout.tex` -- `reports/crpto/tables/crpto_tableA10_conformal_finalist_exact_bound_eval.csv` -- `reports/crpto/tables/crpto_tableA10_conformal_finalist_exact_bound_eval.tex` -- `reports/crpto/tables/crpto_tableA11_enhanced_synthetic_shift.csv` -- `reports/crpto/tables/crpto_tableA11_enhanced_synthetic_shift.tex` -- `docs/research/crpto_conditional_tightening_appendix_2026-05-04.md` -- `models/crpto_evidence_status.json` -- `docs/research/crpto_p1_evidence_2026-05-04.md` - -## Scope notes - -- The nested-holdout evidence is an artifact-level staged confirmation - chain: 5K screening, 25K refinement, and 276K full OOT confirmation. It - is complemented by a strict temporal funded-set confirmation split in - `crpto_tableA9_strict_temporal_holdout.csv`. That strict split evaluates - the frozen policy; it does not reopen the champion search. -- The decision-aware conformal selector is a CROMS-style screen over the - three conformal finalists. Exact 276K bound-aware evaluations now exist - for ranks 1, 2 and 3, while ranks 2 and 3 still fail the conformal policy - gate through minimum group coverage. -- Synthetic shift checks include both covariate reweighting and adversarial - label-flip stress scenarios on OOT labels. They are stronger than the - first pass, but they are still not an external dataset replacement. - -## Key status - -- Nested final return: `170464.542928`. -- Nested final V: `0.028875`. -- Decision-aware selected rank: `1`. -- Worst segment coverage 90: `0.903203`. -- Worst synthetic coverage 90: `0.929714`. - -## Hardening status - -- `strict_temporal_holdout`: `implemented`. -- `funded_set_export`: `implemented`. -- `funded_set_composition`: `implemented`. -- `conformal_finalist_exact_eval`: `implemented`. -- `enhanced_synthetic_shift`: `implemented`. -- `conditional_tightening`: `implemented_as_conditional_appendix`. diff --git a/docs/research/crpto_publication_strategy_2026-05-12.md b/docs/research/crpto_publication_strategy_2026-05-12.md deleted file mode 100644 index c13e3ff..0000000 --- a/docs/research/crpto_publication_strategy_2026-05-12.md +++ /dev/null @@ -1,187 +0,0 @@ -# paper-crpto Publication Strategy - 2026-05-12 - -> Historical note, 2026-07-04: this memo predates the pool93 promotion. Use -> `docs/research/active_claims_2026-07-04.md` and -> `docs/SCOPE_AND_GOVERNANCE.md` for active paper-facing metrics. Mentions of -> `45/45` and the earlier champion are retained as venue-strategy provenance, -> not as the current IJDS headline. - -## Decision - -Write the first submission for **INFORMS Journal on Data Science (IJDS)** and -keep **European Journal of Operational Research (EJOR)** as the primary pivot. - -The working rule is: - -- primary venue: IJDS; -- secondary venue: EJOR; -- stretch venues: INFORMS Journal on Optimization, Management Science and - Operations Research; -- applied fallbacks: Decision Support Systems and Expert Systems with - Applications. - -This decision is encoded in `configs/crpto_publication_targets.yaml`. - -## Why IJDS First - -IJDS is the best first target because paper-crpto is not just a credit-risk -case study. The contribution is a reproducible decision-focused data science -pipeline: calibrated PD, conformal uncertainty, robust optimization, -artifact-backed tables, DVC lineage and a public companion. - -Official IJDS guidance is unusually aligned with the project: - -- initial submissions should fit a 25-page IJDS-style body, excluding - references and appendices; -- appendices and lengthy robustness material should be online supplements; -- submissions use the IJDS LaTeX template; -- IJDS uses double-anonymous review for submissions on or after - January 1, 2025; -- data/code disclosure and reproducibility are explicit parts of the process. - -Sources: - -- IJDS submission guidelines: - -- IJDS data/code disclosure policy: - - -## Why EJOR Second - -EJOR is the strongest second target if the paper reads more as operational -research than data science. Its scope is explicitly about both OR methodology -and decision-making practice. CRPTO can fit as an innovative OR application or -theory/methodology paper if we emphasize conformal uncertainty sets, robust -portfolio decisions, price of robustness and the active finite-grid pool93 -return-bound frontier. - -Source: - -- EJOR journal page and aims/scope: - - -## What This Means For The Manuscript - -The first paper draft should be written as: - -- title: `CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio Decisions`; -- body: 25-page IJDS-style manuscript; -- supplement: A3--A40, proofs, extended tables, reproducibility, - MRM/fairness and external replication; -- review mode: anonymous by default; -- companion: GitHub/DVC/DagsHub/MLflow after the anonymity policy is handled. - -The short paper should keep only the strongest body material: - -- problem and contribution; -- PD calibration summary; -- Mondrian conformal layer; -- robust portfolio formulation; -- policy-aware decision certificate and conditional tightening caveat; -- pool93 body metrics, A35 finite-grid frontier and A40 matched point-PD audit; -- A36--A39 selected-allocation audits; -- regret-auditability frontier with one concise SPO+/DFL comparison; -- data/code reproducibility statement. - -Everything else lives in the supplement or book. - -## Submission Scope Lock - -The current paper is centered on the promoted pool93 finite-grid body claim, -with the frozen upstream CRPTO chain retained as provenance and the declared -return floor. P2/P3 ideas are no longer treated as a blanket exclusion when -they use frozen artifacts and do not change the promoted policy: OCE/CVaR as a -diagnostic, satisficing as margin evidence, regret-auditability as the -SPO+/CRPTO comparator, and dependence-aware theory as a caveated supplement -proposition. - -In scope for the current paper: - -- promoted pool93 finite-grid return-bound frontier; -- frozen upstream CRPTO chain as provenance and return floor; -- calibrated PD -> Mondrian conformal intervals -> robust portfolio decision; -- exact funded-set validation and A35 policy-aware finite-grid frontier; -- A3--A40 as supplement evidence; -- regret-auditability frontier in the body; -- OCE/CVaR tail-risk diagnostics and robust satisficing margins in the - supplement; -- cluster-aware dependence caveat/proposition with the Markov step retained in - the main distribution-free decision certificate; -- external economic replication on Prosper and Freddie/Mendeley as A25--A34, - without reopening the Lending Club body claim; -- reproducibility via Quarto, DVC, DagsHub/MLflow and guardrail tests. - -Out of scope for the current paper: - -- OCE/CVaR as the optimized objective or constraint; -- multi-distribution robust conformal prediction as the promoted layer; -- online conformal recalibration or online DFL; -- SPO+ + conformal hybrid training; -- prospective/live multidataset validation beyond the static Prosper/Freddie - replications; -- causal/CATE CRPTO; -- multi-period portfolio rebalancing; -- field trial, production monitoring dashboard, or open-source package - extraction. - -If any of those become central, the project moves to a new run/protocol rather -than quietly expanding the current manuscript. - -## Template Policy - -Do not optimize the final PDF template before the venue is fixed. The repo now -uses Quarto skeletons for writing: - -- `paper/CRPTO_ijds.qmd`: IJDS first-submission body; -- `paper/supplement_ijds.qmd`: IJDS online supplement; -- `paper/CRPTO.qmd`: generic landing manuscript stub. - -When the text is ready for submission, convert or render the IJDS `.qmd` into -the official IJDS LaTeX template rather than inventing a custom journal style. -The author portal/Overleaf template uses `\documentclass[ijds,dblanonrev]{informs4}` -for double-anonymous review, so the conversion target is explicit. - -## Pivot Rules - -Stay with IJDS if the paper's strongest claim is: - -- reproducible decision-focused data science; -- auditability and code/data disclosure; -- a complete pipeline from prediction to decision. - -Pivot to EJOR if the paper's strongest claim becomes: - -- robust optimization methodology; -- operational decision practice; -- applied OR evidence and sensitivity analysis. - -Pivot to INFORMS Journal on Optimization only if a stronger optimization or -dependence-aware theory contribution is added. - -Treat Management Science and Operations Research as stretch targets, not the -first submission path, unless the manuscript gains broader managerial or OR -theory significance beyond the current Lending Club study. - -## Anonymity And Public Repo - -The repository is public, but IJDS requires double-anonymous manuscripts. Before -submission: - -- remove author names from the manuscript and supplement; -- avoid first-person claims that reveal ownership of the public repo; -- decide whether the GitHub/DagsHub companion is disclosed in the cover letter, - supplement, or only after review according to journal policy; -- create a release tag for the reproducibility bundle once anonymity handling is - settled. - -## Current Implementation - -- Config: `configs/crpto_publication_targets.yaml`. -- Body skeleton: `paper/CRPTO_ijds.qmd`. -- Supplement skeleton: `paper/supplement_ijds.qmd`. -- Book linkage: `book/chapters/06-blueprint-manuscrito.qmd` and - `book/chapters/14-release.qmd`. -- Commands: - - `just paper-ijds` - - `just paper-ijds-supplement` - - `just paper-submission` diff --git a/docs/research/crpto_pyepo_dfl_intake_2026-05-26.md b/docs/research/crpto_pyepo_dfl_intake_2026-05-26.md deleted file mode 100644 index f01d680..0000000 --- a/docs/research/crpto_pyepo_dfl_intake_2026-05-26.md +++ /dev/null @@ -1,206 +0,0 @@ -# PyEPO 1.3 Intake Memo - 2026-05-26 - -> Ported from the CRPTO research archive (`pyepo_1_3_intake_2026-05-26`). -> Evaluates PyEPO 1.3 for the CRPTO project, the Quarto book and the agenda extendida CRPTO/tesis -> agenda. It is not a promotion artifact and does not modify the frozen champion. - -## Bottom Line - -PyEPO 1.3 matured enough to reopen the **SPO/DFL lane as a bounded, isolated -prototype**, especially for the agenda extendida CRPTO/tesis agenda. The 2026-05-28 closeout below -now satisfies the CRPTO stop rule: we import the paired SPO+ rerun as appendix -evidence and, after the 2026-06-06 consolidation, retain only curated full-suite -summary tables under `reports/crpto/extended/`. We do not import solver logs, -run directories, new runtime dependencies, or any claim that would replace -CRPTO. The strongest contribution is a cleaner comparator boundary: - -1. `SPOPlus` as the existing regret-minimizing baseline. -2. `regularizedFrankWolfeFenchelYoung` (RFYL) as the new low-friction - differentiable loss for the current continuous LP-style credit selection - problem. -3. `perturbedFenchelYoungMul` only for positive-cost risk/ECL variants, not for - raw net-return costs that can be negative. -4. `coneAlignedCosine` / CaVE only if we run a **binary** approval or top-k - selection prototype with a Gurobi-backed model. - -The key update relative to the older v32 SPO blocker is that we do **not** need -`cvxpy/cvxpylayers` to make a formal PyEPO prototype. PyEPO 1.3.7 works in an -isolated OR-Tools environment with PyTorch autograd modules. - -## Curated Closeout - 2026-05-28 - -The CRPTO research archive completed the PyEPO 1.3.7 rerun after this intake memo was -created. CRPTO imports only the conclusion needed for IJDS/thesis appendix: - -| Result | Curated value | CRPTO decision | -|---|---:|---| -| SPO+ mean regret | `0.184366` | Appendix comparator only. | -| Two-stage mean regret | `0.358073` | Baseline for the paired comparison. | -| SPO+ improvement | `48.51%` | Curated PyEPO closeout value; separate from the committed A19/Fig. 15 artifact (`49.09%`). | -| Paired Wilcoxon | `p = 3.80e-163` | Statistical support for the comparator. | - -Interpretation: SPO+ remains the regret winner, as expected. CRPTO remains the -coverage/auditability method because the PyEPO suite does not provide conformal -coverage, exact funded-set bound guarantees, or a replacement for the frozen -`bound_aware_276k_economic_champion`. RFYL, CaVE and PFYL-Mul stay in the agenda extendida CRPTO/tesis -DFL lab as comparator research, not as CRPTO promotion evidence. - -Operational decision: do not copy PyEPO run directories, solver logs, Gurobi -artifacts or new dependencies into this repo. Keep this memo, the SPO+ chapter -language and the curated evidence-card CSVs as the self-contained record. - -## Version Check - -| Source | Finding | -|---|---| -| PyPI | Latest package is `pyepo 1.3.7`, released 2026-05-26. Requires Python `>=3.9` and publishes typed wheels. | -| GitHub releases | 1.3 release notes attached to the 2026-05-25 series. | -| Docs | Documentation site reports `PyEPO 1.3.7` with a rebuilt tutorial structure. | -| Repo README | PyEPO is the official MPC-paper implementation and now asks users to cite CaVE separately when using that loss. | - -Sources: https://pypi.org/project/pyepo/ , -https://github.com/khalil-research/PyEPO/releases , -https://khalil-research.github.io/PyEPO/build/html/index.html , -https://github.com/khalil-research/PyEPO - -## What Changed In 1.3 - -### CaVE / CaVE+ - -`pyepo.func.coneAlignedCosine` implements cone-aligned vector estimation for -binary linear programs. Instead of differentiating through the solver or sampling -perturbations, it aligns predicted costs with the normal cone at the true binary -optimum. The default `max_iter=3` is the CaVE+ preset (truncated Clarabel -projection). - -- Strong for a binary approval/top-k prototype. -- Weak for the current continuous/fractional CRPTO champion. -- Requires `optDatasetConstrs`, which needs Gurobi-backed constraint extraction. - -Decision: agenda extendida CRPTO/tesis only, as a small Gurobi-enabled binary lane. Do not import into -the CRPTO main claim. - -### Regularized Frank-Wolfe - -`regularizedFrankWolfeOpt` and `regularizedFrankWolfeFenchelYoung` smooth the LP -oracle with L2 regularization. Best new match for our current credit optimizer -because it only needs a linear optimization oracle and works with our -OR-Tools-style `optModel`. - -- Good for continuous LP portfolio selection; better than CaVE for the current - CRPTO shape. -- Avoids the old `cvxpy/cvxpylayers` dependency blocker. -- Gives agenda extendida CRPTO/tesis a more credible "formal DFL" experiment than the earlier - oracle-regret surrogate. - -Decision: **promote to next experiment** as a third DFL comparator beside -two-stage and SPO+. - -### Multiplicative Perturbation - -`perturbedOptMul` and `perturbedFenchelYoungMul` preserve cost signs via -multiplicative noise (matters when a solver expects nonnegative costs). - -- Good for `PD`, `PD_high`, ECL/loss-only or positive shifted cost variants. -- Dangerous for raw `PD * LGD - int_rate`, because those costs naturally cross - zero; shifting costs changes optimization behavior unless carefully justified. - -Decision: use only in a risk-cost/ECL experiment, not as the default CRPTO regret -comparator. - -### CVRP Models - -PyEPO 1.3 adds CVRP models across Gurobi/COPT/Pyomo backends. Not directly useful -for Lending Club credit selection; only a didactic Quarto sidebar or future -servicing/collections example. Decision: park. - -### Performance And API Hardening - -Release notes report broad 1.0-3.3x speedups for core methods and 6-14x CaVE+ -speedups versus SPO+ on TSP, plus full public type annotations, solution-pool -refactors, expanded tests, CUDA tests and a docs overhaul. Supports tightening -the optional `spo` dependency and makes the DFL lane less fragile in -reviewer-facing reproducibility language. - -## Local Probe - -The isolated probe created a dedicated environment outside the repo with PyTorch + -OR-Tools. The Windows-first equivalent (isolated, optional, never the default -`.venv`): - -```powershell -uv venv .venv-pyepo --python 3.12 -uv pip install --python .venv-pyepo\Scripts\python.exe "pyepo[ortools]==1.3.7" -``` - -Installed versions in the probe: `pyepo` 1.3.7, `torch` 2.12.0+cu130, `ortools` -9.15.6755, `numpy` 2.4.6, `pandas` 3.0.3, `clarabel` 0.11.1. - -Smoke result on a tiny OR-Tools credit-selection LP: - -| Check | Result | -|---|---| -| `optDataset` pre-solve | 48 instances solved | -| `SPOPlus` two-epoch training | loss moved from `1.403071` to `0.618665` | -| `regularizedFrankWolfeFenchelYoung` forward pass | returned finite scalar loss | -| `perturbedFenchelYoungMul` forward pass | returned finite scalar loss on positive shifted costs | - -Interpretation: PyEPO 1.3.7 is usable in an isolated env with OR-Tools; we no -longer describe the whole DFL lane as blocked by `cvxpy/cvxpylayers`. The env is -heavy (`pyepo[ortools]` pulls PyTorch and CUDA wheels), so it stays -optional/isolated. - -## Repo Fit (child) - -| File | Observation | Action | -|---|---|---| -| `pyproject.toml` (`spo` extra, `pyepo>=1.0`) | Too loose for the new claim. | Later tighten to `pyepo[ortools]>=1.3.7,<1.4` in a dedicated dependency PR. | -| `src/optimization/spo_integration.py` | Verify the helper matches PyEPO 1.3 SPO+ signature `SPOPlus(pred, costs, sols, objs)` rather than the old `SPOPlus(pred, costs)`. | Resolved 2026-06-09: module removed as dead code (never imported); `scripts/run_spo_real.py` remains the DFL entrypoint. | -| `scripts/run_spo_real.py` | Canonical DFL prototype entrypoint candidate; confirm it uses the correct `optDataset` tuple. | Make this the canonical DFL prototype entrypoint. | -| `scripts/run_spo_comparison.py`, `scripts/run_crpto_vs_spo_stability.py` | Existing comparator/stability runners. | Keep as the comparator surface; add RFYL only after a real rerun. | -| `book/chapters/09-spo-regret.qmd` | Frames SPO+ as a regret comparator, not a conformal replacement. | Add a short PyEPO 1.3 footnote only after a real rerun. | - -## Recommended Experiments - -Historical intake status: Experiment A is now closed by the 2026-05-28 rerun. -Experiments B--D remain agenda extendida CRPTO/tesis research lanes and are not prerequisites for -the CRPTO IJDS manuscript. - -- **A - SPO+ repro rerun**: run `scripts/run_spo_real.py` in the isolated PyEPO - env (`n_items=100`, `budget=30`, `epochs=50`, `seeds=5`). Gate: reproduces the - current regret-improvement story within tolerance; logs PyEPO/Torch/solver - versions. Sink: CRPTO appendix and Quarto `09-spo-regret`. -- **B - RFYL comparator**: add `regularizedFrankWolfeFenchelYoung` to the same - sampled instances; compare two-stage Ridge, SPO+, RFYL and CRPTO robust costs. - Gate: finite, stable regret across seeds; improves over two-stage or gives a - useful speed/stability trade-off. Sink: agenda extendida CRPTO/tesis DFL lane; optional CRPTO - appendix. -- **C - multiplicative PFYL for risk-only costs**: positive costs only (`PD`, - `PD_high`, ECL). Do not use raw `PD * LGD - int_rate` unless reframed. Sink: - agenda extendida CRPTO/tesis only. -- **D - CaVE binary prototype**: only if Gurobi is available; convert to binary - top-k/fixed-budget, use `optDatasetConstrs` + `coneAlignedCosine`. Sink: agenda extendida CRPTO/tesis - method appendix. - -## Claim Boundaries - -Allowed: "PyEPO 1.3.7 enables a reproducible isolated DFL comparator stack"; -"RFYL is the most natural PyEPO 1.3 method for the current continuous credit LP -prototype"; "CaVE is promising for binary approval/top-k variants but requires a -Gurobi-backed binary model." - -Not allowed: "PyEPO replaces CRPTO"; "SPO+/RFYL/CaVE provide conformal coverage -guarantees"; "CaVE applies to the current fractional champion without -reformulating the decision problem"; "multiplicative perturbation is valid for -signed net-return costs without a documented transformation." - -## Editorial Recommendation - -For **CRPTO**, keep PyEPO as a comparator appendix. The core claim remains CRPTO: -calibrated PD plus Mondrian conformal uncertainty plus a robust portfolio policy -that is auditable. - -For **agenda extendida CRPTO/tesis (agenda)**, reopen the SPO/DFL lane with a new stop rule: one SPO+ -rerun, one RFYL comparator, optional multiplicative PFYL risk-only probe, optional -CaVE binary probe only if Gurobi is available. If those do not change a manuscript -claim or produce a cleaner comparator table, stop. diff --git a/docs/research/crpto_regret_auditability_sandbox_closure_2026-05-28.md b/docs/research/crpto_regret_auditability_sandbox_closure_2026-05-28.md deleted file mode 100644 index b0d3059..0000000 --- a/docs/research/crpto_regret_auditability_sandbox_closure_2026-05-28.md +++ /dev/null @@ -1,163 +0,0 @@ -# CRPTO Regret-Auditability Sandbox Closure - 2026-05-28 - -> Ported from the CRPTO research archive -> (`paper1_regret_auditability_sandbox_closure_2026-05-28`). **Documentation -> only.** This is the closure record of the broader regret-auditability search: -> the frozen champion (`paper-thesis-final-economic-2026-04-06`) is **not** -> replaced. (The earlier feature-search plan and sandbox dependency report were -> retired in the 2026-06-13 docs cleanup; this closure is the surviving record.) - -## Purpose - -Closes the external CRPTO regret-auditability sandbox intake. The sandbox was -created **outside this repository** (in an external experiments volume). The -question was whether a much broader CatBoost monotone + Optuna + Venn-Abers + -Mondrian conformal + robust portfolio search could improve the CRPTO champion, -or at least strengthen the paper's evidence around the regret-auditability -frontier. - -## Closure Decision - -The sandbox was useful, but it does not replace the frozen economic champion by -itself. It produced credible PD and conformal challenger evidence, triggered a -governed champion tournament, and generated negative-result evidence useful for -IJDS-style documentation. However, the downstream portfolio and bound evidence -does not produce a clean new champion that dominates the frozen policy on return, -`V`, `Gamma_CP`, violation, and coverage at the same time. - -Frozen champion reference (unchanged): - -- realized return: `170464.5429284627` -- `V`: `0.03645` -- `Gamma_CP`: `0.18591` -- violation: `0` -- funded coverage: `0.9433` -- policy: `blended_uncertainty`, risk `0.175`, gamma `0.45`, uncertainty - aversion `0.1` -- region: `45/45` - -## What Was Evaluated (archive-staged, not promoted) - -The useful sandbox material was absorbed in the CRPTO research archive through paper-facing and -research artifacts (the dated CSV tables and staged PD challenger models live in -the CRPTO research archive; the child keeps the **conclusions** below, not the -challenger weights). Child-side companions: - -- [crpto_bound_improvement_intake_2026-05-21](crpto_bound_improvement_intake_2026-05-21.md) -- [crpto_champion_reopen_plan_2026-05-21](crpto_champion_reopen_plan_2026-05-21.md) -- [crpto_champion_tournament_protocol_2026-05-25](crpto_champion_tournament_protocol_2026-05-25.md) - -Three external PD challengers were staged in the CRPTO research archive and evaluated: -`bureau_behavior_15`, `affordability_rate_5`, `canonical_4`. None were promoted. - -## PD Findings - -The strongest sandbox contribution was the PD search. It showed the frozen PD -stack was not the predictive ceiling. - -| Role | Candidate | AUC | Brier | ECE | Use | -| --- | ---: | ---: | ---: | --- | -| incumbent | `incumbent__frozen_champion` | `0.712678` | `0.154591` | `0.006152` | frozen reference | -| challenger | `full_challenger_woe__bureau_behavior_15` | `0.720679` | `0.153161` | `0.007689` | main challenger | -| challenger | `full_challenger__canonical_4` | `0.720624` | `0.153182` | `0.005917` | sensitivity baseline | -| challenger | `full_challenger_woe__affordability_rate_5` | `0.720052` | `0.153276` | `0.007502` | sensitivity baseline | - -Interpretation: - -- `bureau_behavior_15` is the best pure discrimination/Brier signal (AUC +0.008), - but its ECE is worse than the incumbent and it carries a higher - feature-governance burden. -- `canonical_4` is a cleaner governance sensitivity: nearly the same AUC lift, - better Brier than incumbent, better ECE than the incumbent replay. -- `affordability_rate_5` tests whether affordability monotonicity and WOE - transformations add a stable signal. - -This is strong enough for a challenger appendix and a gated champion-reopen -protocol, but not enough by itself to change the CRPTO claim. - -## Conformal Findings - -The sandbox selected a usable but non-final conformal configuration (partition -`grade`, raw probability source, 5 score bins, `grade_then_global` fallback, -alpha90 `0.075`, alpha95 `0.06`, min group size `100`, `bernoulli_sqrt` scaling). -The follow-up showed why it should remain a challenger: - -| Candidate | coverage90 | min group cov90 | avg width90 | worst group | Reading | -| --- | ---: | ---: | ---: | --- | --- | -| `affordability_rate_5` | `0.944317` | `0.916647` | `0.806270` | `score_q00` | viable but wider | -| `bureau_behavior_15` | `0.919951` | `0.870059` | `0.749615` | `E` | rare-grade weakness | -| `canonical_4` | `0.931878` | `0.917582` | `0.790729` | `score_q04` | viable sensitivity | -| `official_champion` | `0.929714` | `0.918983` | `0.784230` | `score_q03` | still balanced | - -The conformal search revealed where PD improvements transfer cleanly and where -they create rare-grade weaknesses. `bureau_behavior_15` is predictive, but its -grade `E` weakness makes it hard to promote without further conformal repair. - -## Portfolio And Bound Findings - -The portfolio layer is where the frozen champion remains strongest as a balanced -paper-facing claim. The archived decision table contained 73 exact/pass decision -rows: - -- `35` append-or-park rows with no champion case -- `20` Gamma-only challengers with worse V/return -- `9` V-only challengers with worse Gamma/return -- `7` bound-only challengers with worse return -- `1` return-only challenger with worse bounds -- `1` official baseline row - -The best positive-return challenger found: - -- candidate: `canonical_4_return_aware` -- return: `170611.34163424745` (delta vs champion `+146.80`) -- `V`: `0.058675`; `Gamma_CP`: `0.270366`; violation: `0` -- decision: `return_challenger_only_bound_worse` - -The best V/Gamma challengers improved one bound dimension but paid too much in -return or worsened the other bound dimension. Useful negative evidence: the broad -search did not find a free lunch. - -## Scientific Value For IJDS - -1. Supports an anti-cherry-pick story: the project reopened PD, conformal, and - portfolio under explicit gates rather than stopping at the first champion. -2. Clarifies the regret-auditability frontier: higher-return candidates exist, - but they tend to weaken `V`, `Gamma_CP`, funded coverage, or group coverage. -3. Provides credible negative results: publishable as appendix/robustness - evidence showing why the frozen champion remains the main balanced policy. -4. Separates predictive improvement from decision improvement: better AUC does - not automatically imply a better robust portfolio under conformal guarantees. - -Recommended paper framing: main text keeps the frozen economic champion as the -primary CRPTO result; appendix reports the governed reopen/tournament as -robustness and sensitivity evidence; agenda extendida CRPTO/tesis / methods appendix uses the -regret-auditability frontier and PyEPO regret suite to discuss decision -efficiency vs auditability. - -## What Should Not Be Claimed - -- Do not claim the sandbox produced a new champion unless a later sealed - full-universe confirmation passes all champion replacement gates. -- Do not compare a child 25k quick portfolio return directly against the frozen - 276k champion. -- Do not promote `bureau_behavior_15` on AUC alone while the conformal rare-grade - weakness remains unresolved. -- Do not treat high-return portfolio probes as paper champions when their - `V`/`Gamma_CP` trade-off is worse than the frozen policy. - -## Remaining Optional Work (only if pursuing replacement) - -- run a sealed full-universe cuOpt + HiGHS rerank for a small predeclared set; -- repair or explicitly park the `bureau_behavior_15` grade `E` conformal issue; -- decide whether `canonical_4` becomes the main appendix challenger (cleanest - PD/calibration/conformal balance); -- export a final negative-results registry for the IJDS appendix; -- avoid further open-ended search unless the protocol version is reopened before - seeing downstream results. - -## Final Closure Note - -The sandbox is successful as evidence generation, not as a champion replacement. -It gave stronger PD challengers, a more rigorous conformal/portfolio tournament, -and a clearer empirical case for CRPTO's central tension: robust auditability can -be bought, but it is not free in return/regret space. diff --git a/docs/research/foundations/crpto_calibration_method_selection.md b/docs/research/foundations/crpto_calibration_method_selection.md deleted file mode 100644 index 2dbefe3..0000000 --- a/docs/research/foundations/crpto_calibration_method_selection.md +++ /dev/null @@ -1,142 +0,0 @@ - - -> **RESEARCH / METHOD NOTE** — Conservado como soporte metodológico. La política viva se resume en `docs/MODEL_RISK_MANAGEMENT.md` y en los artifacts canónicos. - -# Selección de Método de Calibración PD — Formal Writeup - -**Estado**: Canónico post paper-grade run 2026-03-13 -**Método seleccionado**: Venn-Abers -**Artefacto fuente**: `models/pd_calibration_diagnostics.json` - ---- - -## El problema de calibración en riesgo de crédito - -Un modelo PD bien calibrado satisface: para todos los préstamos con PD predicha $\hat{p}$, la tasa de default observada se aproxima a $\hat{p}$. Formalmente, la *calibración marginal* exige: - -$$\mathbb{E}[Y \mid \hat{p}(X) = p] \approx p \quad \forall p \in [0,1]$$ - -En la práctica se mide con la **Expected Calibration Error (ECE)**: - -$$\text{ECE} = \sum_{b=1}^{B} \frac{|B_b|}{n} \left| \text{acc}(B_b) - \text{conf}(B_b) \right|$$ - -donde $B_b$ es el bin $b$, $\text{acc}(B_b)$ es la tasa de default observada en el bin, y $\text{conf}(B_b)$ es la media de probabilidades predichas. - -**Por qué importa en crédito:** -- Una PD mal calibrada distorsiona el ECL bajo IFRS9 (`ECL = PD × LGD × EAD`) -- Los intervalos conformales downstream heredan el error de calibración de las PD base -- El pricing basado en riesgo usa PD directamente como input de rentabilidad ajustada - ---- - -## Métodos evaluados (4 candidatos) - -### 1. Platt Scaling (Platt 1999) - -Entrena una regresión logística sobre los scores brutos del modelo: - -$$\hat{p}_{\text{Platt}}(s) = \frac{1}{1 + \exp(As + B)}$$ - -- **Ventaja**: simple, estable, opera sobre 1 parámetro real -- **Limitación**: asume que la relación score → probabilidad es logística; falla cuando la distribución del score es multimodal o tiene colas pesadas -- **Garantía teórica**: ninguna distribución-libre; requiere que el score base sea bien comportado - -### 2. Isotonic Regression (Zadrozny & Elkan 2002) - -Ajuste no paramétrico con restricción de monotonía: - -$$\hat{p}_{\text{iso}} = \arg\min_{f \text{ monótona}} \sum_i (y_i - f(s_i))^2$$ - -- **Ventaja**: más flexible que Platt; captura cualquier relación monótona -- **Limitación**: puede sobreajustar con pocos datos; no tiene garantía de cobertura finita -- **Garantía teórica**: consistencia asintótica (requiere $n \to \infty$) - -### 3. Venn-Abers Calibration (Vovk & Petej 2012; Vovk et al. 2015) - -Método conformal de calibración que produce **intervalos de probabilidad** con garantías de cobertura finita. Para cada nuevo punto $x$, produce un par $(p_0, p_1)$ donde: - -$$p_0 = \text{Venn predictor bajo } y=0, \quad p_1 = \text{Venn predictor bajo } y=1$$ - -La probabilidad calibrada se estima como: - -$$\hat{p}_{\text{VA}} = \frac{p_0 + p_1}{2}$$ - -**Garantía teórica clave**: bajo intercambiabilidad (i.i.d.), Venn-Abers produce probabilidades *marginalmente calibradas* con garantía de cobertura finita — sin hipótesis distribucionales adicionales. - -- **Ventaja**: distribución-libre, calibración con garantía finita, produce bounds que cuantifican incertidumbre epistémica de la calibración -- **Ventaja adicional**: el ancho del intervalo VA (`avg_width`) es un indicador de confianza en la calibración por punto -- **Limitación**: overhead computacional O(n log n) vs O(1) de Platt; no aplica `unbiasedness_in_the_large` cuando hay shift de prevalencia - -### 4. Beta Calibration (Kull et al. 2017) - -Ajuste paramétrico flexible con tres parámetros (a, b, m) que modela la transformación score → probabilidad mediante una distribución Beta: - -$$\hat{p}_{\text{Beta}}(s) = \frac{1}{1 + \frac{1-s^a}{s^a} \cdot \frac{1}{e^{b + m \cdot \log(s/(1-s))}}}$$ - -- **Ventaja**: más flexible que Platt (3 parámetros vs 2), captura asimetrías en la distribución de scores -- **Ventaja adicional**: forma funcional paramétrica → no sobreajusta como Isotonic con muestras pequeñas -- **Limitación**: requiere scores en (0,1); puede ser inestable con scores extremos (< 0.001 o > 0.999) -- **Garantía teórica**: consistencia asintótica; sin garantía distribución-libre finita (a diferencia de Venn-Abers) -- **Implementación**: `betacal.BetaCalibration(parameters="abm")` en `src/models/calibration.py` - ---- - -## Resultados de comparación (paper-grade run 2026-03-13) - -| Método | ECE (OOT) | Brier (OOT) | AUC (OOT) | -|--------|-----------|-------------|-----------| -| Platt | 0.0084 | 0.1546 | 0.7129 | -| Isotonic | 0.0062 | 0.1545 | 0.7128 | -| **Venn-Abers** | **0.0061** | **0.1545** | **0.7128** | - -**Metadatos Venn-Abers (OOT, n=276,869):** -- `mean_p0 = 0.2164`, `mean_p1 = 0.2170` (prevalencia observada: 21.98%) -- `avg_width = 0.0005` (ancho medio del intervalo VA → calibración muy estable) -- `median_width = 0.0003` -- `unbiasedness_in_the_large = False` (señal de drift leve de prevalencia entre cal/test, esperado en split OOT estricto) - -**Interpretación del `avg_width = 0.0005`:** los bounds de Venn-Abers son casi colapsos de punto, indicando que la calibración es altamente estable a través del espacio de scores. Si hubiera incertidumbre epistémica significativa en la calibración, el ancho sería mayor (≥ 0.01). - ---- - -## Rationale de selección - -Venn-Abers fue seleccionado como método canónico por tres razones: - -1. **Mejor ECE en datos OOT**: ECE=0.0061 vs 0.0062 (Isotonic) vs 0.0084 (Platt). La diferencia respecto a Isotonic es pequeña (~1.6%) pero Venn-Abers logra esta mejora con garantías teóricas adicionales. - -2. **Garantía de calibración con muestra finita**: a diferencia de Platt/Isotonic, la calibración Venn-Abers es válida bajo intercambiabilidad sin asunciones distribucionales. Esto es coherente con la filosofía conformal del proyecto. - -3. **Compatibilidad arquitectónica**: el proyecto usa MAPIE Mondrian Conformal para los intervalos de PD. Usar Venn-Abers para la calibración base crea una arquitectura de incertidumbre coherente: calibración conformal + intervalos conformales. - -**Cuándo revisar esta decisión:** -- Si el `avg_width` Venn-Abers crece sustancialmente (> 0.01) indicando inestabilidad -- Si el AUC drop relativo a uncalibrated supera 0.15% -- Si un nuevo método (e.g., Temperature Scaling + conformal) muestra ECE < 0.005 en datos OOT - ---- - -## Referencias clave - -1. **Vovk, V. & Petej, I. (2012)** — Venn prediction. [arXiv:1211.6990] -2. **Vovk, V., Shen, J., Manokhin, V., Xie, M. (2017)** — Nonparametric predictive distributions based on conformal prediction. JMLR. -3. **Platt, J. (1999)** — Probabilistic outputs for SVMs and comparisons to regularized likelihood methods. Advances in Large Margin Classifiers. -4. **Zadrozny, B. & Elkan, C. (2002)** — Transforming classifier scores into accurate multiclass probability estimates. KDD. -5. **Brier, G.W. (1950)** — Verification of forecasts expressed in terms of probability. Monthly Weather Review. -6. **Gneiting, T. & Raftery, A.E. (2007)** — Strictly proper scoring rules, prediction, and estimation. JASA. - ---- - -## Artefactos canónicos - -| Artefacto | Ruta | Contenido | -|-----------|------|-----------| -| Diagnósticos completos | `models/pd_calibration_diagnostics.json` | Comparación 4-way, VA bounds, bins de confiabilidad | -| Calibrador serializado | `models/pd_canonical_calibrator.pkl` | Objeto `VennAbersScoreCalibrator` fitted | -| Estado de calibración rare-event | `models/pd_rare_event_calibration_status.json` | ECE por grupo protegido, worst-grade Brier | -| Reporte rare-event | `data/processed/pd_rare_event_calibration_report.parquet` | Por slice: ECE, Brier, PR-AUC | -| Implementación | `src/models/venn_abers.py` | `VennAbersScoreCalibrator` | - ---- - -*Última actualización: 2026-03-16 | Run: `paper-grade-2026-03-13-final-heavy-2026-03-13-230650`* diff --git a/docs/research/foundations/crpto_conformal_bound_foundations_2026-05-19.md b/docs/research/foundations/crpto_conformal_bound_foundations_2026-05-19.md deleted file mode 100644 index 4fd45e6..0000000 --- a/docs/research/foundations/crpto_conformal_bound_foundations_2026-05-19.md +++ /dev/null @@ -1,45 +0,0 @@ -# CRPTO Conformal Bound Foundations - 2026-05-19 - -> Ported from the CRPTO research archive (`angelopoulos_conformal_bound_intake_2026-05-19`). -> Records how the Angelopoulos / Bates / Barber line updates the CRPTO manuscript -> without reopening the frozen champion search. - -## CRPTO - -CRPTO should use Angelopoulos and coauthors as the modern conformal foundation: - -- `angelopoulos2023`: readable entry point and pedagogical reference for split - conformal, distribution-free uncertainty and examples. -- `angelopoulos2026_foundations`: proof-audit source for exchangeability, - conformal quantiles, split conformal and the logic of distribution-free - inference. -- `bates2021rcps`, `angelopoulos2024risk`, `angelopoulos2025ltt`: the direct - lineage for the funded-set bound, because CRPTO is best framed as a - portfolio-aware instantiation of bounded monotone risk control. - -The central claim remains unchanged: CRPTO is a post-hoc auditable bridge from -calibrated PD and conformal upper bounds to a robust credit portfolio -constraint. The Markov bound remains the main distribution-free statement; the -exact `276k` evidence remains empirical validation of the frozen policy, not a -stronger post-selection theorem. - -## Agenda / agenda extendida CRPTO/tesis lanes - -The same source pack opens four bounded research lanes (agenda, not main claim): - -| Source | Lane | Decision | -| --- | --- | --- | -| CRC/LTT | lane gates for monotone losses | append | -| Label-noise robustness | default/charge-off label governance | append/future | -| Non-monotonic CRC | multi-objective decision-risk gate | park until implemented | -| Gradient equilibrium / time-series repo | online drift recalibration | park unless prospective split exists | - -These are research-program improvements, not new proof that the current frozen -champion should change. - -## Implementation Rule - -Promote to the Quarto book only when a source changes claim boundaries, proof -language or future-work prioritization. Do not create new versioned scripts or -new runs unless the experiment can change a specific claim. The frozen champion -(`paper-thesis-final-economic-2026-04-06`) is not reopened by citation work. diff --git a/docs/research/foundations/crpto_conformal_libraries_comparison.md b/docs/research/foundations/crpto_conformal_libraries_comparison.md deleted file mode 100644 index ae86906..0000000 --- a/docs/research/foundations/crpto_conformal_libraries_comparison.md +++ /dev/null @@ -1,499 +0,0 @@ - - -> **RESEARCH NOTE** — Comparativa de librerías retenida para justificación y anexos; no es el contrato operativo actual. - -# Conformal Prediction Libraries: MAPIE vs Alternatives - -**Project:** Lending Club Risk - Conformal Predict-then-Optimize -**Date:** 2026-02-07 -**Installed / validated in repo (2026-03-13):** -- MAPIE: 1.3.0 -- CREPES: 0.9.0 -- venn-abers: 1.5.1 - -**Latest checked from PyPI on 2026-03-13:** -- MAPIE: 1.3.0 -- CREPES: 0.9.0 -- venn-abers: 1.5.1 - -## Nota de contexto (2026-02-16) - -Esta comparación es de investigación. -El proyecto productivo de storytelling usa artefactos Mondrian ya generados: - -- `models/conformal_results_mondrian.pkl` -- `data/processed/conformal_intervals_mondrian.parquet` - ---- - -## Overview - -This document compares conformal prediction libraries available in Python for credit risk modeling. - ---- - -## Library Comparison Matrix - -| Feature | MAPIE 1.3.0 | CREPES 0.9.0 | Nonconformist | Rolling Your Own | -|---------|-------------|--------------|---------------|------------------| -| **Maintenance** | Active (2024+) | Active (2024+) | Stale (2020) | N/A | -| **Scikit-learn API** | Yes | Partial | Yes | Manual | -| **Split Conformal** | ✅ | ✅ | ✅ | Easy | -| **Cross Conformal** | ✅ | ✅ | ✅ | Medium | -| **Jackknife+** | ❌ | ✅ | ✅ | Hard | -| **CQR (Quantile)** | ✅ | ❌ | ❌ | Hard | -| **Mondrian CP** | Manual | ✅ Built-in | ✅ | Medium | -| **Venn-ABERS** | ✅ (`VennAbersCalibrator`) | ❌ p-values/predictive systems, not Venn-Abers calibration | ❌ | Very Hard | -| **Classification Sets** | ✅ (LAC, APS, RAPS) | ❌ | Limited | Hard | -| **Time Series** | Partial | ✅ | ❌ | Hard | -| **Documentation** | Excellent | Good | Outdated | N/A | -| **Examples** | Many | Moderate | Few | N/A | -| **Installation** | `pip install mapie` | `pip install crepes` | `pip install nonconformist` | N/A | -| **Dependencies** | sklearn, numpy | numpy | sklearn | numpy | -| **GitHub Stars** | ~1.2k | ~200 | ~300 | N/A | -| **Industry Adoption** | High | Low | Medium (legacy) | N/A | - ---- - -## MAPIE 1.3.0 (Recommended for Your Project) - -### Pros -- **Scikit-learn native:** Works seamlessly with sklearn pipelines -- **Production-ready:** Well-tested, stable API -- **Comprehensive:** Regression (Split, Cross, CQR) + Classification (LAC, APS, RAPS) -- **Fast:** Optimized C extensions via sklearn -- **Great docs:** Extensive tutorials, examples -- **Active development:** Regular updates, bug fixes -- **Industry standard:** Used in production at banks, fintechs - -### Cons -- **No Mondrian built-in:** Must implement manually (not hard) -- **No Venn-ABERS:** If you need probability intervals with calibration -- **No Jackknife+:** Only Split and Cross conformal -- **Time series support limited:** No EnbPI (use mlforecast instead) - -### Best For -- **Your thesis project:** PD probability intervals + classification sets -- Production credit risk models -- Integration with sklearn ecosystem -- Standard conformal methods (Split, Cross, CQR) - -### Code Example (Your Current Implementation) -```python -from mapie.regression import SplitConformalRegressor -from src.models.conformal import ProbabilityRegressor - -prob_reg = ProbabilityRegressor(catboost_model) -mapie = SplitConformalRegressor( - estimator=prob_reg, - confidence_level=0.90, - conformity_score='absolute', - prefit=True -) -mapie.fit(X_cal, y_cal) -mapie.conformalize(X_cal, y_cal) -y_intervals = mapie.predict_interval(X_test) -``` - -**Verdict:** ✅ Use MAPIE as your primary library. - ---- - -## CREPES 0.9.0 (Installed but Not Primary) - -### Pros -- **Flexible:** Works with any predictor (not just sklearn) -- **Mondrian built-in:** Easy group-conditional coverage -- **p-values / predictive systems:** strong for conformal research and online diagnostics -- **Conformal Predictive Systems:** Full probability distributions -- **Fewer dependencies:** Just numpy -- **Excellent for research:** Implements cutting-edge methods - -### Cons -- **Less popular:** Smaller community, fewer examples -- **No classification sets:** No LAC/APS/RAPS -- **Steeper learning curve:** Non-sklearn API -- **Less battle-tested:** Fewer production deployments -- **Manual integration:** More boilerplate code - -### Best For -- **Research experiments:** Testing conformal variants and online p-value diagnostics -- Mondrian CP (if you don't want to implement manually) -- Conformal predictive systems -- Non-sklearn models (e.g., custom neural nets) - -### Code Example (Mondrian CP with CREPES) -```python -from crepes import ConformalRegressor - -# Mondrian CP built-in -cr = ConformalRegressor() - -# Fit per group -grades = X_cal['grade'].unique() -predictors = {} - -for grade in grades: - mask = (X_cal['grade'] == grade) - residuals_cal = y_cal[mask] - model.predict(X_cal[mask]) - - cr_grade = ConformalRegressor() - cr_grade.fit(residuals_cal) - predictors[grade] = cr_grade - -# Predict -for grade in grades: - mask = (X_test['grade'] == grade) - y_hat = model.predict(X_test[mask]) - y_intervals[mask] = predictors[grade].predict( - X_test[mask], - confidence=0.90, - y_hat=y_hat - ) -``` - -Important clarification: -- `WrapClassifier.predict_p(...)` returns **p-values**, not calibrated probabilities. -- In este paquete CRPTO, treating `predict_p` as `[p0, p1]` probability bounds was incorrect and was removed. - -**Verdict:** ⚠️ Use CREPES for experiments and p-value diagnostics, not as a drop-in Venn-Abers calibrator. - ---- - -## venn-abers 1.5.1 (Primary for Venn-Abers calibration) - -### Pros -- Dedicated implementation of Venn-Abers calibration. -- Supports score-based usage (`VennAbers`) and sklearn-style wrappers (`VennAbersCalibrator`). -- Better semantic fit for post-hoc probability calibration than repurposing conformal p-values. - -### Cons -- Smaller package and ecosystem than MAPIE. -- Alpha-quality package metadata on PyPI, so changes should be wrapped behind local tests. - -### Best For -- Post-hoc Venn-Abers calibration over raw classifier probabilities. -- Direct comparison against Platt and Isotonic inside PD training. - -### Project decision -- `venn-abers` is the canonical implementation for Venn-Abers in este paquete CRPTO. -- `crepes` remains for conformal p-values / predictive systems / research. -- `mapie` remains primary for conformal intervals and time-series conformal. - ---- - -## Nonconformist (Legacy - Not Recommended) - -### Status -- **Last update:** 2020 (4+ years old) -- **Python 2 legacy code:** Not fully Python 3 compatible -- **Deprecated dependencies:** Old sklearn API - -### Pros -- **Historical:** First major conformal library -- **Jackknife+ support:** If you need this specific method -- **Academic citations:** Many papers reference it - -### Cons -- **Unmaintained:** No recent updates, no bug fixes -- **Compatibility issues:** May break with new sklearn versions -- **Limited features:** No CQR, no classification sets -- **Poor docs:** Examples outdated - -### Best For -- Nothing (use MAPIE or CREPES instead) - -**Verdict:** ❌ Do not use. Migrate to MAPIE. - ---- - -## Rolling Your Own (Custom Implementation) - -### When to Consider -- Very specific use case not covered by MAPIE/CREPES -- Educational purposes (understanding conformal theory) -- Extreme performance optimization needs - -### Complexity Estimates - -| Method | Lines of Code | Difficulty | Dependencies | -|--------|--------------|------------|--------------| -| Split Conformal | ~50 | Easy | numpy | -| Cross Conformal | ~150 | Medium | sklearn (for CV splits) | -| CQR | ~200 | Hard | quantile regressors | -| Mondrian CP | ~100 | Medium | numpy | -| LAC/APS Sets | ~100 | Medium | numpy | -| Venn-ABERS | ~500+ | Very Hard | scipy, optimization | - -### Example: Minimal Split Conformal - -```python -import numpy as np - -class SimpleSplitConformal: - def __init__(self, model, alpha=0.1): - self.model = model - self.alpha = alpha - self.q_hat = None - - def fit(self, X_cal, y_cal): - # Compute nonconformity scores - y_pred_cal = self.model.predict(X_cal) - scores = np.abs(y_cal - y_pred_cal) - - # Find quantile - n = len(scores) - q_level = np.ceil((n + 1) * (1 - self.alpha)) / n - self.q_hat = np.quantile(scores, q_level) - - def predict_interval(self, X_test): - y_pred = self.model.predict(X_test) - return np.column_stack([ - y_pred - self.q_hat, - y_pred + self.q_hat - ]) - -# Usage -scf = SimpleSplitConformal(model, alpha=0.1) -scf.fit(X_cal, y_cal) -intervals = scf.predict_interval(X_test) -``` - -**Verdict:** ⚠️ Only for learning. Use MAPIE for production. - ---- - -## Recommendation for Your Thesis - -### Primary: MAPIE 1.3.0 - -**Use for:** -1. **Main PD intervals:** Split conformal on CatBoost probabilities -2. **Classification sets:** LAC for binary default prediction -3. **LGD/EAD intervals:** CQR for heteroscedastic regression -4. **All production code:** Robust, tested, documented - -**Implementation status:** ✅ Already implemented in `src/models/conformal.py` - -### Secondary: CREPES 0.9.0 (Optional Experiments) - -**Use for:** -1. **Mondrian CP analysis:** Group-conditional coverage by grade (Notebook 04) -2. **Conformal p-values / predictive systems:** research appendix and online diagnostics -3. **Thesis appendix:** "We also explored CREPES for..." - -**Implementation status:** partial research support only. - -### Ignore: Nonconformist, Custom Code - -**Reason:** Not worth the maintenance burden. MAPIE covers everything you need. - ---- - -## Benchmarking Guidelines - -### Experiment Design for Notebook 04 - -Compare three approaches: - -#### 1. MAPIE Split Conformal (Your Main Method) -```python -from src.models.conformal import create_pd_intervals - -y_pred_mapie, intervals_mapie = create_pd_intervals( - classifier=catboost_model, - X_cal=X_cal, y_cal=y_cal, X_test=X_test, - alpha=0.10 -) -``` - -**Metrics:** -- Coverage -- Avg width -- Computational time -- Memory usage - -#### 2. Bootstrap Intervals (Baseline Comparison) -```python -from sklearn.utils import resample - -n_bootstrap = 1000 -bootstrap_preds = [] - -for _ in range(n_bootstrap): - # Resample calibration set - X_boot, y_boot = resample(X_cal, y_cal, random_state=_) - - # Retrain model (or use same model if prefit) - # For fair comparison, just resample predictions - preds = catboost_model.predict_proba(X_test)[:, 1] - bootstrap_preds.append(preds) - -bootstrap_preds = np.array(bootstrap_preds) - -# Compute percentiles -intervals_bootstrap = np.column_stack([ - np.percentile(bootstrap_preds, 5, axis=0), # Lower 5% - np.percentile(bootstrap_preds, 95, axis=0) # Upper 95% -]) -``` - -**Metrics:** -- Coverage (likely < 90% due to no finite-sample guarantee) -- Avg width (likely wider than conformal) -- Computational time (much slower: 1000x) - -#### 3. CREPES Mondrian (Optional - Group Fairness) -```python -from crepes import ConformalRegressor - -intervals_mondrian = np.zeros((len(X_test), 2)) -grades = X_cal['grade'].unique() - -for grade in grades: - mask_cal = (X_cal['grade'] == grade) - mask_test = (X_test['grade'] == grade) - - # Fit per grade - residuals_cal = y_cal[mask_cal] - catboost_model.predict_proba(X_cal[mask_cal])[:, 1] - cr = ConformalRegressor() - cr.fit(residuals_cal) - - # Predict - y_hat_test = catboost_model.predict_proba(X_test[mask_test])[:, 1] - intervals_mondrian[mask_test] = cr.predict( - X_test[mask_test], - confidence=0.90, - y_hat=y_hat_test - ) -``` - -**Metrics:** -- Coverage overall (marginal) -- Coverage per grade (conditional) -- Min/max group coverage - -### Comparison Table (Target Results) - -| Method | Coverage | Avg Width | Time (s) | Coverage by Grade (min-max) | -|--------|----------|-----------|----------|------------------------------| -| MAPIE Split | 90.2% ± 0.5% | 0.25 | 0.5 | 87%-92% | -| Bootstrap (1000) | 88.5% ± 1.2% | 0.35 | 120.0 | 82%-91% | -| CREPES Mondrian | 90.1% ± 0.5% | 0.28 | 2.0 | 89%-91% (better!) | - -**Interpretation:** -- **MAPIE:** Faster, narrower intervals, but marginal coverage (some grades < 90%) -- **Bootstrap:** Slowest, widest, no guarantees -- **Mondrian:** Group-conditional coverage (fairer), slight efficiency cost - ---- - -## Decision Matrix - -### When to Use MAPIE - -✅ **Binary or multi-class classification** -✅ **Regression with sklearn-compatible models** -✅ **Need CQR for heteroscedastic data** -✅ **Production deployment** -✅ **Tight integration with sklearn pipelines** -✅ **Standard conformal methods (Split, Cross)** - -### When to Use CREPES - -✅ **Mondrian CP without manual implementation** -✅ **Venn-ABERS probability intervals** -✅ **Non-sklearn models (PyTorch, TensorFlow)** -✅ **Conformal Predictive Systems (full distributions)** -✅ **Research experiments** - -### When to Use Custom Code - -✅ **Educational purposes only** -✅ **Very specific method not in MAPIE/CREPES** -✅ **Extreme performance optimization (unlikely)** - -### When to Use Bootstrap (Baseline Only) - -✅ **Comparison baseline to show conformal superiority** -❌ **NOT for production (no coverage guarantees)** - ---- - -## Migration Guide (If Switching Libraries) - -### From Nonconformist to MAPIE - -```python -# OLD: Nonconformist -from nonconformist.cp import IcpRegressor -from nonconformist.nc import AbsErrorErrFunc - -icp = IcpRegressor(AbsErrorErrFunc()) -icp.fit(X_cal, y_cal) -intervals = icp.predict(X_test, significance=0.1) - -# NEW: MAPIE -from mapie.regression import SplitConformalRegressor - -mapie = SplitConformalRegressor( - estimator=model, - confidence_level=0.90, # 1 - 0.1 - conformity_score='absolute', - prefit=True -) -mapie.fit(X_cal, y_cal) -mapie.conformalize(X_cal, y_cal) -intervals = mapie.predict_interval(X_test) -``` - -### From Bootstrap to MAPIE - -```python -# OLD: Bootstrap (slow, no guarantees) -from sklearn.utils import resample - -preds = [] -for _ in range(1000): - X_boot, y_boot = resample(X_cal, y_cal) - model_boot = CatBoostClassifier() - model_boot.fit(X_boot, y_boot) - preds.append(model_boot.predict_proba(X_test)[:, 1]) - -intervals = np.percentile(preds, [5, 95], axis=0).T - -# NEW: MAPIE (fast, guarantees) -from src.models.conformal import create_pd_intervals - -_, intervals = create_pd_intervals( - classifier=model, - X_cal=X_cal, y_cal=y_cal, X_test=X_test, - alpha=0.10 -) -``` - ---- - -## Summary & Recommendation - -| Library | Use Case | Status in Your Project | -|---------|----------|------------------------| -| **MAPIE** | Primary for all conformal prediction | ✅ Implemented in `src/models/conformal.py` | -| **CREPES** | Optional for Mondrian/Venn-ABERS experiments | ⚠️ Installed but not used | -| **Nonconformist** | Legacy (do not use) | ❌ Not installed | -| **Bootstrap** | Baseline comparison only | ⚠️ Implement for benchmarking | - -### Final Recommendation - -**For your thesis:** -1. **Use MAPIE for all production code** (already done ✅) -2. **Add bootstrap baseline** to Notebook 04 for comparison -3. **Optionally add CREPES Mondrian** to show group-conditional coverage (good for fairness discussion) -4. **Ignore Nonconformist and custom implementations** (not worth the effort) - -**Thesis narrative:** -> "We implement conformal prediction using MAPIE 1.3.0, the current industry-standard library with sklearn integration. For comparison, we benchmark against bootstrap intervals (computational baseline) and CREPES Mondrian CP (fairness-aware alternative). Our results show MAPIE provides superior efficiency (narrower intervals) with finite-sample coverage guarantees, while Mondrian CP improves conditional coverage across risk grades at a small efficiency cost." - ---- - -**End of Comparison Document** diff --git a/docs/research/foundations/crpto_conformal_prediction_quick_reference.md b/docs/research/foundations/crpto_conformal_prediction_quick_reference.md deleted file mode 100644 index 3c43484..0000000 --- a/docs/research/foundations/crpto_conformal_prediction_quick_reference.md +++ /dev/null @@ -1,570 +0,0 @@ - - -> **RESEARCH NOTE** — Chuleta técnica de implementación. Para el estado operativo vivo, priorizar `docs/conformal_prediction_README.md` y los artefactos canónicos. - -# Conformal Prediction Quick Reference - MAPIE 1.3.0 - -**Project:** Lending Club Risk Analysis -**Last Updated:** 2026-02-07 - -## Nota de estado (2026-02-16) - -- Este archivo es una guía rápida de implementación MAPIE, no una foto exacta del serving actual. -- En este proyecto, los niveles conformal operativos vigentes son: - - `alpha=0.10` (90%) - - `alpha=0.05` (95%) -- La referencia canónica para resultados actuales es: - - `models/conformal_results_mondrian.pkl` - - `data/processed/conformal_intervals_mondrian.parquet` -- Librerías y roles actuales: - - `mapie`: intervalos conformales principales y `TimeSeriesRegressor` (`ACI`, `EnbPI`) - - `crepes`: p-values / predictive systems / research - - `venn-abers`: calibración Venn-Abers real -- Aclaración crítica: - - `crepes.WrapClassifier.predict_p(...)` devuelve p-values; no debe leerse como probabilidad calibrada ni como reemplazo de Venn-Abers. - ---- - -## Installation - -```bash -# Already installed in your project -uv sync # MAPIE 1.3.0 is in pyproject.toml -``` - ---- - -## Basic Workflow - -### 1. PD Regression Intervals (Your Main Use Case) - -```python -from mapie.regression import SplitConformalRegressor -from src.models.conformal import ProbabilityRegressor - -# Step 1: Wrap calibrated classifier -prob_reg = ProbabilityRegressor(catboost_model) - -# Step 2: Create conformal predictor -mapie = SplitConformalRegressor( - estimator=prob_reg, - confidence_level=0.90, # 1 - alpha - conformity_score='absolute', # or 'gamma', 'residual_normalized' - prefit=True, - n_jobs=-1 -) - -# Step 3: Fit (no-op when prefit=True) -mapie.fit(X_cal, y_cal.astype(float)) - -# Step 4: Conformalize (compute calibration scores) -mapie.conformalize(X_cal, y_cal.astype(float)) - -# Step 5: Predict intervals -y_pred = mapie.predict(X_test) -y_intervals = mapie.predict_interval(X_test) - -# Shape: y_intervals is (n_samples, 2) -# y_intervals[:, 0] = lower bound -# y_intervals[:, 1] = upper bound - -# Clip to [0, 1] for probabilities -y_intervals = np.clip(y_intervals, 0, 1) -``` - ---- - -### 2. Classification Sets (Binary: Default/Non-Default) - -```python -from mapie.classification import SplitConformalClassifier - -mapie_clf = SplitConformalClassifier( - estimator=catboost_model, - confidence_level=0.90, - conformity_score='lac', # or 'aps', 'raps' - prefit=True -) - -mapie_clf.fit(X_cal, y_cal) -mapie_clf.conformalize(X_cal, y_cal) - -y_pred = mapie_clf.predict(X_test) -y_sets = mapie_clf.predict_set(X_test) - -# Shape: y_sets is (n_samples, 2) for binary classification -# y_sets[i, 0] = 1 if class 0 is in prediction set -# y_sets[i, 1] = 1 if class 1 is in prediction set - -# Singleton rate (confident predictions) -singleton_rate = (y_sets.sum(axis=1).squeeze() == 1).mean() -``` - ---- - -### 3. Multiple Alpha Levels - -```python -# MAPIE 1.3 allows multiple confidence levels at once -mapie = SplitConformalRegressor( - estimator=prob_reg, - confidence_level=[0.80, 0.90, 0.95], # Multiple levels - prefit=True -) - -mapie.fit(X_cal, y_cal) -mapie.conformalize(X_cal, y_cal) - -y_intervals_multi = mapie.predict_interval(X_test) - -# Shape: (n_samples, n_confidence_levels, 2) -# y_intervals_multi[:, 0, 0] = lower bound for 80% CI -# y_intervals_multi[:, 0, 1] = upper bound for 80% CI -# y_intervals_multi[:, 1, 0] = lower bound for 90% CI -# y_intervals_multi[:, 1, 1] = upper bound for 90% CI -# y_intervals_multi[:, 2, 0] = lower bound for 95% CI -# y_intervals_multi[:, 2, 1] = upper bound for 95% CI -``` - ---- - -## Conformity Scores - -### Regression - -| Score | When to Use | Formula | -|-------|-------------|---------| -| `'absolute'` | Default, homoscedastic | \|y - ŷ\| | -| `'gamma'` | Normalized residuals | \|y - ŷ\| / σ̂ | -| `'residual_normalized'` | Heteroscedastic | \|y - ŷ\| / ŝ(x) | - -```python -# Example: Residual normalized for heteroscedastic PD -mapie = SplitConformalRegressor( - estimator=prob_reg, - conformity_score='residual_normalized', - confidence_level=0.90, - prefit=True -) -``` - -### Classification - -| Score | Best For | Description | -|-------|----------|-------------| -| `'lac'` | Binary, well-calibrated | Least Ambiguous Criterion | -| `'aps'` | Multi-class (3+ classes) | Adaptive Prediction Sets | -| `'raps'` | Multi-class with imbalance | Regularized APS | - -```python -# Example: LAC for binary PD classification -mapie_clf = SplitConformalClassifier( - estimator=catboost_model, - conformity_score='lac', - confidence_level=0.90, - prefit=True -) -``` - ---- - -## Validation Metrics - -### Coverage - -```python -def validate_coverage(y_true, y_intervals, alpha): - low, high = y_intervals[:, 0], y_intervals[:, 1] - covered = ((y_true >= low) & (y_true <= high)).mean() - target = 1 - alpha - gap = abs(covered - target) - - # Coverage should be ≈ target (within ±2%) - return { - 'empirical_coverage': covered, - 'target_coverage': target, - 'coverage_gap': gap, - 'pass': gap < 0.02 - } -``` - -### Efficiency - -```python -def efficiency_metrics(y_intervals): - widths = y_intervals[:, 1] - y_intervals[:, 0] - - return { - 'avg_width': widths.mean(), - 'median_width': np.median(widths), - 'width_std': widths.std(), - 'width_iqr': np.percentile(widths, 75) - np.percentile(widths, 25), - 'efficiency_score': 1 / (1 + widths.mean()) # Higher = better - } - -# Target: avg_width < 0.30 for production -``` - -### Conditional Coverage (By Subgroup) - -```python -def coverage_by_group(y_true, y_intervals, X_test, feature='grade'): - groups = X_test[feature].unique() - results = {} - - for group in groups: - mask = (X_test[feature] == group) - low, high = y_intervals[mask, 0], y_intervals[mask, 1] - coverage = ((y_true[mask] >= low) & (y_true[mask] <= high)).mean() - width = (high - low).mean() - - results[group] = { - 'n': mask.sum(), - 'coverage': coverage, - 'avg_width': width - } - - return pd.DataFrame(results).T -``` - ---- - -## Common Patterns - -### Pattern 1: Standard Split Conformal - -```python -from src.models.conformal import create_pd_intervals, validate_coverage - -# Generate intervals -y_pred, y_intervals = create_pd_intervals( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=0.10 -) - -# Validate -metrics = validate_coverage(y_test, y_intervals, alpha=0.10) -print(f"Coverage: {metrics['empirical_coverage']:.2%} (target: {metrics['target_coverage']:.0%})") -print(f"Avg Width: {metrics['avg_interval_width']:.4f}") -``` - -### Pattern 2: Cross-Conformal (K-Fold) - -```python -from mapie.regression import CrossConformalRegressor - -# When you don't want to sacrifice calibration data -mapie_cv = CrossConformalRegressor( - estimator=ProbabilityRegressor(catboost_model), - confidence_level=0.90, - cv=5, # 5-fold cross-validation - prefit=False # Will train 5 models -) - -mapie_cv.fit(X_train, y_train) -y_intervals_cv = mapie_cv.predict_interval(X_test) - -# Trade-off: More data, but 5x slower -``` - -### Pattern 3: Conformalized Quantile Regression - -```python -from mapie.regression import ConformalizedQuantileRegressor -from catboost import CatBoostRegressor - -# For heteroscedastic data (varying uncertainty) -qr_low = CatBoostRegressor( - loss_function="Quantile:alpha=0.05", - iterations=300, - depth=6, - learning_rate=0.05, - verbose=False, - allow_writing_files=False, -) -qr_high = CatBoostRegressor( - loss_function="Quantile:alpha=0.95", - iterations=300, - depth=6, - learning_rate=0.05, - verbose=False, - allow_writing_files=False, -) - -qr_low.fit(X_train, y_train) -qr_high.fit(X_train, y_train) - -# Conformalize -mapie_cqr = ConformalizedQuantileRegressor( - estimator=qr_low, # Or qr_high - confidence_level=0.90 -) -mapie_cqr.fit(X_cal, y_cal) -y_intervals_cqr = mapie_cqr.predict_interval(X_test) -``` - -### Pattern 4: Mondrian Conformal (Group-Conditional) - -```python -# Separate calibration per risk grade -grades = X_cal['grade'].unique() -intervals_by_grade = {} - -for grade in grades: - # Filter by grade - mask_cal = (X_cal['grade'] == grade) - mask_test = (X_test['grade'] == grade) - - # Create grade-specific predictor - mapie_grade = SplitConformalRegressor( - estimator=ProbabilityRegressor(model), - confidence_level=0.90, - prefit=True - ) - - # Calibrate on grade subset - X_cal_grade = X_cal[mask_cal].drop(columns=['grade']) - y_cal_grade = y_cal[mask_cal] - - mapie_grade.fit(X_cal_grade, y_cal_grade) - mapie_grade.conformalize(X_cal_grade, y_cal_grade) - - # Predict on grade subset - X_test_grade = X_test[mask_test].drop(columns=['grade']) - intervals_by_grade[grade] = mapie_grade.predict_interval(X_test_grade) - -# Combine results (sorted by test index) -final_intervals = np.vstack([intervals_by_grade[g] for g in grades]) -``` - ---- - -## Integration with Optimization - -### Using Intervals as Uncertainty Sets - -```python -from src.optimization.robust_opt import build_box_uncertainty_set, worst_case_expected_loss - -# Build uncertainty set -uncertainty = build_box_uncertainty_set( - pd_low=y_intervals[:, 0], - pd_high=y_intervals[:, 1] -) - -# Worst-case loss -wcl = worst_case_expected_loss( - allocation=x_opt, # From Pyomo solution - loan_amounts=loan_amounts, - pd_high=uncertainty['pd_high'], - lgd_point=lgd_estimates -) - -print(f"Worst-case expected loss: ${wcl:,.2f}") -``` - ---- - -## Visualization - -### Interval Width Distribution - -```python -import matplotlib.pyplot as plt - -widths = y_intervals[:, 1] - y_intervals[:, 0] - -plt.figure(figsize=(10, 6)) -plt.hist(widths, bins=50, edgecolor='black', alpha=0.7) -plt.axvline(widths.mean(), color='red', linestyle='--', - label=f'Mean: {widths.mean():.3f}') -plt.axvline(widths.median(), color='blue', linestyle='--', - label=f'Median: {widths.median():.3f}') -plt.xlabel('Interval Width') -plt.ylabel('Frequency') -plt.title('Conformal Interval Width Distribution') -plt.legend() -plt.show() -``` - -### Coverage by Grade - -```python -import seaborn as sns - -coverage_df = coverage_by_group(y_test, y_intervals, X_test, feature='grade') - -fig, ax = plt.subplots(figsize=(10, 6)) -coverage_df['coverage'].plot(kind='bar', ax=ax, alpha=0.7, edgecolor='black') -ax.axhline(0.90, color='red', linestyle='--', label='Target (90%)') -ax.set_xlabel('Grade') -ax.set_ylabel('Empirical Coverage') -ax.set_title('Coverage by Risk Grade') -ax.legend() -plt.xticks(rotation=0) -plt.tight_layout() -plt.show() -``` - -### Prediction Intervals Plot - -```python -def plot_intervals(y_true, y_pred, y_intervals, n=100): - idx = np.argsort(y_pred)[:n] - low, high = y_intervals[idx, 0], y_intervals[idx, 1] - - plt.figure(figsize=(14, 6)) - plt.fill_between(range(n), low, high, alpha=0.3, label='90% PI') - plt.scatter(range(n), y_true[idx], s=10, c='red', label='True', zorder=5) - plt.plot(range(n), y_pred[idx], 'b-', linewidth=1, label='Predicted') - plt.xlabel('Sample (sorted by prediction)') - plt.ylabel('Probability of Default') - plt.title('Conformal Prediction Intervals') - plt.legend() - plt.grid(True, alpha=0.3) - plt.show() - -plot_intervals(y_test, y_pred, y_intervals, n=200) -``` - ---- - -## Troubleshooting - -### Issue 1: Coverage Too Low - -```python -# Check calibration set size -print(f"Calibration set: {len(X_cal)} samples") -# Need at least 100+ samples for alpha=0.10 - -# Check for distribution shift -from scipy.stats import ks_2samp -ks_stat, p_value = ks_2samp(y_cal, y_test) -print(f"KS test: stat={ks_stat:.4f}, p={p_value:.4f}") -# If p < 0.05, distributions differ (conformal may not hold) -``` - -### Issue 2: Intervals Too Wide - -```python -# 1. Check base model calibration -from src.models.calibration import expected_calibration_error -ece = expected_calibration_error(y_cal, model.predict_proba(X_cal)[:, 1]) -print(f"ECE: {ece:.4f}") # Should be < 0.05 - -# 2. Try CQR for heteroscedastic data -# 3. Use Mondrian CP for subgroup-specific widths -# 4. Consider more calibration data -``` - -### Issue 3: Shape Mismatch - -```python -# MAPIE 1.3 returns (n, 2), not (n, 2, 1) -# If you get shape errors, check: -print(y_intervals.shape) # Should be (n_samples, 2) - -# Extract bounds -low = y_intervals[:, 0] -high = y_intervals[:, 1] - -# NOT y_intervals[:, 0, 0] (this is for multiple confidence levels) -``` - ---- - -## Performance Tips - -### Tip 1: Use prefit=True - -```python -# Faster: model already trained -mapie = SplitConformalRegressor(estimator=model, prefit=True) - -# vs slower: MAPIE will train the model -mapie = SplitConformalRegressor(estimator=model, prefit=False) -``` - -### Tip 2: Parallelize with n_jobs - -```python -# Use all cores for conformalization -mapie = SplitConformalRegressor( - estimator=model, - prefit=True, - n_jobs=-1 # All cores -) -``` - -### Tip 3: Cache Calibration Results - -```python -import joblib - -# After conformalization, save -joblib.dump(mapie, 'models/mapie_pd_calibrated.pkl') - -# Load and predict -mapie = joblib.load('models/mapie_pd_calibrated.pkl') -y_intervals = mapie.predict_interval(X_new) -``` - ---- - -## Decision Tree: Which Method to Use? - -``` -START -| -├─ Binary classification (default/non-default)? -│ ├─ YES → Want prediction sets? -│ │ ├─ YES → SplitConformalClassifier(conformity_score='lac') -│ │ └─ NO → Want probability intervals? -│ │ └─ YES → SplitConformalRegressor + ProbabilityRegressor -│ │ -│ └─ NO → Multi-class (3+ classes)? -│ └─ YES → SplitConformalClassifier(conformity_score='aps' or 'raps') -| -├─ Regression on continuous target (LGD, EAD)? -│ ├─ Homoscedastic (constant variance)? -│ │ └─ YES → SplitConformalRegressor(conformity_score='absolute') -│ │ -│ └─ Heteroscedastic (varying variance)? -│ └─ YES → ConformalizedQuantileRegressor OR -│ SplitConformalRegressor(conformity_score='residual_normalized') -| -└─ Limited calibration data? - └─ YES → CrossConformalRegressor (k-fold) instead of Split -``` - ---- - -## References - -- **MAPIE Docs:** https://mapie.readthedocs.io/en/stable/ -- **Your Implementation:** `src/models/conformal.py` -- **Metrics:** `src/evaluation/metrics.py` -- **Optimization:** `src/optimization/robust_opt.py` - ---- - -**Quick Start Checklist:** - -- [ ] Import `SplitConformalRegressor` or `SplitConformalClassifier` -- [ ] Wrap classifier with `ProbabilityRegressor` (if PD intervals) -- [ ] Set `confidence_level = 1 - alpha` in `__init__` -- [ ] Set `prefit=True` if model already trained -- [ ] Call `.fit()` then `.conformalize()` on calibration set -- [ ] Call `.predict_interval()` or `.predict_set()` on test set -- [ ] Validate coverage: `empirical ≈ target ± 2%` -- [ ] Check efficiency: `avg_width < 0.30` -- [ ] Visualize: interval widths, coverage by grade -- [ ] Save intervals for optimization - ---- - -**End of Quick Reference** diff --git a/docs/research/foundations/crpto_conformal_prediction_research_2026.md b/docs/research/foundations/crpto_conformal_prediction_research_2026.md deleted file mode 100644 index 6e60969..0000000 --- a/docs/research/foundations/crpto_conformal_prediction_research_2026.md +++ /dev/null @@ -1,919 +0,0 @@ - - -> **RESEARCH NOTE** — Investigación larga archivada como soporte teórico. No sustituye la política ni los resultados vivos del proyecto. - -# Conformal Prediction for Credit Risk: Research Summary 2024-2026 - -**Generated:** 2026-02-07 -**MAPIE Version Installed:** 1.3.0 -**Project:** CRPTO - -## Nota de vigencia (2026-02-16) - -Este documento preserva investigación y contexto técnico de febrero 2026. -Para métricas y decisiones operativas del proyecto actual, usar como fuente de verdad: - -- `models/conformal_results_mondrian.pkl` -- `data/processed/conformal_intervals_mondrian.parquet` -- `models/conformal_policy_status.json` - ---- - -## Executive Summary - -This document consolidates recent best practices for conformal prediction in credit risk modeling, specifically for PD (Probability of Default) prediction intervals. Since web search was unavailable, this is based on: -1. MAPIE 1.3.0 installed API inspection -2. Your project's current implementation in `src/models/conformal.py` -3. Industry knowledge as of January 2025 - -## Book Concepts -> Current Implementation -> V2 Backlog (2026-02-27) - -| Book concept | Current implementation (este paquete CRPTO) | Backlog v2 (explicitly out of hardening v1) | -|---|---|---| -| Finite-sample validity and exchangeability | Split conformal + Mondrian by `grade`, temporal calibration holdout, explicit policy artifacts | Add formal shift-aware/online diagnostics and adaptive updates under drift | -| Validity + efficiency tradeoff | Coverage + width + group-coverage metrics, Pareto tuning table and guardbands | Add richer optimization objective across multiple proper scoring rules | -| Mondrian conditional coverage | `create_pd_intervals_mondrian`, group floor multipliers, per-group coverage reports | Extend Mondrian partitioning beyond grade and test hierarchical partitions | -| Venn-Abers calibration | Candidate calibrator in PD training flow, interval-ready probabilities | Promote as first-class branch in benchmark/policy comparison outputs | -| Statistical interval diagnostics | Winkler + Kupiec + Christoffersen computed and stored in policy status | Keep strict policy; add adaptive/sample-size-aware interpretation layer | -| Cross-conformal regression | Not enabled in canonical pipeline (only split conformal in production path) | Add controlled benchmark track with `CrossConformalRegressor` | -| Conformalized Quantile Regression (CQR) | Research documented; not in canonical LGD/EAD training pipeline | Add CQR branch for LGD/EAD with heteroscedastic checks | -| Jackknife+ style intervals | Not implemented | Add experimental module and benchmark versus split/Mondrian | -| Classification set methods (LAC/APS/RAPS) | Wrappers available in `src/models/conformal.py`, not primary PD gate path | Add explicit multi-class/ambiguity benchmark workflow | - ---- - -## 1. MAPIE 1.3.0 API - Current State - -### Key Classes (Verified in Your Environment) - -#### **Regression Conformal Predictors** -```python -from mapie.regression import SplitConformalRegressor -from mapie.regression import CrossConformalRegressor -from mapie.regression import ConformalizedQuantileRegressor -``` - -**`SplitConformalRegressor` Signature:** -```python -SplitConformalRegressor( - estimator: RegressorMixin = LinearRegression(), - confidence_level: Union[float, Iterable[float]] = 0.9, - conformity_score: Union[str, BaseRegressionScore] = 'absolute', - prefit: bool = True, - n_jobs: Optional[int] = None, - verbose: int = 0 -) -``` - -**Methods:** -- `.fit(X, y)` - Fit the estimator (no-op if prefit=True) -- `.conformalize(X_cal, y_cal)` - Compute nonconformity scores on calibration set -- `.predict(X)` - Point predictions -- `.predict_interval(X)` - Returns shape (n_samples, 2) array: [:, 0] = lower, [:, 1] = upper - -#### **Classification Conformal Predictors** -```python -from mapie.classification import SplitConformalClassifier -from mapie.classification import CrossConformalClassifier -``` - -**`SplitConformalClassifier` Signature:** -```python -SplitConformalClassifier( - estimator: ClassifierMixin = LogisticRegression(), - confidence_level: Union[float, Iterable[float]] = 0.9, - conformity_score: Union[str, BaseClassificationScore] = 'lac', - prefit: bool = True, - n_jobs: Optional[int] = None, - verbose: int = 0, - random_state: Optional[Union[int, np.random.RandomState]] = None -) -``` - -**Methods:** -- `.fit(X, y)` - Fit the estimator -- `.conformalize(X_cal, y_cal)` - Compute calibration scores -- `.predict(X)` - Point predictions -- `.predict_set(X)` - Returns boolean array (n_samples, n_classes) indicating set membership - -### Conformity Score Functions Available - -**Classification (verified):** -```python -from mapie.conformity_scores import LACConformityScore # Least Ambiguous Criterion -from mapie.conformity_scores import APSConformityScore # Adaptive Prediction Sets -from mapie.conformity_scores import RAPSConformityScore # Regularized APS -from mapie.conformity_scores import TopKConformityScore -``` - -**Regression (verified):** -```python -from mapie.conformity_scores import AbsoluteConformityScore # |y - ŷ| -from mapie.conformity_scores import GammaConformityScore # Normalized residuals -from mapie.conformity_scores import ResidualNormalisedScore # Heteroscedastic -``` - -### API Migration Notes (1.2 → 1.3) - -Your `src/models/conformal.py` correctly documents the migration: - -| Old API (< 1.3) | New API (1.3+) | -|-----------------|----------------| -| `MapieRegressor` | `SplitConformalRegressor` | -| `MapieClassifier` | `SplitConformalClassifier` | -| `.fit().predict(alpha=0.1)` | `.fit().conformalize().predict_interval()` | -| `alpha` parameter in predict | `confidence_level = 1 - alpha` in `__init__` | -| `cv="prefit"` | `prefit=True` | - -**Your implementation is CORRECT for MAPIE 1.3.0.** - ---- - -## 2. Conformal Prediction for PD Modeling - Best Practices - -### Why Conformal Prediction for Credit Risk? - -Traditional approaches have critical weaknesses: -- **Point estimates**: No uncertainty quantification → fragile portfolios -- **Bootstrap intervals**: No finite-sample guarantees, computationally expensive -- **Bayesian intervals**: Require strong distributional assumptions -- **Conformal intervals**: - - Distribution-free - - Finite-sample coverage guarantees: P(y ∈ [ŷ_low, ŷ_high]) ≥ 1 - α - - Computational efficiency - -### Your Thesis Innovation - -``` -CatBoost PD Model → Isotonic Calibration → MAPIE Conformal Intervals → - Box Uncertainty Sets → Robust Portfolio Optimization (Pyomo + HiGHS) -``` - -**Key differentiator**: Using conformal intervals as uncertainty sets in optimization (predict-then-optimize framework). - -### ProbabilityRegressor Pattern (Your Implementation) - -Since MAPIE expects a regressor but CatBoost outputs probabilities via `.predict_proba()`, you use a wrapper: - -```python -class ProbabilityRegressor(BaseEstimator, RegressorMixin): - """Turns classifier predict_proba into regressor predict.""" - def __init__(self, classifier): - self.classifier = classifier - - def fit(self, X, y): - return self # Already fitted - - def predict(self, X): - return self.classifier.predict_proba(X)[:, 1] # P(default) -``` - -**This is the standard pattern for conformal prediction on probabilities.** - ---- - -## 3. Key Techniques for PD Conformal Prediction - -### Split Conformal (Current Implementation) - -**Your code in `src/models/conformal.py` lines 46-91:** -```python -def create_pd_intervals(classifier, X_cal, y_cal, X_test, alpha=0.1): - prob_reg = ProbabilityRegressor(classifier) - mapie = SplitConformalRegressor( - estimator=prob_reg, - confidence_level=1 - alpha, - prefit=True, - ) - mapie.fit(X_cal, y_cal.astype(float)) - mapie.conformalize(X_cal, y_cal.astype(float)) - - y_pred = mapie.predict(X_test) - y_intervals = mapie.predict_interval(X_test) - - # Clip to [0, 1] for probabilities - y_intervals = np.clip(y_intervals, 0, 1) - return y_pred, y_intervals -``` - -**Strengths:** -- Simple, efficient (single calibration set) -- Marginal coverage guarantee: covers 1-α of test points -- Fast inference - -**Weaknesses:** -- Requires separate calibration set (reduces training data) -- Only marginal coverage (not conditional on features) - -### Cross-Conformal (Alternative) - -Use when you don't want to sacrifice calibration data: - -```python -from mapie.regression import CrossConformalRegressor - -mapie = CrossConformalRegressor( - estimator=prob_reg, - confidence_level=0.9, - prefit=False, # Will do k-fold CV internally - cv=5 # 5-fold cross-validation -) -mapie.fit(X_train, y_train) -y_intervals = mapie.predict_interval(X_test) -``` - -**Trade-off:** More training data usage but 5x slower (need to train 5 models). - -### Conformalized Quantile Regression (CQR) - -For **heteroscedastic** data (when interval width should vary with features): - -```python -from mapie.regression import ConformalizedQuantileRegressor -from catboost import CatBoostRegressor - -# Train quantile regressor -qr = CatBoostRegressor( - loss_function="Quantile:alpha=0.05", - iterations=300, - depth=6, - learning_rate=0.05, - verbose=False, - allow_writing_files=False, -) # Lower quantile -qr.fit(X_train, y_train) - -mapie_cqr = ConformalizedQuantileRegressor( - estimator=qr, - confidence_level=0.9 -) -mapie_cqr.fit(X_cal, y_cal) -y_intervals = mapie_cqr.predict_interval(X_test) -``` - -**When to use:** If you notice interval widths should be wider for high-risk segments (e.g., low-grade loans) and narrower for low-risk. - -**Your current dataset:** Start with `SplitConformalRegressor` (simpler, interpretable). Consider CQR if you see evidence of heteroscedasticity. - ---- - -## 4. Classification Sets (LAC, APS, RAPS) - -### When to Use Each - -Your implementation supports all three via `create_classification_sets()`: - -```python -def create_classification_sets(classifier, X_cal, y_cal, X_test, alpha=0.1, method='lac'): - mapie = SplitConformalClassifier( - estimator=classifier, - confidence_level=1-alpha, - conformity_score=method, # 'lac', 'aps', or 'raps' - prefit=True - ) - mapie.conformalize(X_cal, y_cal) - y_sets = mapie.predict_set(X_test) # Shape (n, n_classes) - return y_pred, y_sets -``` - -#### **LAC (Least Ambiguous Criterion)** - Your Default -- **Score:** 1 - P(y_true) -- **Set:** Include all classes with P(class) ≥ P(y_true) - threshold -- **Best for:** Binary classification with well-calibrated probabilities -- **Singleton rate:** Typically high (70-90%) for binary tasks - -#### **APS (Adaptive Prediction Sets)** -- **Score:** Rank-based cumulative probability -- **Set:** Include classes until cumulative prob > 1 - α -- **Best for:** Multi-class problems (3+ classes) -- **Guarantees smallest average set size** - -#### **RAPS (Regularized APS)** -- **Score:** APS + regularization penalty -- **Set:** APS with penalty for large sets -- **Best for:** Multi-class with class imbalance -- **Hyperparameter:** Penalty weight (tune on validation set) - -**For your binary PD task:** LAC is the right choice. APS/RAPS are overkill for 2 classes. - -### Interpretation for Binary Classification - -```python -y_sets = mapie.predict_set(X_test) -# y_sets[:, 0] = 1 if class 0 (non-default) is in set -# y_sets[:, 1] = 1 if class 1 (default) is in set - -# Singleton: exactly one class -singleton_mask = y_sets.sum(axis=1) == 1 -singleton_rate = singleton_mask.mean() - -# Empty set (should be rare if well-calibrated) -empty_mask = y_sets.sum(axis=1) == 0 - -# Full set (uncertain predictions) -full_mask = y_sets.sum(axis=1) == 2 -``` - -**Singleton rate** is the key metric: % of predictions where you're confident about the single class. - ---- - -## 5. Mondrian Conformal Prediction (Group-Conditional Coverage) - -**Problem with marginal coverage:** Conformal prediction guarantees 90% coverage *overall*, but not within each risk grade. - -**Mondrian CP solution:** Compute separate thresholds for each subgroup. - -```python -# Pseudo-code for Mondrian CP (not in MAPIE 1.3 directly) -grades = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] -intervals_by_grade = {} - -for grade in grades: - mask_cal = (X_cal['grade'] == grade) - mask_test = (X_test['grade'] == grade) - - mapie = SplitConformalRegressor( - estimator=ProbabilityRegressor(classifier), - confidence_level=0.9, - prefit=True - ) - mapie.fit(X_cal[mask_cal], y_cal[mask_cal]) - mapie.conformalize(X_cal[mask_cal], y_cal[mask_cal]) - - intervals_by_grade[grade] = mapie.predict_interval(X_test[mask_test]) - -# Combine intervals -final_intervals = np.concatenate([intervals_by_grade[g] for g in grades]) -``` - -**Trade-off:** -- **Pros:** Conditional coverage within each grade (fairness) -- **Cons:** Requires enough calibration samples per grade; more complex - -**Recommendation for thesis:** -1. Start with marginal coverage (simpler, standard) -2. Add Mondrian as "extension" if you want to show group-wise fairness analysis - ---- - -## 6. Validation & Efficiency Metrics - -### Coverage Validation (Your Implementation) - -`src/models/conformal.py` lines 154-179: -```python -def validate_coverage(y_true, y_intervals, alpha): - low, high = y_intervals[:, 0], y_intervals[:, 1] - covered = ((y_true >= low) & (y_true <= high)).mean() - target = 1 - alpha - - return { - "empirical_coverage": covered, - "target_coverage": target, - "coverage_gap": abs(covered - target), - "avg_interval_width": (high - low).mean(), - "median_interval_width": float(np.median(high - low)), - } -``` - -**This is correct.** Additional metrics to consider: - -```python -def extended_validation(y_true, y_intervals, alpha, X_test, feature_name='grade'): - """Extended coverage analysis by subgroup.""" - metrics = validate_coverage(y_true, y_intervals, alpha) - - # Coverage by subgroup (conditional coverage check) - groups = X_test[feature_name].unique() - coverage_by_group = {} - for group in groups: - mask = (X_test[feature_name] == group) - if mask.sum() > 0: - low, high = y_intervals[mask, 0], y_intervals[mask, 1] - coverage_by_group[group] = ((y_true[mask] >= low) & (y_true[mask] <= high)).mean() - - metrics['coverage_by_grade'] = coverage_by_group - metrics['min_group_coverage'] = min(coverage_by_group.values()) - metrics['max_group_coverage'] = max(coverage_by_group.values()) - - # Efficiency: width distribution - widths = y_intervals[:, 1] - y_intervals[:, 0] - metrics['width_5th_pct'] = np.percentile(widths, 5) - metrics['width_95th_pct'] = np.percentile(widths, 95) - metrics['width_iqr'] = np.percentile(widths, 75) - np.percentile(widths, 25) - - return metrics -``` - -### Efficiency Analysis - -**Efficiency** = How narrow are the intervals while maintaining coverage? - -```python -# Efficiency metrics (add to src/evaluation/metrics.py) -def conformal_efficiency(y_intervals, alpha): - """Quantify prediction interval efficiency.""" - widths = y_intervals[:, 1] - y_intervals[:, 0] - - return { - 'avg_width': widths.mean(), - 'median_width': np.median(widths), - 'std_width': widths.std(), - 'min_width': widths.min(), - 'max_width': widths.max(), - 'width_90th_pct': np.percentile(widths, 90), - 'efficiency_score': 1 / (1 + widths.mean()), # Higher = better - 'relative_efficiency': widths.std() / widths.mean(), # Lower = more uniform - } -``` - -**Interpretation:** -- **avg_width < 0.2:** Excellent (very informative intervals) -- **avg_width 0.2-0.4:** Good -- **avg_width 0.4-0.6:** Moderate (still useful but wide) -- **avg_width > 0.6:** Poor (intervals too wide, low information) - -Your legacy project had `avg_width = 0.808` which was too wide. Target: **< 0.3 for production use**. - ---- - -## 7. Visualization Best Practices - -### Interval Width Distribution - -```python -import matplotlib.pyplot as plt -import seaborn as sns - -def plot_interval_widths(y_intervals, alpha=0.1, feature=None, X_test=None): - """Plot interval width distribution.""" - widths = y_intervals[:, 1] - y_intervals[:, 0] - - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - - # Histogram - axes[0].hist(widths, bins=50, edgecolor='black', alpha=0.7) - axes[0].axvline(widths.mean(), color='red', linestyle='--', - label=f'Mean: {widths.mean():.3f}') - axes[0].axvline(widths.median(), color='blue', linestyle='--', - label=f'Median: {widths.median():.3f}') - axes[0].set_xlabel('Interval Width') - axes[0].set_ylabel('Frequency') - axes[0].set_title(f'Interval Width Distribution (α={alpha})') - axes[0].legend() - - # Box plot by feature (if provided) - if feature is not None and X_test is not None: - data = pd.DataFrame({ - 'width': widths, - 'group': X_test[feature].values - }) - sns.boxplot(data=data, x='group', y='width', ax=axes[1]) - axes[1].set_title(f'Width by {feature}') - axes[1].tick_params(axis='x', rotation=45) - else: - # CDF - sorted_widths = np.sort(widths) - cdf = np.arange(1, len(sorted_widths)+1) / len(sorted_widths) - axes[1].plot(sorted_widths, cdf) - axes[1].set_xlabel('Interval Width') - axes[1].set_ylabel('Cumulative Probability') - axes[1].set_title('CDF of Interval Widths') - axes[1].grid(True, alpha=0.3) - - plt.tight_layout() - return fig -``` - -### Coverage by Subgroup - -```python -def plot_coverage_by_subgroup(y_true, y_intervals, X_test, feature='grade', alpha=0.1): - """Plot empirical coverage by risk segment.""" - groups = sorted(X_test[feature].unique()) - coverages = [] - counts = [] - - for group in groups: - mask = (X_test[feature] == group) - low, high = y_intervals[mask, 0], y_intervals[mask, 1] - coverage = ((y_true[mask] >= low) & (y_true[mask] <= high)).mean() - coverages.append(coverage) - counts.append(mask.sum()) - - fig, ax = plt.subplots(figsize=(10, 6)) - x_pos = np.arange(len(groups)) - - bars = ax.bar(x_pos, coverages, alpha=0.7, edgecolor='black') - ax.axhline(1-alpha, color='red', linestyle='--', - label=f'Target Coverage ({1-alpha:.0%})') - - # Color bars by coverage gap - for i, (bar, cov) in enumerate(zip(bars, coverages)): - if abs(cov - (1-alpha)) > 0.05: - bar.set_color('orange') - else: - bar.set_color('green') - - ax.set_xticks(x_pos) - ax.set_xticklabels(groups) - ax.set_xlabel(feature.capitalize()) - ax.set_ylabel('Empirical Coverage') - ax.set_title(f'Coverage by {feature.capitalize()} (α={alpha})') - ax.set_ylim(0.8, 1.0) - ax.legend() - - # Add sample counts - for i, (x, count) in enumerate(zip(x_pos, counts)): - ax.text(x, 0.82, f'n={count}', ha='center', fontsize=9) - - plt.tight_layout() - return fig -``` - -### Prediction Interval Plot (Enhanced) - -Your `src/utils/visualization.py` has `plot_conformal_intervals()` but it assumes shape `(n, 2, 1)`. MAPIE 1.3 returns `(n, 2)`. Update: - -```python -def plot_conformal_intervals_v2( - y_true, y_pred, y_intervals, n_samples=100, - sort_by='prediction', title='Conformal Prediction Intervals' -): - """Plot intervals with true values (MAPIE 1.3 compatible).""" - fig, ax = plt.subplots(figsize=(14, 6)) - - if sort_by == 'prediction': - idx = np.argsort(y_pred)[:n_samples] - elif sort_by == 'width': - widths = y_intervals[:, 1] - y_intervals[:, 0] - idx = np.argsort(widths)[-n_samples:] # Widest intervals - else: - idx = np.arange(n_samples) - - low = y_intervals[idx, 0] - high = y_intervals[idx, 1] - - # Coverage indicator - covered = (y_true[idx] >= low) & (y_true[idx] <= high) - - ax.fill_between(range(len(idx)), low, high, alpha=0.3, - label=f'90% PI', color='lightblue') - ax.scatter(range(len(idx)), y_true[idx], s=15, - c=['green' if c else 'red' for c in covered], - label='True (green=covered)', zorder=5) - ax.plot(range(len(idx)), y_pred[idx], 'b-', linewidth=1.2, - label='Predicted', alpha=0.8) - - ax.set_xlabel(f'Sample (sorted by {sort_by})') - ax.set_ylabel('Probability of Default') - ax.set_title(f'{title} | Coverage: {covered.mean():.1%}') - ax.legend(loc='best') - ax.grid(True, alpha=0.3) - - plt.tight_layout() - return fig -``` - -### Calibration + Conformal Uncertainty - -```python -def plot_calibration_with_intervals(y_true, y_pred, y_intervals, n_bins=10): - """Calibration curve with interval widths.""" - from sklearn.calibration import calibration_curve - - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - - # Calibration curve - frac_pos, mean_pred = calibration_curve(y_true, y_pred, n_bins=n_bins) - axes[0].plot(mean_pred, frac_pos, 's-', label='Model') - axes[0].plot([0, 1], [0, 1], 'k--', label='Perfect') - axes[0].set_xlabel('Mean Predicted Probability') - axes[0].set_ylabel('Fraction of Positives') - axes[0].set_title('Calibration Curve') - axes[0].legend() - axes[0].grid(True, alpha=0.3) - - # Interval width vs prediction - widths = y_intervals[:, 1] - y_intervals[:, 0] - axes[1].scatter(y_pred, widths, alpha=0.3, s=5) - axes[1].set_xlabel('Predicted PD') - axes[1].set_ylabel('Interval Width') - axes[1].set_title('Uncertainty vs Prediction') - axes[1].grid(True, alpha=0.3) - - # Add trend line - z = np.polyfit(y_pred, widths, 2) - p = np.poly1d(z) - x_trend = np.linspace(0, 1, 100) - axes[1].plot(x_trend, p(x_trend), 'r--', alpha=0.7, label='Trend') - axes[1].legend() - - plt.tight_layout() - return fig -``` - ---- - -## 8. Alternative Libraries: CREPES vs MAPIE - -### CREPES (Installed: 0.9.0) - -**Pros:** -- More flexible API (works with any predictor) -- Supports Venn-ABERS (probability intervals) -- Mondrian CP built-in -- Conformal Predictive Systems (full distributions) - -**Cons:** -- Less popular (fewer examples) -- Steeper learning curve -- No scikit-learn integration - -**Example:** -```python -from crepes import ConformalRegressor - -# Wrap fitted model -cr = ConformalRegressor() -cr.fit(residuals_cal) # Just needs calibration residuals! - -# Predict intervals -y_pred, y_intervals = cr.predict(X_test, confidence=0.9, y_hat=model.predict(X_test)) -``` - -**When to use CREPES:** -- If you want Venn-ABERS for probability intervals -- If you need Mondrian CP without manual implementation -- If MAPIE doesn't support your use case - -**For your thesis:** Stick with MAPIE (industry standard, better docs, sklearn integration). - -### Nonconformist (Legacy) - -**Status:** Not actively maintained (last update 2020). Use MAPIE instead. - ---- - -## 9. Code Patterns for Your Notebooks - -### Notebook 04: Conformal Prediction (Implementation Template) - -```python -# Cell 1: Imports -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -from loguru import logger - -from src.models.conformal import ( - create_pd_intervals, - validate_coverage, - create_classification_sets -) -from src.evaluation.metrics import conformal_metrics -from src.utils.visualization import plot_conformal_intervals - -# Cell 2: Load data and model -train = pd.read_parquet('data/processed/train.parquet') -test = pd.read_parquet('data/processed/test.parquet') -calibration = pd.read_parquet('data/processed/calibration.parquet') - -from catboost import CatBoostClassifier - -model = CatBoostClassifier() -model.load_model('models/pd_canonical.cbm') -# Optional (recommended for current Mondrian pipeline flows): -# with open('models/pd_canonical_calibrator.pkl', 'rb') as f: -# calibrator = pickle.load(f) - -# Cell 3: Split features/target -X_cal = calibration.drop(columns=['default_flag']) -y_cal = calibration['default_flag'] -X_test = test.drop(columns=['default_flag']) -y_test = test['default_flag'] - -# Cell 4: Generate conformal intervals (alpha = 0.10 = 90% coverage) -y_pred, y_intervals_90 = create_pd_intervals( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=0.10 -) - -# Cell 5: Validate coverage -metrics_90 = validate_coverage(y_test, y_intervals_90, alpha=0.10) -print(f"Empirical Coverage: {metrics_90['empirical_coverage']:.4f}") -print(f"Target Coverage: {metrics_90['target_coverage']:.4f}") -print(f"Avg Interval Width: {metrics_90['avg_interval_width']:.4f}") - -# Cell 6: Try multiple alpha levels -alphas = [0.01, 0.05, 0.10, 0.20] -results = [] - -for alpha in alphas: - _, intervals = create_pd_intervals(model, X_cal, y_cal, X_test, alpha) - metrics = conformal_metrics(y_test, intervals, alpha) - metrics['alpha'] = alpha - results.append(metrics) - -results_df = pd.DataFrame(results) -print(results_df) - -# Cell 7: Efficiency analysis -widths = y_intervals_90[:, 1] - y_intervals_90[:, 0] -print(f"Width distribution:") -print(f" Min: {widths.min():.4f}") -print(f" 25th: {np.percentile(widths, 25):.4f}") -print(f" Median: {widths.median():.4f}") -print(f" 75th: {np.percentile(widths, 75):.4f}") -print(f" 95th: {np.percentile(widths, 95):.4f}") -print(f" Max: {widths.max():.4f}") - -# Cell 8: Coverage by risk grade -grades = X_test['grade'].unique() -coverage_by_grade = {} - -for grade in sorted(grades): - mask = (X_test['grade'] == grade) - low, high = y_intervals_90[mask, 0], y_intervals_90[mask, 1] - coverage = ((y_test[mask] >= low) & (y_test[mask] <= high)).mean() - width = (high - low).mean() - coverage_by_grade[grade] = { - 'n_samples': mask.sum(), - 'coverage': coverage, - 'avg_width': width - } - -coverage_df = pd.DataFrame(coverage_by_grade).T -print("\nCoverage by Grade:") -print(coverage_df) - -# Cell 9: Visualizations -fig1 = plot_interval_widths(y_intervals_90, alpha=0.10, - feature='grade', X_test=X_test) -plt.show() - -fig2 = plot_coverage_by_subgroup(y_test, y_intervals_90, X_test, - feature='grade', alpha=0.10) -plt.show() - -fig3 = plot_conformal_intervals_v2(y_test, y_pred, y_intervals_90, - n_samples=200, sort_by='prediction') -plt.show() - -# Cell 10: Classification sets (LAC vs APS vs RAPS) -y_pred_lac, y_sets_lac = create_classification_sets( - model, X_cal, y_cal, X_test, alpha=0.10, method='lac' -) -y_pred_aps, y_sets_aps = create_classification_sets( - model, X_cal, y_cal, X_test, alpha=0.10, method='aps' -) - -singleton_lac = (y_sets_lac.sum(axis=1).squeeze() == 1).mean() -singleton_aps = (y_sets_aps.sum(axis=1).squeeze() == 1).mean() - -print(f"\nClassification Sets:") -print(f" LAC singleton rate: {singleton_lac:.2%}") -print(f" APS singleton rate: {singleton_aps:.2%}") - -# Cell 11: Save intervals for optimization -output = pd.DataFrame({ - 'loan_id': test.index, - 'pd_point': y_pred, - 'pd_low': y_intervals_90[:, 0], - 'pd_high': y_intervals_90[:, 1], - 'pd_width': y_intervals_90[:, 1] - y_intervals_90[:, 0], - 'covered': (y_test >= y_intervals_90[:, 0]) & (y_test <= y_intervals_90[:, 1]) -}) -output.to_parquet('data/processed/conformal_intervals_mondrian.parquet') -logger.info(f"Saved {len(output)} conformal intervals") -``` - ---- - -## 10. Integration with Robust Optimization - -### Using Conformal Intervals as Uncertainty Sets - -Your `src/optimization/robust_opt.py` already implements this: - -```python -from src.optimization.robust_opt import build_box_uncertainty_set - -# Build uncertainty set from conformal intervals -uncertainty_set = build_box_uncertainty_set( - pd_low=y_intervals[:, 0], - pd_high=y_intervals[:, 1] -) - -# Use in robust optimization -from src.optimization.portfolio_model import build_portfolio_model - -model = build_portfolio_model( - loan_amounts=loan_amounts, - pd_estimates=uncertainty_set['pd_high'], # Worst-case - lgd_estimates=lgd_point, - budget=1_000_000, - risk_tolerance=0.05 -) -``` - -**Key insight:** The conformal intervals provide **distribution-free uncertainty sets** with coverage guarantees → robust optimization uses upper bounds for conservative allocations. - ---- - -## 11. Key Takeaways for Thesis - -### What Makes Your Work Novel - -1. **First application of MAPIE 1.3+ to credit risk PD modeling** (to our knowledge) -2. **Conformal-enhanced IFRS9:** Using interval widths for SICR detection -3. **Predict-then-optimize with conformal uncertainty:** Direct integration into Pyomo optimization -4. **Comparison with traditional methods:** Bootstrap vs Bayesian vs Conformal -5. **Efficiency analysis by risk segment:** Demonstrating practical applicability - -### Recommended Experiments for Notebook 04 - -1. **Coverage validation:** α ∈ {0.01, 0.05, 0.10, 0.20} -2. **Efficiency benchmarking:** Compare MAPIE interval widths with bootstrap (1000 samples) -3. **Conditional coverage:** By grade, by loan amount quartile, by year -4. **Calibration impact:** Uncalibrated vs Isotonic calibrated CatBoost on conformal intervals -5. **Mondrian CP:** Split by grade, show fairness improvement -6. **Time complexity:** Measure conformalize() time vs bootstrap resampling - -### Metrics to Report - -| Metric | Formula | Target | -|--------|---------|--------| -| Empirical Coverage | mean(y ∈ [ŷ_low, ŷ_high]) | ≈ 1 - α | -| Coverage Gap | \|empirical - target\| | < 0.02 | -| Avg Width | mean(ŷ_high - ŷ_low) | < 0.30 | -| Width IQR | P75(width) - P25(width) | Low = uniform | -| Singleton Rate | mean(set_size == 1) | > 0.80 | -| Min Group Coverage | min over grades | > 0.85 | - ---- - -## 12. Open Questions / Future Extensions - -1. **Adaptive Conformal Prediction:** Update intervals online as new loans default (sequential learning) -2. **Conformal Survival Analysis:** Extend to time-to-default prediction (via lifelines) -3. **Conformal CATE:** Uncertainty on causal effects (econML + MAPIE) -4. **Conformal Time Series:** EnbPI for forecasting with MAPIE (not implemented in 1.3 for regression) -5. **Deep Learning + Conformal:** Neural networks for PD + MAPIE wrappers - ---- - -## 13. References & Documentation - -### Primary Resources -- **MAPIE Documentation:** https://mapie.readthedocs.io/en/stable/ -- **MAPIE GitHub:** https://github.com/scikit-learn-contrib/MAPIE -- **CREPES:** https://github.com/henrikbostrom/crepes - -### Key Papers (As of Jan 2025 Knowledge) -1. **Split Conformal Prediction:** Vovk et al. (2005) - foundational theory -2. **CQR:** Romano et al. (2019) - Conformalized Quantile Regression -3. **LAC/APS:** Sadinle et al. (2019) - Classification sets -4. **RAPS:** Angelopoulos et al. (2020) - Regularized APS -5. **Predict-then-Optimize:** Elmachtoub & Grigas (2022) - SPO+ framework - -### Your Implementation Files (Cross-Reference) -- **Conformal logic:** `src/models/conformal.py` -- **Metrics:** `src/evaluation/metrics.py` (lines 59-84) -- **Robust optimization:** `src/optimization/robust_opt.py` -- **Calibration:** `src/models/calibration.py` -- **Visualization:** `src/utils/visualization.py` (needs update for MAPIE 1.3 shape) - ---- - -## Summary - -**Your current implementation is production-ready for MAPIE 1.3.0.** Key strengths: -- Correct API usage (SplitConformalRegressor with prefit=True) -- Proper ProbabilityRegressor wrapper -- Comprehensive metrics (coverage, width, efficiency) -- Integration with robust optimization - -**Next steps for Notebook 04:** -1. Run experiments with multiple alpha levels -2. Add conditional coverage by grade -3. Compare efficiency with bootstrap -4. Visualize intervals (use updated plotting functions) -5. Save intervals for downstream optimization (Notebook 08) - -**For thesis contribution:** -- Emphasize distribution-free guarantees vs Bayesian assumptions -- Show efficiency gains vs bootstrap (computational + statistical) -- Demonstrate fairness via Mondrian CP (if time permits) -- Integrate into full pipeline (NB09: end-to-end) - ---- - -**Generated for:** CRPTO -**Author:** Claude Code Analysis -**Date:** 2026-02-07 -**Version:** MAPIE 1.3.0 diff --git a/docs/research/foundations/crpto_decision_changes_and_learnings.md b/docs/research/foundations/crpto_decision_changes_and_learnings.md deleted file mode 100644 index 5c27d92..0000000 --- a/docs/research/foundations/crpto_decision_changes_and_learnings.md +++ /dev/null @@ -1,103 +0,0 @@ - - -# Decision Changes and Learnings Log -Version: 2026-03-31 - -This file stores project history: decision changes, mistakes, inconsistencies, and practical learnings. -Do not store this type of historical content in `CLAUDE.md` or `docs/PROJECT_JUSTIFICATION.md`. - ---- - -## 1) Decision Change Log - -| Date | Topic | Before | After | Why | Evidence | -|------|-------|--------|-------|-----|----------| -| 2026-02-20 | PD architecture comparison | `y_prob_cb_default` and `y_prob_cb_tuned` were effectively equal in export path | Default and tuned predictions are generated and stored independently | Avoid false "no improvement" conclusions | `scripts/train_pd_model.py`, `tests/test_models/test_pd_model.py` | -| 2026-02-20 | Feature contract source | Runtime relied on legacy static subset (11 features) | Runtime resolves feature sets from persisted feature config and persists contract | Recover predictive signal from full engineered feature set | `src/models/pd_model.py`, `configs/pd_model.yaml`, `models/pd_model_contract.json` | -| 2026-02-20 | Calibration policy | Fixed narrative said "Platt selected" | Temporal multi-metric selection policy between Platt/Isotonic | Better OOT probability quality and less narrative drift | `scripts/train_pd_model.py`, `data/processed/model_comparison.json` | -| 2026-02-20 | CatBoost tuning process | Tuning existed but was not consistently surfaced in artifacts | Long-run Optuna config enabled (400 trials, TPE multivariate, pruning, SQLite study) | Make HPO real, traceable, and reproducible | `configs/pd_model.yaml`, `src/models/pd_model.py`, `models/optuna_pd_catboost.db` | -| 2026-02-27 | Conformal promotion gate semantics | Promotion gate blocked by strict policy `overall_pass` including Kupiec/Christoffersen | Promotion gate now blocks on business checks (coverage/group/Winkler/critical alerts) and keeps statistical tests as diagnostics | Avoid false promotion blocks due to sample-size sensitivity while preserving strict policy traceability | `scripts/run_comparison.py`, `tests/test_scripts/test_run_comparison.py`, `docs/RUNBOOK.md` | -| 2026-03-16 | Paper-grade run: integral promotion deferred | `comparison.json.overall_pass=false` (two failing semantic gates) | Fixed: causal-only mismatch exemption + `paper_grade_closure_authoritative` flag; added `operational_overall_pass=true` as promotion gate | Root cause was design decisions (causal not regenerated = `insights_only`; strict conformal gate vs paper-grade closure), not model quality failures | `scripts/run_comparison.py`, `reports/run_comparisons/paper-grade-2026-03-13-final-heavy-2026-03-13-230650/comparison.json` | -| 2026-03-16 | Selective promotion of paper-grade components | No explicit promotion flags on portfolio policy, LGD conformal variant, fairlearn/skops | Portfolio policy (risk_tolerance=0.18), LGD `direct_adaptive_grade_temporal`, fairlearn+skops marked `promoted: true` in their artifacts | Full integral run NOT promoted (`overall_pass` strict=false); individual components with passing quality gates promoted selectively | `models/champion_portfolio_policy.json`, `models/conformal_lgd_ead_status.json`, `models/conformal_method_registry.json`, `models/champion_registry.json` | -| 2026-03-05 | Official baseline freeze + canonical artifacts | Core runs depended on manually passed baselines and migration dual-write artifacts (`*_v2`) | Baseline registry + freeze CLI + launcher default baseline resolution; canonical single-write for conformal/fairness/governance status | Remove ambiguity in reruns and reduce legacy artifact noise in operations | `configs/baselines/core_official_baseline.json`, `scripts/freeze_core_baseline.py`, `scripts/start_long_run.sh`, `scripts/validate_conformal_policy.py`, `scripts/run_fairness_audit.py`, `scripts/generate_governance_status.py` | -| 2026-03-16 | CRPTO absorbed into CRPTO | CRPTO (CP + Robust Opt) and CRPTO were separate concepts | CRPTO absorbs CRPTO + adds theoretical bound alpha-Gamma, SPO+ regret comparison, and uncertainty set baselines | Avoid overlap between papers; concentrate strongest contribution in flagship venue (MS/OR/EJOR) | `docs/backlog-papers-unified.md`, `book/chapters/14-crpto/*` | -| 2026-03-16 | Calibration config: method: platt → auto | 7 YAML configs hardcoded `method: platt` | All configs changed to `method: auto` with comment explaining runtime auto-selection | Config said platt but runtime selected Venn-Abers via temporal policy; misleading for readers | `configs/pd_model*.yaml` | -| 2026-03-16 | SPO+ integration surfaced | `src/optimization/spo_integration.py` existed but was not in any pipeline step | Created `scripts/run_spo_comparison.py` + added to out_of_scope_insights_factory research profile | Code existed since early development but was never executed; needed for CRPTO | `scripts/run_spo_comparison.py`, `scripts/run_out_of_scope_insights_factory.py` | -| 2026-06-09 | Dead-module cleanup pre-IJDS | `src/evaluation/{encoding_stability,monotonicity,slicing_functions}.py` and `src/optimization/{sda,spo_integration}.py` had zero imports anywhere in the repo | Removed the five modules; frozen audit JSONs (`models/encoding_stability_status.json`, `models/monotonicity_audit_status.json`) remain the evidence surface; six zero-reference one-shot scripts moved to `scripts/archive/` | Shrink the packaging surface for the IJDS reproducibility bundle without touching frozen artifacts | `CHANGELOG.md`, `scripts/archive/README.md`, `configs/pipeline_registry/script_role_registry.yaml` | -| 2026-06-10 | April-lineage unification of the canonical PD identity | Drift harness proved the frozen conformal intervals came from the April search candidate (AUC 0.7139), while `pd_canonical.cbm`/`test_predictions.parquet` were later retrains (AUC 0.7124 April, 0.7127 June) that never fed the certificate; the paper mixed both lineages | With explicit approval: canonical binaries replaced by byte-copies of the candidate bundle; `test_predictions` rebuilt from it with a hard equality assert against the frozen intervals; paper PD metrics moved to the certificate lineage (AUC 0.7139, Brier 0.1544, ECE 0.0070); A7/A8 kept frozen (degenerate LP re-solve), A5/A9/A10 re-frozen under the locked stack; manifest gains `april_lineage_unification` | One lineage, one story: every paper number now traces to the binaries that produced the certificate, and the chain is bit-exact reproducible (harness drift 0.0) | `docs/refactor/drift_report_mapie_2026-06.md`, `scripts/rebuild_test_predictions_from_frozen.py`, `EXTRACTION_MANIFEST.json`, `CHANGELOG.md` | -| 2026-06-13 | Feature config pickle retired | Live feature contract still kept a Python pickle in DVC/manifest | `feature_config.yml` + `feature_config.parquet` replace `feature_config.pkl`; drift-gate remains bit-exact | Remove pickle portability/security risk while preserving the champion feature contract | `src/features/feature_config_io.py`, `dvc.yaml`, `EXTRACTION_MANIFEST.json`, `docs/refactor/archive/FEATURE_CONFIG_PARQUET_PLAN.md` | -| 2026-03-17 | SPO+ v2: 5 architectural fixes | SPO+ v1 used flat MLP (500-dim input), binary costs, single seed; showed only ~2.5% improvement | v2: point-wise permutation-equivariant MLP (10-dim), calibrated PD costs (continuous), multi-seed (5 seeds), conformal robust as 3rd method, n_items=100 | Binary costs → flat landscape; flat MLP → vanishing gradients; single seed → high variance | `scripts/run_spo_real.py` (SCHEMA_VERSION 2026-03-17.2), `models/spo_real_training_status.json` | -| 2026-03-17 | SICR conformal trigger (paper temporal/IFRS9 fuera de alcance CRPTO) | IFRS9 SICR used only PD threshold (12m) | Width of conformal interval as additional SICR signal; optimal t*=0.30 via F1 grid; ECL alpha sensitivity quantified | Loans with high model uncertainty (wide intervals) are SICR candidates regardless of PD level; regulatory cost of confidence level choice now measurable | `scripts/run_sicr_conformal.py`, `data/processed/sicr_conformal_grid.parquet`, `data/processed/ecl_alpha_sensitivity.parquet` | -| 2026-03-23 | Beta calibration as 4th candidate | 3 calibration candidates: Platt, Isotonic, Venn-Abers | 4 candidates: Platt, Isotonic, Venn-Abers, Beta (betacal library) | Broader calibrator search space; Beta calibration is theoretically appropriate for probability outputs | `src/models/calibration.py::calibrate_beta()`, `scripts/train_pd_model.py` | -| 2026-03-26 | Feature pipeline closure (rerun V2) | Runtime still depended on historical FE artifacts not rebuilt canonically from splits | Canonical feature producer materializes `train_fe/calibration_fe/test_fe`, feature config, `woe_encoders.pkl`, and `feature_manifest_v2` as an explicit stage | Remove notebook/artifact drift and make reruns reconstructible from data stages | `scripts/materialize_feature_artifacts.py`, `src/features/feature_engineering.py`, `dvc.yaml` | -| 2026-03-26 | Causal lane positioning | Causal artifacts and docs still sounded like a promotable end-to-end lane | Causal becomes research-grade pricing intervention lane; official output stays `insights_only` unless overlap, sensitivity and policy gates all pass | Full-data rerun showed positive ATE/policy value but failed sensitivity, so causal remains valuable science without contaminating champion promotion | `scripts/estimate_causal_effects.py`, `models/causal_effect_status.json`, `models/causal_policy_rule.json`, `models/cate_portfolio_status.json` | -| 2026-03-26 | Challenger monotonic constraints | Monotonic constraints existed but their governance meaning was underspecified | Monotonic constraints kept as challenger-only audited spec, not injected into the canonical champion | Interpretability/governance gains were real, but predictive cost (`auc_drop`, `ece_delta`) and drift guardrails blocked promotion | `src/models/pd_model.py::resolve_monotonic_constraints`, `models/pd_challenger_spec.json`, `models/challenger_promotion_report.json` | -| 2026-03-26 | Long-run observability | Several long phases could still fail late or leave ambiguous status after reboot/OOM | Runtime statuses, checkpoints, last-valid-artifact markers and atomic writes expanded across PD, survival, LGD/EAD, causal, IFRS9 and optimization | Convert reruns from “best effort” into resumable audited execution | `src/utils/pipeline_runtime.py`, `scripts/run_long_pipeline.py`, `models/*_runtime_status.json` | -| 2026-03-26 | Bundle traceability after partial reruns | `champion_search_bundle.json` hid that some subsystems came from a later causal rerun | Bundle now exposes `artifact_run_tags` and `mixed_run_tags_present` | Avoid false impression that all artifacts come from a single homogeneous run tag | `scripts/build_champion_search_bundle.py`, `models/champion_search_bundle.json` | -| 2026-03-26 | Conformal policy recovery v3 | Canonical conformal artifact stayed blocked by `avg_width_90`, despite good coverage, because the selected tuning path was overly conservative and `shrinkback` cancelled useful group-floor adjustments | Canonical conformal now uses the grade-scaled configuration without shrinkback; `overall_pass=true`, `methodological_justification_pass=true`, and MRM returns to 4/4 PASS | Close the operational conformal gate without hiding statistical over-coverage in large OOT samples | `scripts/generate_conformal_intervals.py`, `scripts/backtest_conformal_coverage.py`, `scripts/validate_conformal_policy.py`, `models/conformal_policy_status.json` | -| 2026-03-23 | Notebooks 10-12 included in pipeline | `include_notebooks=False` in canonical and paper-grade profiles | `include_notebooks=True` in both profiles; NB10-12 executed with outputs | Paper notebooks should have outputs for Quarto book and reproducibility | `configs/run_profiles/canonical_operational.yaml`, `configs/run_profiles/paper_grade_final.yaml` | -| 2026-03-23 | Calibration monitoring enriched | Basic Brier/ECE metrics per calibrator | Log-loss per temporal fold, degradation rate, Murphy diagram, monthly calibration monitoring | Richer calibration diagnostics for paper-grade evidence and MRM defense | `scripts/train_pd_model.py`, `src/utils/visualization.py::plot_murphy_diagram()` | -| 2026-03-23 | Metadata run_tags fixed | MRM and pd_rare_event artifacts had `run_tag=None`/`untracked` | All metadata run_tags corrected to paper-grade run tag; `mrm_report_status.json` wrapper created | Metadata consistency for artifact traceability | `models/mrm_report_status.json`, `models/pd_rare_event_calibration_status.json` | -| 2026-03-23 | Conformal policy test fixed | Test expected strict `overall_pass` which fails on large OOT (276K) | Test now validates `methodological_justification_pass` logic correctly | Kupiec/Christoffersen are known to fail at high sample sizes; the methodological justification path is the correct paper-grade gate | `scripts/validate_conformal_policy.py`, tests | -| 2026-03-25 | Streamlit reduced to 5 local labs | Streamlit local retained 31 pages and broad narrative duplication | Streamlit local reduced physically to 5 optional labs; Quarto becomes explicit source of truth and public Streamlit is treated as historical showcase | Remove editorial duplication, keep only high-value interaction, and decouple local development from frozen showcase | `streamlit_app/pages/`, `book/chapters/E-streamlit-companion.qmd`, `docs/STREAMLIT_QUARTO_MIGRATION_REGISTRY.yml` | -| 2026-03-31 | Pipeline-first repository cleanup | Long-run orchestration, docs, reports, scripts, and scratch artifacts still mixed active and historical surfaces | Active vs history/research namespaces enforced; pre-Quarto helpers archived; notebook taxonomy updated; scratch runtime checkpoints purged conservatively | Reduce noise, stop accidental dependence on stale artifacts, and make sync/rebuild behavior easier to defend | `docs/DOCUMENTATION_MAP.md`, `docs/ARTIFACT_RETENTION_POLICY.md`, `scripts/history/`, `reports/history/`, `configs/pipeline_registry/*` | - ---- - -## 2) Errors and Inconsistencies Found - -| Date | Issue | Impact | Resolution | Evidence | -|------|-------|--------|------------|----------| -| 2026-02-20 | Peer-reviewed Lending Club link pointed to unrelated COVID paper (`PMC9533764`) | External benchmark credibility risk | Corrected to Lending Club paper `PMC9222552` and updated benchmark artifact | `reports/history/hpo_research_notes_2026-02-19.md`, `scripts/benchmark_kaggle_lendingclub.py` | -| 2026-02-20 | Hardcoded Streamlit claims for old calibration/method snapshots | UI could present stale or incorrect conclusions | Replaced with dynamic/neutral text tied to artifacts | `streamlit_app/pages/*.py`, `reports/history/consistency_audit_2026-02-20.md` | -| 2026-02-20 | Confusion between Optuna trial validation AUC and final OOT calibrated AUC | Misinterpretation of model selection quality | Documented split clearly: trial score is validation; final score is calibrated OOT | `models/pd_training_record.pkl`, `data/processed/model_comparison.json` | - ---- - -## 3) Practical Learnings - -1. Temporal validation is mandatory for Lending Club; random split benchmarks often overstate performance. -2. OOT calibrated AUC can be lower than best validation AUC without indicating a bug. -3. Calibration quality (Brier/ECE) can improve materially even when AUC changes little. -4. Narrative drift is a recurring risk; docs and UI must read artifact outputs, not fixed metric strings. -5. Feature-contract governance is as important as model hyperparameters in credit-risk pipelines. -6. Kupiec/Christoffersen tests are sample-size sensitive and will reject at n=276K even with coverage close to nominal; methodological justification with Winkler compensation is the correct defense. -7. Test skips should be removed once the underlying issue is fixed; carrying skips obscures the true test count. - ---- - -## 4) Open Follow-Ups - -All items resolved: - -1. ~~Align `configs/pd_model.yaml` calibration wording with runtime policy~~ — **RESOLVED 2026-03-16**: all 7 pd_model YAML configs changed from `method: platt` to `method: auto`. -2. ~~Continue pruning historical snapshots with retention policy~~ — **RESOLVED 2026-03-18**: historical docs already carry HISTORICAL/DEPRECATED banners. `DOCUMENTATION_MAP.md` created. -3. ~~Add an automated "reference integrity" check for external URLs~~ — **DEFERRED 2026-03-18**: low ROI for thesis context; external links are manually audited in `docs/research/foundations/crpto_references_state_of_art.md`. - ---- - -## 5) Related Audit Reports - -- `reports/history/consistency_audit_2026-02-20.md` -- `reports/history/hpo_research_notes_2026-02-19.md` -- `reports/history/before_after_recompute_comparison_longrun.json` - ---- - -## 6) Session History (Consolidated) - -This section replaces the need for a separate `SESSION_HISTORY.md`. - -| Date | Session | What was executed | Outcome | Evidence | -|------|---------|-------------------|---------|----------| -| 2026-02-17 | Post-reboot recovery | Quality gates (`ruff`, `pytest`), DVC local/cloud status, DVC push smoke, DAG verification | Environment recovered and synchronized; integrity checks green | `SESSION_STATE.md` (section "Post-Reboot Recovery Log") | -| 2026-02-18 | Repro-contract closure | `dvc repro` for pipeline/export stages, DVC push, MLflow suite backfill | Reproducibility contract restored; artifacts and tracking refreshed | `SESSION_STATE.md` (section "Repro-Contract Closure Log") | -| 2026-02-18 | Validity hardening phases 0-5 | Leakage hardening, optimization fixes, dynamic narrative updates, CP/OR benchmark updates, temporal causal backtest | `pytest` green and DVC status consistent after rerun | `SESSION_STATE.md` (section "Validity Hardening Log"), `reports/history/PHASES_0_5_EXECUTION_2026-02-18.md` | -| 2026-02-19 | HPO research + long-run setup | CatBoost/Optuna best practices review; HPO policy strengthened (multivariate TPE + pruning + persistent study) | Long-run search configured for 400 trials with reproducible tracking | `reports/history/hpo_research_notes_2026-02-19.md`, `configs/pd_model.yaml` | -| 2026-02-20 | Consistency and reference audit | External link verification, narrative consistency cleanup, artifact/process cross-check | Incorrect literature link fixed; stale claims reduced; open items documented | `reports/history/consistency_audit_2026-02-20.md` | -| 2026-02-27 | Conformal rerun hardening v1 | Soft promotion diagnostics for statistical tests, governance stage added before MRM, optional-failure exit code fix, incremental HPO budget | Rerun orchestration aligned with production promotion policy and resumable core profile | `scripts/run_comparison.py`, `scripts/run_long_pipeline.py`, `configs/run_profiles/overnight_full.yaml`, `configs/pd_model.yaml` | -| 2026-03-13 | Paper-grade final run | Run `paper-grade-2026-03-13-final-heavy` completed: AUC=0.7128, conformal 90%≥0.92, fairness 6/6 PASS, governance 6/6 PASS, Venn-Abers selected (ECE=0.0061), HPO trial 151 seed-invariant | All individual module gates PASS; full integral promotion blocked by `comparison.json` semantic gate (fixed 2026-03-16). TS intervals remain research_only. | `models/paper_grade_protocol_status.json`, `reports/run_comparisons/paper-grade-2026-03-13-final-heavy-2026-03-13-230650/comparison.json` | -| 2026-03-16 | P0/P1 fixes and selective promotions | Fixed semantic gates in `run_comparison.py` (P0.1), skops render bug (P0.3), `_row_number` added to conformal intervals (P0.4); promoted portfolio policy, LGD conformal, fairlearn+skops | `operational_overall_pass=true`, `card_render_status=rendered`, parquet has `_row_number`. 3 components promoted with explicit flags. | `models/champion_portfolio_policy.json`, `models/conformal_lgd_ead_status.json`, `models/conformal_method_registry.json` | -| 2026-03-17 | SPO+ v2 + paper temporal/IFRS9 fuera de alcance CRPTO IFRS9 analysis | SPO+ v2 (5 fixes): point-wise MLP, calibrated PD costs, multi-seed, conformal robust, n_items=100. SICR conformal: t* grid + ECL alpha sensitivity. | 49.1% regret reduction (Wilcoxon p=0.0000); t*=0.30, $56.6M ECL_add; alpha sensitivity +22% ECL from 90→99% conf. 665 tests passing. | `scripts/run_spo_real.py`, `scripts/run_sicr_conformal.py`, `models/spo_real_training_status.json`, `models/sicr_conformal_status.json` | -| 2026-03-21 | MAPIE deep-dive Sprint 1-4 | HSIC/SSC/MWI/CWC diagnostics, VennAbersCalibrator migrated to MAPIE native, classification set benchmark (LAC/APS/RAPS), Quarto book enriched with MAPIE diagnostics. | 674 tests passing (1 skipped). Quarto book 94/94 files render. | `scripts/backtest_conformal_coverage.py`, `scripts/run_classification_set_benchmark.py`, `book/` | -| 2026-03-23 | Calibration gap analysis + doc sync | Beta calibration added as 4th candidate, log-loss/degradation rate/Murphy diagram added, NB10-12 executed, CRPTO test skip removed, metadata run_tags fixed, conformal policy test fixed. | 690 tests passing, 0 failures, 0 skips. All metadata gaps resolved. | `scripts/train_pd_model.py`, `configs/run_profiles/`, `models/mrm_report_status.json`, `models/pd_rare_event_calibration_status.json` | diff --git a/docs/research/foundations/crpto_documentation_map.md b/docs/research/foundations/crpto_documentation_map.md deleted file mode 100644 index ae2ae96..0000000 --- a/docs/research/foundations/crpto_documentation_map.md +++ /dev/null @@ -1,77 +0,0 @@ - - -# Documentation Map - -Quick reference for the current documentation stack after the pipeline-first refactor, the monotonic promotion work, and the ADSFCR-inspired documentation refresh. - -## Keep Closest To Hand - -| Category | File | Purpose | -|---|---|---| -| **Canonical editorial ledger** | `docs/CANONICAL_DOCUMENTATION_AND_QUARTO_TRACEABILITY_2026-03-30.md` | Master map between live techniques, artifacts, Quarto chapters, references, and legacy claims to retire | -| **Current state** | `SESSION_STATE.md` | Operational snapshot and runtime-facing source list | -| **MRM / governance** | `docs/MODEL_RISK_MANAGEMENT.md` | SR 11-7 style governance narrative and control framing | -| **ADSFCR adoption** | `docs/ADSFCR_AUDIT_AND_MONOTONIC_CHALLENGER_PLAN_2026-03-29.md` | Detailed audit of the external repo, adoption decisions, and tranche-by-tranche implementation status | -| **ADSFCR next work** | `docs/ADSFCR_EXECUTABLE_BACKLOG_2026-03-30.md` | Execution-oriented backlog for the remaining ADSFCR items that still look worth implementing | -| **Quarto contract** | `docs/QUARTO_BOOK_BLUEPRINT.md` | Book architecture, editorial contract, and maintenance rules | -| **Project rationale** | `docs/PROJECT_JUSTIFICATION.md` | Methodological and architectural why | -| **Runbook** | `docs/RUNBOOK.md` | Reproducibility playbook | -| **Search wave 2026-04** | `docs/PIPELINE_FIRST_TOPOLOGY_2026-03-31.md` | Pipeline-first taxonomy including the new `search_out_of_scope_temporal_ifrs9_ifrs9` lane for exhaustive April runs | -| **TS vNext decision** | `docs/TIME_SERIES_VNEXT_DECISION_2026-04-02.md` | Current keep / research / do-not-promote decision for the time-series redesign lane | -| **History / learnings** | `docs/DECISION_CHANGES_AND_LEARNINGS.md` | Historical decisions, fixes, and practical learnings | -| **Paper references** | `docs/research/foundations/crpto_references_state_of_art.md` | Curated literature map for papers and thesis chapters | -| **Backlog** | `docs/backlog-papers-unified.md` | Unified backlog for papers, experiments, and documentation follow-ups | -| **Conformal note** | `docs/conformal_prediction_README.md` | Compact operational/research entrypoint for conformal material | - -## Runtime Sources Of Truth - -Use these before trusting any prose: - -- `models/champion_registry.json` -- `data/processed/pipeline_summary.json` -- `models/fairness_audit_status.json` -- `models/threshold_semantics.json` -- `models/governance_status.json` -- `models/model_shift_status.json` -- `models/monotonicity_audit_status.json` -- `models/pd_backtesting_status.json` -- `models/bootstrap_validation_status.json` -- `models/pd_validation_interpretation_status.json` -- `models/calibration_mapping_status.json` -- `models/ifrs9_diagnostics_status.json` -- `models/encoding_stability_status.json` -- `reports/mrm/mrm_validation_report.json` -- `reports/run_comparisons/canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129/comparison.json` - -## Directory Contract - -- `docs/` root contains active technical and editorial surfaces only. -- `docs/history/` contains archived plans, audits, and historical snapshots retained for provenance. -- `docs/research/` contains literature notes, exploratory comparisons, and research-only reference material that is not part of the live operational contract. - -## Historical but Still Useful - -| File | Why it remains | -|---|---| -| `docs/history/OFFICIAL_RERUN_MASTER_PLAN_2026-02-27.md` | Provenance of the earlier paper-grade rerun program | -| `docs/history/PROMOTION_DOSSIER_2026-03-01.md` | Historical promotion snapshot; not live policy state | -| `docs/history/ENGINEERING_ENVS_AND_UPGRADE_PLAN_2026-02-25.md` | Environment migration notes if tooling breaks | -| `docs/history/DEPLOY_STREAMLIT_FREE.md` | Historical showcase deployment only | - -## Research-Only References - -| File | Why it remains | -|---|---| -| `docs/research/conformal_prediction_research_2026.md` | Deep conformal theory and implementation notes | -| `docs/research/conformal_prediction_quick_reference.md` | Coding patterns and formula crib sheet | -| `docs/research/conformal_libraries_comparison.md` | Library comparison retained for justification and appendix work | -| `docs/research/CALIBRATION_METHOD_SELECTION.md` | Calibration writeup preserved as research/method note | -| `docs/research/portfolio_selector_literature_2020_2026.md` | Literature support for portfolio selection writing | - -## Editorial Rule - -If Quarto, docs, Streamlit, and runtime artifacts disagree: - -1. Trust runtime artifacts first. -2. Trust the canonical traceability doc second. -3. Treat older markdown snapshots as historical unless they explicitly say they are live. diff --git a/docs/research/foundations/crpto_metrics_triage_2026-05-18.md b/docs/research/foundations/crpto_metrics_triage_2026-05-18.md deleted file mode 100644 index adc0f24..0000000 --- a/docs/research/foundations/crpto_metrics_triage_2026-05-18.md +++ /dev/null @@ -1,112 +0,0 @@ -# Metrics for Credit and ML Models - triage 2026-05-18 - -> Ported from the CRPTO research archive -> (`metrics_credit_ml_models_triage_2026-05-18`). Closes the intake of the -> practitioner binder "Metrics for Credit and ML Models.pdf" (five pieces on -> discrimination, calibration, equity and trust). Decides what enters CRPTO, -> what enters agenda extendida CRPTO/tesis (agenda), and what stays parked. - -## Source status - -| Source | Status | Verified source | Project use | -| --- | --- | --- | --- | -| Wuthrich, "Model Selection with Gini Indices under Auto-Calibration" | peer-reviewed journal article, European Actuarial Journal 2023 | https://doi.org/10.1007/s13385-022-00339-9 | CRPTO + agenda extendida CRPTO/tesis | -| Albanesi and Vamossy, "Credit Scores: Performance and Equity" | NBER Working Paper 32917, 2024 | https://doi.org/10.3386/w32917 | agenda extendida CRPTO/tesis strong context; CRPTO light context | -| Dinga et al., "Beyond accuracy" | bioRxiv preprint, 2019 | https://doi.org/10.1101/743138 | metric taxonomy context only | -| Somers' D / ordered association material | binder material | binder PDF | optional metric appendix | -| ReScorer | binder material, GenAI/e-commerce reason scoring | binder PDF | park | - -## Prudential (Paper 2) absorption check - -Paper 2 no longer needs to be a near-term standalone target. agenda extendida CRPTO/tesis is the live -lab and future full-paper container, so it absorbs all useful prudential -evidence: ECL scenarios, conformal ECL ranges, SICR by conformal width, -CIF/prepayment, stage-cost governance and TS-to-ECL context. Absorption is -complete for planning, but bounded in claim language: agenda extendida CRPTO/tesis keeps the material -as an `IFRS9-inspired SICR/ECL proxy` lane, not a contractual IFRS 9 -implementation. - -| Prudential evidence | agenda extendida CRPTO/tesis destination | Decision | -| --- | --- | --- | -| ECL by scenario and conformal range | IFRS9 evidence card | append | -| SICR conformal threshold `t*=0.30`, recall 75.8%, ECL +56.6M | IFRS9 evidence card | append strong | -| CIF vs Kaplan-Meier prepayment correction | IFRS9 evidence card | append | -| Stage misclassification cost | governance/threshold appendix | append if staging is discussed | -| TS to ECL intervals | stress context | context-only | -| Near-term standalone Paper 2 | no longer required | park/supersede | -| Contractual IFRS9 claim | no destination | keep false | - -## What changes for CRPTO - -The binder does not add a new contribution to CRPTO. It strengthens two phrases -that were already true: - -1. **Wuthrich** supports the statement that AUC/Gini are not sufficient for model - selection unless the score is calibration-gated. This fits the narrative: - CRPTO is not an AUC leaderboard; it uses calibrated PD plus conformal - uncertainty plus a robust decision. -2. **Albanesi and Vamossy** support the credit-scoring/equity motivation: better - ML scores can reduce misclassification of traditional score bands and change - borrower standing. This is context, not legal fair-lending evidence. - -Do not add Dinga, Somers' D, or ReScorer to the CRPTO bibliography unless a -future appendix explicitly needs them. - -## What changes for agenda extendida CRPTO/tesis (agenda) - -| Source | Use in agenda extendida CRPTO/tesis | Decision | -| --- | --- | --- | -| Wuthrich | Calibration-gated Gini/AUC: ranking is interpreted only after calibration | append methodological | -| Albanesi and Vamossy | FICO/score proxy vs champion ML, misclassification, ranking difference, observable vulnerable groups | append strong | -| Dinga et al. | Metric taxonomy: discrimination, calibration, utility and equity | context | -| Somers' D | Bootstrap/tie sensitivity for Gini/AUC/Somers' D | optional | -| ReScorer | Only if we audit LLM reasons in the research workflow | park | - -## Executed bounded experiment for agenda extendida CRPTO/tesis - -The compact experiment was executed on 2026-05-19 through the consolidated agenda extendida CRPTO/tesis -frontier runner, without creating a new versioned wave. - -**Question.** Does the project champion materially improve ranking and -misclassification relative to an origin-time FICO proxy, especially for -observable vulnerable or low-data-quality groups? - -**Data.** `fico_range_low`/`fico_range_high`/`fico_score`; `default_flag` or -project default target; champion calibrated PD; `annual_inc`, `dti`, -`home_ownership`, `mort_acc`, `total_acc`, `earliest_cr_line`, `addr_state`, -`zip3`, `grade`, `sub_grade`, `purpose`. - -**Protocol.** Build a FICO proxy (midpoint of FICO range, risk bands by decile); -compare FICO proxy vs champion on AUC/Gini/Somers' D; calibrate the FICO proxy -with one simple monotone mapping before comparing Brier/ECE (Wuthrich's gate -makes raw rank metrics insufficient); compute ranking difference -`champion_percentile - fico_percentile`; measure misclassification by risk band; -slice by observable groups (income quintile, thin-file proxy, no-mortgage, -delinquency proxy, state/zip3 support cells, grade); close with one decision. - -**Result.** On the latest 40% of the OOT window (`n=103,865`), the calibrated -champion beats the origin-time FICO proxy on the intended metric stack: - -| metric | champion | FICO proxy | delta champion - FICO | -| --- | ---: | ---: | ---: | -| AUC | 0.700477 | 0.592906 | 0.107570 | -| Gini/Somers' D | 0.400953 | 0.185813 | 0.215140 | -| Brier | 0.140380 | 0.151195 | -0.010815 | -| ECE 10-bin | 0.028995 | 0.051132 | -0.022137 | -| decile band MAE | 0.030545 | 0.051076 | -0.020532 | - -`48.6526%` of loans move at least 20 percentile points between the FICO proxy -ranking and the champion ranking. The result survives the intended observable -slices (income tails, high DTI, no-mortgage, thin-file proxy, grades A-E) as an -appendix diagnostic. - -**Decision.** Append to agenda extendida CRPTO/tesis as metric-governance evidence. CRPTO can use -this only as light context if needed. No fair-lending legal claim is allowed. - -## Editorial decision - -- CRPTO gets a surgical calibration/equity-context reinforcement. -- agenda extendida CRPTO/tesis gets an IFRS9 evidence card and a future metric-governance experiment - design. -- No new `extended_agenda_v###`, no new exploratory CSV wave, and no - commit-per-iteration artifact should be created from this intake. diff --git a/docs/research/foundations/crpto_portfolio_selector_literature_2020_2026.md b/docs/research/foundations/crpto_portfolio_selector_literature_2020_2026.md deleted file mode 100644 index 3bfeeb0..0000000 --- a/docs/research/foundations/crpto_portfolio_selector_literature_2020_2026.md +++ /dev/null @@ -1,241 +0,0 @@ - - -> **RESEARCH NOTE** — Revisión de literatura retenida para writing y anexos. No es parte del runbook ni del contrato canónico. - -# Portfolio Selector Literature 2020-2026 - -## Purpose - -This note tracks recent literature that is directly relevant to the project's -portfolio policy selection problem: - -- PD point estimates are available, but portfolio decisions depend on whether - the uncertainty-adjusted policy is economically viable. -- Conformal prediction provides loan-level uncertainty (`pd_high - pd_point`). -- The key practical problem is not computing a robust frontier, but selecting a - portfolio policy that preserves return while reacting to uncertainty in a - locally meaningful way. - -The literature below supports three design decisions already implemented in the -codebase: - -1. separate frontier generation from canonical policy selection, -2. localize uncertainty treatment instead of applying global worst-case - penalties, -3. evaluate candidate policies using downstream economic impact on the actual - decision universe. - -## Recent papers that matter most - -### 1. Localized Conformal Prediction - -- Leying Guan, "Localized Conformal Prediction: A Generalized Inference - Framework for Conformal Prediction", arXiv:2106.08460, 2021 - Link: https://arxiv.org/abs/2106.08460 - -Why it matters: -- The paper argues that conformal uncertainty should adapt to the local region - of the test point rather than remain globally uniform. -- This directly motivates policy families that only penalize uncertainty where - it is locally high instead of globally pushing every loan toward `pd_high`. - -Project implication: -- Supports `tail_blended_uncertainty` and - `segment_tail_blended_uncertainty`. -- Suggests the right direction is localized or segment-aware uncertainty, not - uniform worst-case robustification. - -### 2. Conformal Contextual Robust Optimization - -- Yash Patel, Sahana Rayan, Ambuj Tewari, "Conformal Contextual Robust - Optimization", arXiv:2310.10003, 2023 - Link: https://arxiv.org/abs/2310.10003 - -Why it matters: -- The paper studies predict-then-optimize with context-dependent conformal - uncertainty regions. -- The key lesson is that contextual uncertainty sets can be less conservative - than global uncertainty sets while retaining distribution-free guarantees. - -Project implication: -- Supports policy families that use loan context (`grade`, `term`, potentially - `verification_status`) to determine which part of the uncertainty should - matter for optimization. -- Supports the architectural change that policy selection must be downstream of - uncertainty, not a fixed ex ante ranking detached from the actual decision - universe. - -### 3. Conformal Robustness Control - -- "Conformal Robustness Control: Uncertainty Sets for Robust Optimization from - Contextual Data", arXiv preprint, 2026 - Search entry used during design: - https://arxiv.org/search/?query=Conformal+Robustness+Control&searchtype=all - -Why it matters: -- The framing is almost identical to our problem: use conformalized uncertainty - to define uncertainty sets for robust optimization, but control the amount of - robustness. - -Project implication: -- Supports explicit policy families where robustness is a tunable design choice - (`gamma`, caps, tails, contextual segments), not a binary switch. -- Supports maintaining a canonical fallback to non-robust decisions when no - robust policy survives the economic constraints. - -### 4. Optimal Model Selection for Conformalized Robust Optimization - -- "Optimal Model Selection for Conformalized Robust Optimization", arXiv - preprint, 2025 - Search entry used during design: - https://arxiv.org/search/?query=Optimal+Model+Selection+for+Conformalized+Robust+Optimization&searchtype=all - -Why it matters: -- The core message is that predictive models should be selected by downstream - optimization quality, not only by predictive metrics. - -Project implication: -- Supports the shift already made in the project: - - `tradeoff` produces candidates, - - `select_economic_portfolio_policy.py` chooses the canonical policy on the - actual A/B universe, - - `simulate_ab_test.py` becomes audit-only in the canonical path. -- This is directly aligned with our `economic_actual_ab_v1` selector. - -### 5. Group-Weighted Conformal Prediction - -- Aabesh Bhattacharyya, Rina Foygel Barber, "Group-Weighted Conformal - Prediction", arXiv:2401.17452, 2024 - Link: https://arxiv.org/abs/2401.17452 - -Why it matters: -- Weighted conformal methods are useful when exchangeability or sampling - assumptions are not fully homogeneous across groups. -- Group-weighted conformal gives a practical way to improve uncertainty quality - under group-driven shift. - -Project implication: -- Suggests future work where conformal intervals are weighted or localized by - group (`grade`, `term`, `verification_status`) before feeding robust - optimization. -- This is especially relevant because the portfolio selector is already moving - toward group/segment-aware robustification. - -### 6. Conformal Predictive Systems under Covariate Shift - -- Jef Jonkers, Glenn Van Wallendael, Luc Duchateau, Sofie Van Hoecke, - "Conformal Predictive Systems Under Covariate Shift", arXiv:2404.15018, 2024 - Link: https://arxiv.org/abs/2404.15018 - -Why it matters: -- This work extends conformal predictive systems to covariate shift using - weighting. -- While the Lending Club dataset is frozen, this still matters for our - train/cal/test mismatch and for the interpretation of uncertainty in specific - subpopulations. - -Project implication: -- Supports future work where conformal widths are not treated as equally - reliable across the entire test population. -- Motivates weighting or local calibration instead of purely global conformal - widths in the policy selector. - -## Related domain papers - -### 7. Probability of default for lifetime credit loss for IFRS 9 using machine learning competing risks survival analysis models - -- Expert Systems with Applications, 2024 - Link: https://www.sciencedirect.com/science/article/pii/S095741742400472X - -Why it matters: -- Shows that lifetime PD modeling for IFRS 9 benefits from survival-type - approaches rather than static one-period classification alone. - -Project implication: -- Reinforces the current project architecture where PD, survival, IFRS9 and - uncertainty should be connected rather than treated as isolated modules. -- Suggests that future robust policies could potentially use lifetime-tail risk - signals instead of only one-period conformal width. - -### 8. Approaches for modelling the term-structure of default risk under IFRS 9: A tutorial using discrete-time survival analysis - -- arXiv:2507.15441, 2025 / journal tutorial in 2026 - Link: https://arxiv.org/abs/2507.15441 - -Why it matters: -- Reviews term-structure modeling for lifetime PD under IFRS 9. - -Project implication: -- Supports keeping the canonical pipeline's survival/lifetime PD path and using - uncertainty-aware portfolio policies as a separate decision layer rather than - collapsing everything into a single static PD score. - -### 9. Stabilising Lifetime PD Models under Forecast Uncertainty - -- arXiv:2509.10586, 2025 - Link: https://arxiv.org/abs/2509.10586 - -Why it matters: -- Explicitly studies uncertainty in lifetime PD forecasting and stabilization. - -Project implication: -- Supports the broader project thesis that uncertainty should be propagated - into decision-making layers, not merely reported in dashboards. - -## What the literature suggests for this project - -### A. The current global robust policies are too coarse - -Empirical result in the project: -- robust policies with `gamma > 0` often destroy return in A/B on the real - decision universe. - -Literature-backed interpretation: -- global worst-case or globally blended conformal penalties are too - conservative for portfolio selection. -- localized/contextual uncertainty is the correct direction. - -Implemented project response: -- `tail_blended_uncertainty` -- `segment_tail_blended_uncertainty` - -### B. Canonical policy selection must be decision-focused - -Empirical result in the project: -- selecting a champion directly from the frontier and auditing afterwards was - structurally wrong. - -Literature-backed interpretation: -- downstream decision quality should drive model/policy choice. - -Implemented project response: -- `select_economic_portfolio_policy.py` -- `economic_actual_ab_v1` -- `simulate_ab_test.py` in canonical mode as audit-only - -### C. If no robust policy survives, the system should say so explicitly - -Empirical result in the project: -- several runs end in `fallback_nonrobust`. - -Literature-backed interpretation: -- that is not a failure of robust optimization; it is a valid finding that the - uncertainty treatment is not yet economically viable. - -Implemented project response: -- canonical policies now record `selection_outcome` -- non-robust fallback is explicit rather than disguised as a robust champion - -## Next research directions supported by the literature - -1. Weighted or group-weighted conformal intervals for the portfolio selector - path. -2. Segment-aware robustification beyond `grade x term`, e.g. - `grade x term x verification_status`. -3. Decision-focused uncertainty features such as: - - absolute conformal width, - - relative width, - - local width percentile within segment, - - width combined with expected return floor. -4. Offline research grids, but fixed canonical policy in the heavy main once - the dataset is considered static and the champion is frozen. diff --git a/docs/research/foundations/crpto_state_of_art_review_2026-05-18.md b/docs/research/foundations/crpto_state_of_art_review_2026-05-18.md deleted file mode 100644 index b056e51..0000000 --- a/docs/research/foundations/crpto_state_of_art_review_2026-05-18.md +++ /dev/null @@ -1,236 +0,0 @@ -# CRPTO State-of-the-Art Review - 2026-05-18 - -> Ported from the CRPTO research archive -> (`crpto_state_of_art_review_2026-05-18`). Consolidates a deeper -> literature pass for the official CRPTO manuscript and the long-horizon agenda extendida CRPTO/tesis -> living lab agenda. It is not a new experimental loop and not a Quarto rewrite. -> The output is a source map and a claim-governance recommendation. - -## Executive Decision - -CRPTO should stay centered on the economic champion and the conformal-robust -portfolio contribution. The best literature additions are positioning and -reviewer defense, not new claims: - -- Use conformal risk control, Learn-then-Test, conditional-coverage limits and - conformal robust optimization to explain why the method controls decision risk - without pretending to solve exact conditional validity. -- Use robust optimization and predict-then-optimize references to position CRPTO - as prescriptive analytics with calibrated uncertainty, not merely prediction. -- Use credit scoring, fintech lending and model-risk references to explain why - the Lending Club application is practically relevant and why auditability - matters. -- Mention CVaR/OCE only as a tail-risk challenger appendix/sensitivity; do not - reopen the champion. - -agenda extendida CRPTO/tesis remains a governed living lab (agenda). Its strongest contribution is -not a near-term publication claim; it is a documented boundary map for seven -ambitious extensions. Four lanes are append-worthy as evidence appendices, while -three remain parked. The prudential IFRS9-inspired material enters agenda extendida CRPTO/tesis as -proxy evidence (ECL scenarios, conformal ECL ranges, SICR width signal, -CIF/prepayment correction, stage-cost governance, TS-to-ECL stress context), -never as contractual IFRS 9. - -| lane | decision | literature role | claim boundary | -| --- | --- | --- | --- | -| IFRS9/SICR | append | IFRS 9, ECL, survival and competing-risk sources justify a proxy diagnostic | not contractual IFRS 9 without monthly DPD, contractual terms and macro scenarios | -| CVaR/OCE | append | CVaR and OCE sources justify tail-risk challenger analysis | not champion replacement without paired wealth gains | -| fair-lending proxy | append | BISG, proxy-fairness and ML underwriting governance sources justify proxy-risk governance | not legal fair-lending evidence without surname, protected attributes or tract-level geography | -| DLA/ADP | append | SDAM/ADP sources justify sequential rollout framing | not exact Bellman optimality without decision logs and state transition panels | -| online conformal | park | ACI and multi-source conformal explain what would be needed | no live feedback or external source distribution | -| CATE/policy value | park | causal and double-ML sources support a sensitivity appendix only | no causal policy claim without rejected applicants or an instrument | -| SPO/DFL | park | SPO+, PyEPO and cvxpylayers explain the prototype direction | no main-pipeline integration while benefit is only toy/oracle-regret evidence | - -## CRPTO: Best Literature Fit - -### 1. Conformal Decision Guarantees - -The official manuscript already has a solid conformal backbone. The highest -value additions are not more conformal breadth, but clearer hierarchy: - -- Foundation: Vovk et al. and Romano et al. establish distribution-free - conformal prediction and conformalized quantile regression. -- Risk control: Bates et al., Angelopoulos et al. on Conformal Risk Control, and - Learn-then-Test support the paper's risk-control language. -- Boundary: Barber, Candes, Ramdas and Tibshirani on limits of distribution-free - conditional predictive inference should be used to prevent overclaiming source - or subgroup validity. -- Prescriptive bridge: Johnstone and Cox, Patel et al., and Sun et al. - strengthen the conformal robust optimization / robust contextual LP framing. - -Recommended use: one compact paragraph in the theory/literature section, plus a -reviewer-facing note that CRPTO controls an operational risk target and does not -claim exact conditional coverage for every borrower/source subgroup. - -### 2. Robust Optimization and Prescriptive Analytics - -The robust optimization literature should position CRPTO as a decision pipeline, -not as a forecasting-only paper: - -- Bertsimas and Sim is the core robust optimization reference for uncertainty - budgets and the price of robustness. -- Conformal uncertainty sets for robust optimization and conformal contextual - robust optimization are the closest methodological neighbors. -- Predict-then-Calibrate is especially useful because it frames robust contextual - LP through calibration rather than pure point prediction. -- SPO+ and DFL should be cited only as adjacent decision-focused learning. CRPTO - does not need to become a differentiable optimization paper. - -Recommended use: add a "closest methods" contrast: CRPTO is closer to calibrated -robust prescriptive analytics than to end-to-end SPO/DFL. - -### 3. Credit Scoring, Fintech Lending and Model Risk - -The credit literature should support empirical relevance and governance: - -- Lessmann et al. provide a widely cited credit-scoring benchmark and connect - predictive accuracy with business value. -- Jagtiani and Lemieux are directly relevant because they study Lending Club and - fintech lending with alternative data and loan grades. -- SR 11-7 and FinRegLab support model-risk governance, auditability, - explainability and fair-lending sensitivity language. - -Recommended use: strengthen the introduction and empirical setting. This is the -cleanest way to make the Lending Club application feel less like a dataset -exercise and more like a regulated credit decision problem. - -### 4. Tail-Risk Challenger, Not Champion Reopening - -CVaR and OCE references are valuable for reviewer defense if a reader asks why -the official champion is not tail-risk optimized: - -- Rockafellar and Uryasev support the CVaR optimization formulation. -- Ben-Tal and Teboulle support OCE as a convex risk-measure family. -- The agenda extendida CRPTO/tesis CVaR/OCE experiment is useful as a challenger appendix because it - improved tail framing but did not beat paired wealth. - -Recommended use: keep this as appendix/sensitivity. Do not reopen the economic -champion unless future paired replay beats it robustly. - -## agenda extendida CRPTO/tesis (agenda): Best Literature Fit By Lane - -### IFRS9/SICR - -The literature supports a proxy diagnostic, not a full IFRS 9 implementation. -IFRS Foundation and Basel/EBA materials establish the accounting and supervision -context. Recent survival, competing-risk and term-structure ECL work supports -the idea that default timing and lifetime PD matter. Keep as an appendix: the -project has useful cashflow, hardship and recovery fields, but lacks monthly -contractual days-past-due history, original effective interest rate accounting -infrastructure and macro scenario paths. The honest title is "IFRS9-inspired -SICR/ECL proxy diagnostic." - -### Online Conformal / Source Holdouts - -ACI, multi-source conformal and multi-distribution conformal references justify -the direction of the lane. The conditional-coverage impossibility literature is -also important: source-aware coverage is not free. Keep the existing source -holdout evidence as retrospective governance only. Do not claim online/adaptive -validity until there is live feedback, a production-like stream, or a genuinely -external source distribution. - -### CVaR/OCE - -Append-worthy. CVaR/OCE literature gives a clean mathematical language for tail -utility and risk aversion, and the existing experiment is useful as a -stress/challenger result. The blocker is not theory; it is empirical dominance. -The challenger did not replace the economic champion. Destination: appendix tail -challenger and caveat for CRPTO. - -### CATE / Policy Value - -The causal literature is useful mainly because it tells us to stop. DoWhy, -double/debiased ML and causal-forest references require explicit identification, -overlap and refutation. Lending Club accepted-loan data can support an -observational sensitivity screen, but not a strong policy-value claim because -rejected applicants, randomized pricing or a credible instrument are absent. -Destination: parked with a causal-identification memo, not a promoted -experiment. - -### Fair-Lending Proxy - -Valuable for claim boundaries. CFPB BISG and Zhang's proxy-method literature -show why surname plus fine geography matter. The project has state and zip3, not -surname or protected attributes. Destination: appendix source/proxy-governance -risk. Legal fair-lending claims remain false. A good example of a valuable -negative result. - -### DLA/ADP - -Powell's SDAM/ADP framing fits the sequential decision lab, but the data blocks -exact dynamic programming. Snapshot fields can support rollout-style diagnostics -and state summaries, not Bellman optimality. Destination: appendix -rollout/sequential analytics. Good framing, weak optimality claim. - -### SPO/DFL - -SPO+, PyEPO and cvxpylayers establish a legitimate frontier, but the current -project only has isolated toy/oracle-regret evidence. Integrating this would -increase dependency and maintenance risk without improving the official -champion. Destination: parked prototype. Reopen only on a specific reviewer -request or a compact optimization benchmark that directly dominates a CRPTO -comparator. See [crpto_pyepo_dfl_intake_2026-05-26](../crpto_pyepo_dfl_intake_2026-05-26.md) -for the PyEPO 1.3.7 update. - -## Candidate Additions To Bibliography (small, text-tied) - -Highest priority candidates not yet fully integrated into the official narrative: - -- Barber et al. on limits of distribution-free conditional predictive inference. -- Jagtiani and Lemieux on Lending Club fintech lending. -- FinRegLab's ML credit underwriting policy/empirical reports. -- Rockafellar and Uryasev on CVaR if the tail-risk appendix is cited. -- Ben-Tal and Teboulle on OCE if OCE remains in the appendix text. -- CFPB BISG and Zhang if a fair-lending/proxy governance appendix is kept. -- Agrawal et al. and Tang/Khalil only if the SPO/DFL prototype is referenced. - -## Anti-Loop Reopen Gates - -Reopen a parked or append lane only if one of these happens: - -- A monthly servicing panel becomes available (contractual DPD, payment states, - macro scenario paths). -- Rejected-applicant data or randomized pricing/instrumental variation appears. -- Surname, tract-level geography or protected-attribute proxy inputs appear with - an approved governance plan. -- A reviewer explicitly asks for a lane and the response fits one compact - table/memo. -- A future run can change the official champion under the paired wealth gate. - -Otherwise, the correct next move is not more experiments; it is citation -integration and manuscript extraction. - -## Bibliography Integration Patch (2026-05-18) - -Adds only sources that now appear in the manuscript; destinations expressed by -manuscript role (standalone CRPTO chapter labels differ from the old `14x` scheme): - -| source | manuscript role | reason | -| --- | --- | --- | -| `barber2021_limits_conditional` | theory / claim frontier | Bound source/subgroup claims and avoid overstating conditional validity. | -| `jagtiani2019` | introduction / empirical setting | Establish Lending Club as a real fintech-lending empirical setting. | -| `rockafellar2000_cvar` | tail-risk appendix | Ground CVaR as a canonical tail-risk diagnostic. | -| `bental_teboulle2007_oce` | tail-risk appendix | Ground OCE as convex risk-measure framing. | -| `finreglab2023_ml_credit` | governance / fairness proxy | Support ML credit underwriting governance, explainability and fairness context. | -| `cfpb_bisg_proxy` | fairness proxy boundary | Document why surname plus fine geography are needed for BISG-style proxy analysis. | -| `zhang2018_fair_proxy` | fairness proxy boundary | Support the fair-lending proxy boundary and why current fields are insufficient for legal claims. | - -No IFRS9, SPO/DFL prototype or agenda extendida CRPTO/tesis-only sources were integrated into CRPTO -in this patch. Those remain agenda extendida CRPTO/tesis (agenda) material unless a future appendix or -reviewer request creates a concrete textual need. - -## Metrics Binder Addendum - -The metrics binder was triaged on 2026-05-18 and logged in -[crpto_metrics_triage_2026-05-18](crpto_metrics_triage_2026-05-18.md). The only -source that should enter CRPTO as methodological support is Wuthrich's -Gini/autocalibration result: it reinforces the claim that CRPTO is not an AUC -leaderboard, and that rank metrics are meaningful only after the PD layer is -calibration-gated. Albanesi and Vamossy can support the credit scoring/equity -motivation as context only; it does not authorize a legal fair-lending claim. - -For agenda extendida CRPTO/tesis, the binder opens one bounded future appendix: FICO/score proxy vs -champion ML, with misclassification, ranking difference and observable-group -diagnostics. Dinga et al. is taxonomy-only, Somers' D is optional metric -sensitivity, and ReScorer stays parked unless the project later audits -LLM-generated research reasons. diff --git a/docs/research/future_work/crpto_classification_conformal_memo_2026-04-03.md b/docs/research/future_work/crpto_classification_conformal_memo_2026-04-03.md deleted file mode 100644 index 06ed387..0000000 --- a/docs/research/future_work/crpto_classification_conformal_memo_2026-04-03.md +++ /dev/null @@ -1,326 +0,0 @@ - - -# Chapter 3 Memo: Classification Conformal Ideas To Port Into This Repo - -Date: 2026-04-03 - -Source reviewed during the original extraction: a local copy of -`Applied_Conformal_Prediction (2).pdf`, Chapter 3, especially Sections 3.2-3.5 -(pp. 63-87 in the PDF pagination shown by the text extract). The standalone -CRPTO repo does not depend on the original local filesystem path. - -## Bottom line - -For este paquete CRPTO, Chapter 3 does not point to a radical replacement of the current PD interval stack. It points to three concrete search expansions: - -1. Search over calibrated probability spaces, not only over Mondrian partition and residual scaling. -2. Add explicit binary classification-set score families to the research sidecar search, especially hinge/LAC vs margin. -3. Treat Venn-Abers as a first-class conformal search dimension, not just as an upstream calibration artifact. - -The highest-value idea for binary credit-risk PD is still: - -- probability-first decisions, -- Venn-Abers or similarly strong calibration first, -- conformal as a guardrail around that calibrated score, -- Mondrian/group-aware calibration where subgroup reliability matters. - -## Repo observations that matter - -Current repo behavior is narrower than the chapter's design space: - -- `scripts/generate_conformal_intervals.py` searches only over `alpha`, partition, `scaled_scores`, and `min_group_size`. -- `scripts/benchmark_conformal_variants.py` benchmarks interval variants, but not nonconformity-score families for classification sets. -- `scripts/benchmark_pd_set_prediction.py` benchmarks only one MAPIE classification-set method (`lac` by default). -- `src/models/conformal.py` already contains: - - interval construction, - - binary classification sets via MAPIE, - - Mondrian partition builders, - - a Venn-Abers interval helper. -- Score-band Mondrian partitions are currently built from raw `predict_proba` outputs, not calibrated probabilities. -- The binary set benchmark currently does not consume the chosen probability calibrator, even though Chapter 3 repeatedly recommends "calibrate probabilities first, then conformalize." - -That last point is the biggest implementation mismatch with Chapter 3. - -## 1. Best ideas to test in our conformal search - -### A. Calibrator-aware conformal search - -Why: - -- Chapter 3 makes calibration upstream of conformal efficiency, especially for hinge, margin, APS/RAPS, and conditional-coverage behavior. -- Better calibrated probabilities should improve set efficiency and reduce subgroup coverage gaps even when marginal coverage is already valid. - -What to test: - -- `venn_abers` vs `isotonic` vs `platt` vs `beta` as the probability source feeding conformal. -- For interval search, compare the same Mondrian config under different calibrators. -- For binary set search, compare singleton rate, ambiguity rate, and grade-level coverage under each calibrator. - -### B. Partition score space as a search dimension - -Why: - -- The repo's score-decile and grade x score-band Mondrian partitions currently use raw model scores. -- Chapter 3 recommends calibrated probabilities as the working object for tabular classification. - -What to test: - -- partition on raw probability, -- partition on calibrated probability, -- partition on logit(raw probability), -- partition on logit(calibrated probability). - -Expected value: - -- likely better subgroup alignment than raw-score bins, -- likely better match between partitioning and the actual score used downstream for uncertainty. - -### C. Binary set nonconformity search: hinge/LAC vs margin - -Why: - -- Chapter 3's cleanest practical comparison for classification is hinge vs margin. -- For binary credit decisions, this is directly useful for abstention/triage around the approval threshold. - -What to test: - -- current `lac` baseline, -- custom hinge implementation, -- custom margin implementation, -- optionally calibrated-threshold versions around the economic decision threshold. - -Metrics: - -- set coverage, -- singleton rate, -- ambiguity rate, -- empty-set rate, -- default rate among ambiguous cases, -- group coverage by `grade`, -- temporal stability. - -### D. Mondrian/fair coverage for classification sets, not only intervals - -Why: - -- Chapter 3 explicitly calls out conditional, group-conditional, and fair coverage as the right extension for lending-like settings. -- The repo already applies Mondrian thinking to intervals, but not to binary set search. - -What to test: - -- global binary sets, -- `grade`-Mondrian binary sets, -- `score_decile_mondrian` binary sets, -- `grade_x_scoreband_mondrian` binary sets, -- optional fairness slices if minimum support is satisfied. - -### E. Venn-Abers width as an uncertainty side signal - -Why: - -- Chapter 3's Venn-Abers section makes the interval `(p0, p1)` itself an uncertainty object, not just a way to get a midpoint probability. -- In este paquete CRPTO, Venn-Abers width may be a useful side feature for triage, deferral, or conformal partitioning. - -What to test: - -- compare Venn width against conformal set ambiguity, -- use Venn width deciles as a candidate Mondrian partition, -- use Venn width as a routing feature for human-review or policy override sidecars. - -## 2. Highest-value ideas for binary credit-risk PD intervals/sets - -### Highest value - -#### 1. Calibrated hinge-style binary sets with Mondrian partitions - -This is the strongest near-term fit. - -Why: - -- Binary credit risk is probability-first, not label-list-first. -- Hinge has direct threshold semantics on calibrated `P(default)`. -- The repo already works with calibrated PDs and Mondrian partitions. -- It aligns with the repo's existing abstention/triage sidecar much better than multiclass APS/RAPS-style machinery. - -Expected outcome: - -- cleaner singleton/ambiguity behavior, -- easier explanation to model risk and governance stakeholders, -- better grade-level reliability than a global binary set benchmark. - -#### 2. Venn-Abers as the outer calibration choice for conformal - -This is high value because the repo already supports it and the book's logic is explicit: better calibration improves conformal usefulness. - -Expected outcome: - -- better efficiency for the same coverage, -- better subgroup stability, -- better probability semantics for cost-sensitive lending decisions. - -### Medium value - -#### 3. Binary margin sets for abstention near the policy boundary - -Useful if the business question is not only "is coverage valid?" but "where should we defer?" - -Why: - -- Margin focuses on closeness to the competing class. -- In binary PD, that means it concentrates ambiguity near the decision boundary. - -Best role: - -- research sidecar, -- approval-review deferral policy, -- analyst escalation. - -Less likely role: - -- primary replacement for the current interval stack. - -### Lower value for the current binary repo - -#### Brier/proper-scoring-rule nonconformity as a separate binary search family - -Low priority because in binary classification the Brier NCM collapses to a monotone transform of hinge: - -- for class 1, `s_brier = (1 - p)^2`, -- for class 0, `s_brier = p^2`. - -That means it is unlikely to produce meaningfully different conformal sets from hinge after quantile calibration. - -#### Rank-only top-k - -Very low value in binary: - -- either top-1 is enough, or -- the method collapses to `{0,1}` very often. - -This is too coarse for PD work. - -#### APS / RAPS / SAPS as a main binary focus - -Low value right now. - -Reason: - -- their main advantage is richer instance-adaptive set sizing in `K > 2` problems. -- in binary PD they mostly reduce to threshold shifts over singleton vs `{0,1}` behavior. - -These become much more interesting only if the repo moves to: - -- multiclass delinquency state prediction, -- multi-stage credit status prediction, -- richer action labels beyond default/non-default. - -#### Exotic model-specific scores - -Examples like embedding-distance NCMs are not a natural next move for CatBoost tabular PD. - -If a model-specific score is tested here, the best candidates are simpler: - -- raw-logit space, -- calibrated-logit space, -- Venn-width-conditioned partitions, -- score-conditioned Mondrian routing. - -## 3. Code and search-space changes implied - -### `src/models/conformal.py` - -Add explicit binary classification-set builders that do not rely only on MAPIE's built-in score names. - -Suggested additions: - -- `compute_binary_nonconformity_scores(...)` -- `create_binary_classification_sets_custom(...)` -- `create_binary_classification_sets_mondrian(...)` - -Searchable score families: - -- `hinge` -- `margin` -- `brier` as a documented sanity-check / expected-near-duplicate of hinge -- `rank_topk` as a negative-control baseline - -Also add an option to apply the repo calibrator before set construction. - -### `scripts/benchmark_pd_set_prediction.py` - -Expand from one-method benchmarking to a small search runner. - -Suggested new CLI dimensions: - -- `--score_families lac,hinge,margin` -- `--partition_candidates global,grade,score_decile_mondrian,grade_x_scoreband_mondrian` -- `--partition_score_space raw_prob,calibrated_prob,raw_logit,calibrated_logit` -- `--calibrator_override_path ...` - -Suggested new outputs: - -- by-variant benchmark table, -- by-group conditional coverage table, -- ambiguity concentration near decision-threshold slices, -- fairness slice table when configured. - -### `scripts/generate_conformal_intervals.py` - -Keep the current interval pipeline, but extend the search space with score-space awareness. - -Concrete additions: - -- `partition_score_space` -- optionally `residual_score_space` (`prob`, `logit`) -- optionally `calibrator_family` or `calibrator_tag` - -Important correction: - -- if the final interval is built around calibrated probabilities, the score-band partition should be allowed to use those same calibrated probabilities. - -### `scripts/benchmark_conformal_variants.py` - -Add calibrated global baseline and score-space diagnostics. - -Right now the "global split" reference is not using the calibrator path, while the Mondrian interval variants do. - -That should become: - -- `global_split_raw` -- `global_split_calibrated` - -and the benchmark should record the score space used for partitioning. - -### `configs/profiles/search_conformal_exhaustive.yaml` - -Extend the conformal search space with: - -- `partition_score_spaces` -- `classification_set_score_families` -- `classification_set_partition_candidates` -- `calibrator_candidates_for_conformal` - -This is the main config change needed to make the Chapter 3 ideas searchable rather than ad hoc. - -## Recommended execution order - -1. Fix the score-space mismatch: - - allow calibrated probabilities to drive score-band Mondrian partitions. -2. Add calibrator-aware binary set benchmarking: - - hinge/LAC vs margin. -3. Add Mondrian partitions to the binary set benchmark. -4. Only then test lower-priority families: - - Brier, - - rank-only, - - APS/RAPS/SAPS in binary. - -## Recommendation - -If the goal is to improve the repo's real conformal search rather than just widen the method catalog, the best next experiment is: - -- Venn-Abers-calibrated binary hinge sets, -- benchmarked globally and with `grade` / `score_decile_mondrian` / `grade_x_scoreband_mondrian`, -- with partitioning based on calibrated probability rather than raw score, -- compared against a binary margin sidecar. - -That is the highest signal-to-effort move suggested by Chapter 3 for este paquete CRPTO's current binary credit-risk setting. diff --git a/docs/research/future_work/crpto_temporal_uncertainty_vnext_2026-04-02.md b/docs/research/future_work/crpto_temporal_uncertainty_vnext_2026-04-02.md deleted file mode 100644 index d1382c8..0000000 --- a/docs/research/future_work/crpto_temporal_uncertainty_vnext_2026-04-02.md +++ /dev/null @@ -1,134 +0,0 @@ - - -# Time Series vNext Decision - -Date: 2026-04-02 - -## Purpose - -This document closes the `time_series_vnext` redesign lane as an executed research program. It records what was implemented, what was tested, what should remain in the project, and what should not be promoted yet. - -Artifacts of record: - -- `models/time_series_vnext_status.json` -- `models/time_series_policy_review.json` -- `data/processed/time_series_vnext.parquet` -- `data/processed/time_series_panel_vnext.parquet` -- `data/processed/ts_backtest_metrics_vnext.parquet` -- `data/processed/ts_interval_eval_vnext.parquet` -- `data/processed/ts_joint_path_eval_vnext.parquet` -- `data/processed/ts_ifrs9_scenarios_vnext.parquet` -- `data/processed/ts_ecl_intervals_vnext.parquet` -- `data/processed/time_series_policy_review_matrix.parquet` - -## What Was Implemented - -- Enriched internal-only monthly vintage contract in the legacy data-product - builder later retired during the R0 cleanup; current CRPTO keeps this as - research provenance, not as an active pipeline dependency. -- Parallel targets: - - `raw_rate` - - `logit_rate` -- Point benchmarking in `src/models/time_series_vnext.py` -- Marginal interval benchmarking with governed comparison against adaptive candidates -- Joint uncertainty via: - - `gaussian_copula` - - `schaake_shuffle` -- TS -> IFRS9 / ECL translation in `scripts/run_time_series_vnext.py` -- Policy review artifact and recommendation matrix - -## Executed Outcome - -Selected target: -- `raw_rate` - -Point layer: -- champion: `AutoARIMA` -- promotable: `true` -- interpretation: operationally usable, but not materially better than the canonical point lane - -Interval layer: -- best backtest challenger: `MAPIE_ENBPI` -- promotable: `false` -- key reason: improved the 90% coverage gap versus the canonical interval layer, but still failed the governed promotion threshold - -Operational forward interval generation: -- backtest winner `MAPIE_ENBPI` is not a forward-producing operational model in the current implementation -- forward interval artifact therefore falls back to `AutoARIMA` - -Joint uncertainty: -- both `gaussian_copula` and `schaake_shuffle` produced usable path artifacts -- they are informative for accumulated-risk and ECL-width analysis -- they are not yet promoted to official operational policy - -TS -> ECL: -- vNext ECL translation was numerically stable -- this makes it worth keeping as research support -- it is not yet a canonical downstream dependency - -Overall recommendation: -- `maintain_canonical_keep_vnext_research` - -## Keep / Research / Do Not Promote - -### Keep - -- Canonical point forecast lane -- Canonical IFRS9 temporal overlay -- Canonical publication of diagnostic intervals -- vNext TS -> ECL artifact as research support - -### Keep As Research Only - -- Enriched internal-only time-series contract -- `MAPIE_ENBPI` / adaptive interval challengers -- `gaussian_copula` sample paths -- `schaake_shuffle` sample paths -- `time_series_vnext` orchestration lane and outputs - -### Do Not Promote Yet - -- Replacing canonical `time_series` inputs with the enriched vNext contract -- Replacing canonical interval semantics with adaptive intervals -- Treating sample-path uncertainty as an official IFRS9 policy layer -- Rewiring canonical `run_ifrs9_sensitivity.py` to consume vNext artifacts by default - -## Why The Canonical Lane Stays - -- The redesign did not produce a material point-forecast improvement. -- The interval challenger improved diagnostics but still did not pass the governed promotion threshold. -- The strongest new value came from prudential interpretation, not from a clean operational replacement. - -That means the right decision is not to discard the work, but to contain it as a reproducible research lane. - -## Relation To Papers And Mega Extension - -- **paper temporal/IFRS9 fuera de alcance CRPTO**: the time-series lane supports IFRS9 scenario sensitivity and - TS->ECL interpretation. The point forecast can remain as overlay input, while - interval outputs should be described as analytical support rather than an - official monthly provisioning policy. -- **CRPTO**: the current CRPTO champion is one-shot/funded-set and does - not consume time-series intervals. TS becomes relevant only for a future - multi-period or sequential CRPTO extension. -- **paper Mondrian complementario fuera de alcance CRPTO / Mondrian**: the failed promotion of temporal intervals motivates - online/adaptive conformal coverage as future work, not as current evidence. -- **Mega extension IFRS9 + CATE + CRPTO**: vNext sample paths and TS->ECL - artifacts are natural inputs to a future state variable `S_t` with macro - forecasts, stage mix and ECL by policy, but they require new selector - artifacts before becoming central to a champion. - -## Practical Repo Policy - -- `scripts/forecast_default_rates.py` remains the canonical producer. -- `scripts/run_time_series_vnext.py` remains a namespaced research producer. -- `models/time_series_status.json` remains the only canonical status contract for operational consumption. -- `models/time_series_vnext_status.json` and `models/time_series_policy_review.json` remain research governance artifacts. - -## Next Promotion Condition - -Promotion should be reconsidered only if a future rerun demonstrates all of the following: - -- no material deterioration in point forecasting -- interval coverage clears the governed threshold -- forward interval generation is operationally coherent, not only backtest-strong -- the added complexity is justified by downstream IFRS9 or governance value diff --git a/docs/research/future_work/crpto_validation_hardening_notes_2026-03-30.md b/docs/research/future_work/crpto_validation_hardening_notes_2026-03-30.md deleted file mode 100644 index 8ee75dc..0000000 --- a/docs/research/future_work/crpto_validation_hardening_notes_2026-03-30.md +++ /dev/null @@ -1,25 +0,0 @@ - - -# CRPTO Validation Hardening Notes - -This note keeps only the CRPTO-relevant lessons from the ADSFCR-inspired audit and executable backlog. It is support material for governance, appendix writing and reviewer responses, not a new dependency of the CRPTO champion. - -## What Matters For CRPTO - -- Bootstrap diagnostics strengthen interpretation of PD, conformal coverage and MRM decisions when asymptotic tests become over-sensitive at large samples. -- Calibration mapping diagnostics tested a plausible post-hoc repair path for cohort persistence. The executed shadow validation was negative but valuable: lightweight intercept/isotonic sidecars did not improve the champion enough to justify promotion. -- Model-shift and p-value semantics are useful for explaining why a statistically visible diagnostic alert is not automatically a failed economic policy. -- Monotonicity and constrained-threshold challenger ideas remain useful future-work for governance, but they do not replace the official CRPTO champion. -- Encoding/binning stability is relevant as a feature-contract appendix because CRPTO depends on stable PD, calibrated probabilities and conformal residual behavior. - -## What Stays Out Of Scope - -- LGD survival, IFRS9 macro/ECL modeling and scenario temporal overlays are not CRPTO base-lane artifacts. -- Causal pricing, CATE portfolio surfaces and broad research-lab adoptions remain future work unless a later paper deliberately reopens them. -- External ADSFCR notebooks/PDFs are literature/support context only; CRPTO should cite methods directly rather than depend on that repository. - -## Carry-Forward Checklist - -- Keep CRPTO's evidence focused on: PD calibration, conformal coverage/width, bound-aware portfolio selection, exact alpha-gamma validation, funded-set composition, fairness and MRM. -- Mention negative calibration-mapping results only if a reviewer asks why the residual cohort persistence was not fixed through a simple remap. -- Use bootstrap and model-shift diagnostics as governance support, not as a new acceptance gate for the already frozen champion. diff --git a/docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md b/docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md new file mode 100644 index 0000000..f35f382 --- /dev/null +++ b/docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md @@ -0,0 +1,62 @@ +# IJDS Allocation-Granularity Sensitivity Protocol V3 - 2026-07-16 + +## Numerical Recovery + +V2 completed the outcome-free freeze but its evaluator stopped before writing +contrasts because the fixed-capital validator used an absolute `1e-8` check on +USD 15 million aggregated tracks. Existing solver budget residuals are accepted +at the monthly protocol tolerance but can exceed that unscaled aggregate check +by microdollars. V3 changes only this guard to `isclose` with relative tolerance +`1e-10` and absolute tolerance `1e-8`; it changes no exposure, transform, +endpoint, estimand, or reporting grid. V2 is stopped provenance, not evidence. + +## Lineage Note + +V1 was locked but not executed because the clean repository HEAD advanced to +record the separately stopped and recovered fitting-label protocol. V2 changes +only the run/tag identity needed for a clean tagged execution. The USD 25 rule, +inputs, 96-track census, estimands, and stop rules are unchanged from V1. + +## Question + +Is the continuous dollar-allocation relaxation materially different from a +simple operational allocation expressed in USD 25 lots? + +## Locked Design + +- Use the complete active baseline structure: USD 1 million monthly budget, + 25% purpose cap, LGD 0.45, both rulers, all three coordinates, both gamma + endpoints, eight windows, and all fifteen primary OOT months. +- For every positive exposure, apply + `25 * floor(exposure / 25)` and retain the residual as zero-risk, zero-payoff + cash. Never round upward and never reoptimize or refill the portfolio. +- Preserve all loan identities, scores, conformal endpoints, contractual rates, + and the active six-month evaluation endpoint. +- Validate that the transform cannot increase invested capital, score use, or + purpose concentration and that every retained exposure is a USD 25 multiple. +- Report all 1,440 monthly portfolios and all 96 window-by-candidate tracks. + +## Estimands + +For each track, report the residual-cash share and sharp +rounded-minus-continuous bounds for status-indexed payoff-proxy rate, weighted +default, and weighted miscoverage. Both policies use the same committed-capital +denominator, so residual cash remains explicit rather than disappearing through +renormalization. + +## Interpretation Boundary + +This is a deterministic granularity diagnostic, not an integer-programming +challenger, optimized discrete policy, or implementation claim. A small +perturbation supports the numerical adequacy of the continuous relaxation only +for this archive and declared USD 25 floor rule. It cannot select a ruler, +coordinate, gamma endpoint, or portfolio. + +## Stop Rules + +- Stop on tag, commit, lock, parent hash, portfolio census, period grid, or + outcome-ID mismatch. +- Stop if rounding increases exposure or violates a risk or purpose constraint. +- Stop if any of the 96 contrast tracks or any bound is absent or nonfinite. +- Report every track regardless of direction. +- Do not execute or overwrite protected historical stages or artifacts. diff --git a/docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md b/docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md new file mode 100644 index 0000000..55a01b7 --- /dev/null +++ b/docs/research/ijds_binary_geometry_frontier_v4_protocol_2026-07-12.md @@ -0,0 +1,195 @@ +# Binary-Geometry and Comparator-Frontier V4 Protocol + +## Status + +This protocol defines a new isolated retrospective audit over a previously +inspected archive. It does not alter or overwrite fixed-taxonomy V1--V3, the +historical champion, or `EXTRACTION_MANIFEST.json`. It cannot become +confirmatory, prospective, or preregistered. Locking limits additional analyst +degrees of freedom and makes every reported specification auditable. + +Required protocol tag: +`protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1`. + +Executable config: +`configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml`. + +Before the required tag was created, implementation smoke tests reloaded the +previously inspected archive, fitted both declared learners, checked all group +sizes and residual-window recipes, timed one monthly basis frontier, and +reconciled one guardrail/C2 cell. These checks motivated only computational +reuse of a HiGHS basis and shared point-frontier storage. They did not change +the eight windows, learner roles, nine policies, comparator definitions, +simulation grid, outcomes, hypotheses, or stop rules. V4 therefore remains a +retrospective specification audit, not a preregistration. + +## Research Question + +When an absolute-residual conformal construction for a binary outcome is used +as a score inside a constrained optimizer, how do binary prediction-set +geometry, temporal transport, and score-cap comparator choice jointly affect +the conclusions that can be drawn? + +The estimand is the finite Lending Club archive declared below. No result is a +causal effect, selected-set guarantee, investor return, live policy result, or +superpopulation statement. + +## Locked Information Contract + +- Raw source, 36-month term, March 31, 2016 information cutoff, September 30, + 2020 administrative snapshot, endpoint taxonomy, features, CatBoost + specification, 2007--2010 development block, 2011 Platt block, and 2011 + score-taxonomy source are inherited from V3. +- Candidate membership is determined by issue date and contractual term only. +- Fully Paid is zero, Charged Off is one, and exact Default plus every + nonterminal snapshot state are unresolved. +- Label availability uses `last_pymnt_d`; a Charged Off label becomes available + six calendar months later. Each included residual month must retain at least + 99% of terminal labels by the information cutoff. +- Outcomes are absent from prediction, policy, comparator, and frontier + construction. Every allocation is persisted and hashed before one validated + outcome join. + +## Complete Residual-Window Specification + +The primary temporal specification contains every consecutive six-month +window that starts in 2012 and ends no later than January 2013: + +1. January--June 2012; +2. February--July 2012; +3. March--August 2012; +4. April--September 2012; +5. May--October 2012; +6. June--November 2012; +7. July--December 2012; and +8. August 2012--January 2013. + +All eight windows are co-primary for coverage transport. No window is selected, +weighted, promoted, or removed using fit or OOT coverage. Outcome-free policy +development is February--December 2013 for every window, ensuring that the +same menus support every comparator definition. + +The canonical taxonomy has five score strata fixed from all status-independent +2011 Platt scores. One, two, and ten strata are closed coverage diagnostics. +The exact split rank is `ceil((n_g + 1) * (1 - alpha))` at `alpha=0.10`. + +## Binary Geometry + +For every window, taxonomy, role, and score stratum, the run must report: + +- continuous interval coverage and width mean/quantiles; +- discrete intersection with `{0,1}` as empty, `{0}`, `{1}`, or `{0,1}`; +- lower-endpoint positivity and upper-endpoint saturation; +- group prevalence, residual quantile, and score range; and +- below/above-fit score-range counts. + +The implemented object is a clipped residual interval for the observed binary +outcome. It is not a latent-PD interval and not the convex hull of its discrete +intersection. + +The paper may state a binary phase-transition proposition only for the exact +conditions proved in code and supplement. Empirical strata with varying scores +are an application diagnostic, not proof of the constant-score proposition. + +## Learner Diagnostic + +The CatBoost/Platt stack is primary. A numeric-feature logistic regression, +followed by an independent 2011 Platt map and its own fixed 2011 taxonomy, is a +coverage-only negative control. It does not select the primary learner and does +not enter portfolio optimization. Both learners must report every residual +window even if their conclusions differ. + +## Closed Portfolio Family + +All nine combinations of `tau in {0.15,0.17,0.19}` and +`gamma in {0.25,0.50,0.75}` are co-primary. The canonical empirical portfolio +audit uses seed 42, a 25% purpose cap, LGD 0.45, fifteen monthly USD 1 million +menus from April 2016 through June 2017, and every residual window. No winner +or aggregate vote across policies or windows is permitted. + +Early/late seed and purpose-cap results from V2/V3 remain historical +sensitivities. V4 does not add another policy HPO or seed search. + +## Comparator Identification + +All comparators use the same menu, full budget, per-loan bounds, purpose cap, +and plug-in objective `(1-p)r-p*LGD`. + +- C0 uses the point score and copies the guardrail's numeric cap. +- C1 uses, for each window and policy, the capital-weighted mean of the eleven + outcome-free monthly point-score moments generated by that guardrail on the + common 2013 development menus. +- C2 uses, for each OOT policy-month pair, the point-score moment of the already + frozen guardrail allocation. Its numerical residual must not exceed `1e-10`. +- The development-admissible comparator set for a window-policy pair is the + closed interval from the minimum to the maximum of its eleven monthly + development point-score moments. No rounding or outcome enters this set. +- A broad stress interval of `[0.05,0.12]` is secondary only. + +The point-score LP is represented with a binding budget equality, so its risk +cap enters as a right-hand side. HiGHS basis-ranging endpoints define the exact +piecewise-linear cap frontier. Every development-support endpoint, C0/C1/C2 +cap, and basis breakpoint inside the declared interval must be evaluated. +Fixed-grid interpolation is not allowed to support an exact-frontier claim. +If a development-admissible endpoint lies outside the secondary broad stress +interval, basis enumeration expands mechanically to the closed hull of the +two declared supports; it may not truncate the development support. + +For any guardrail allocation `x_q`, C2 sets +`tau_C2 = p' x_q / B`. Because `x_q` is then feasible for the point-score LP +under unchanged nonrisk constraints, the optimized point-score plug-in +objective must weakly dominate the guardrail objective. Every solve must +reconcile this proposition numerically. + +## Outcomes and Identification + +- Primary outcomes are standardized payoff, exposure-weighted terminal + default, and exposure-weighted binary-interval miscoverage. +- Standardized payoff is not IRR, NPV, welfare, or a lifetime cash-flow model. +- Unresolved outcomes receive sharp fixed-allocation and common-outcome paired + bounds. Sampling confidence intervals are not implied. +- A comparator envelope is reported once per policy, metric, window, and named + scope. Nested scopes are not counted as independent confirmations. + +## Factorial Simulation + +The mechanism experiment crosses score shift, outcome-prevalence shift, +taxonomy size, and censoring rate using deterministic independent streams. +Every cell reports binary-set geometry, candidate coverage, same-cap and C2 +allocation contrasts, and C2 objective dominance. The simulation explains +mechanisms only and cannot validate Lending Club signs. + +The illustrative allocation mechanism is fixed at a 0.40 score cap, 0.25 +upper-endpoint blend, 50 funded units from 2,000 equal-size candidates, no +binding segment concentration, and LGD 0.45. These values ensure that the +closed factorial can expose the binary phase transition without silently +dropping infeasible cells. They are not members of, or candidates for, the +empirical nine-policy family. + +## Stop Rules + +1. Stop if any outcome-derived field reaches fitting outside declared label + blocks, policy construction, comparator construction, or frontier ranging. +2. Stop if any residual month has retention below 99%, a fixed edge repeats, a + five-group window has fewer than 1,000 observations in a group, or a rank + fails exact reconciliation. +3. Stop if any budget is not filled, HiGHS is not optimal, a C2 moment misses + by more than `1e-10`, or point objective dominance fails by more than + `1e-5` dollars. +4. Report all eight windows. If any OOT upper coverage bound reaches 0.90, the + paper must state timing heterogeneity rather than invariant failure. +5. If CatBoost and logistic controls disagree, the result is learner-dependent. +6. If a direction fails anywhere in its named comparator support, report + comparator dependence. Do not expand or contract support after outcomes. +7. Do not report `N/N` across nested scopes as independent evidence. +8. Do not add HPO, a policy selector, a preferred window, or a new external + dataset after inspecting this run. +9. If the audit lacks a generalizable methodological contribution after the + complete run, narrow the paper or stop rather than revive pool93 or v7. + +## Reproducibility Contract + +The implementation and protocol must be committed and tagged before the V4 +run. Outputs use a fresh isolated run tag, immutable directories, atomic +writes, SHA-256 descriptors, and new DVC pointers. A clean-clone replay must +rebuild evidence and manuscript surfaces without executing protected stages. diff --git a/docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md b/docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md new file mode 100644 index 0000000..79da965 --- /dev/null +++ b/docs/research/ijds_binary_geometry_frontier_v4_v2_recovery_2026-07-12.md @@ -0,0 +1,35 @@ +# Binary-Geometry V4 V2 Evaluation Recovery + +## Status + +V4-v1 completed its outcome-free freeze at commit +`2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd`. An external pre-evaluation +audit found 1,872 guardrail solves, 1,080 C2 solves, a maximum absolute C2 +match residual of `8.33e-17`, no forbidden outcome columns, and complete +eight-window/nine-policy cardinality. Its `protocol_freeze.json` SHA-256 is +`c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c`. + +A no-write post-freeze smoke test then exposed an evaluation-only dataframe +collision. Shared frontier rows intentionally lacked window-specific +conformal endpoints, but concatenation with other allocation rows retained +all-null endpoint columns. Merging the selected window's endpoints therefore +created pandas suffixes instead of canonical `conformal_lower` and +`conformal_upper` columns. No evaluation artifact or deterministic summary +was written. + +## V2 Rule + +V4-v2 changes no scientific specification, model, score, residual recipe, +window, policy, comparator, frontier, simulation, outcome, hypothesis, or +stop rule. It: + +1. imports the V4-v1 outcome-free freeze only after verifying its identity, + SHA-256, and every artifact descriptor; +2. drops any placeholder window-specific endpoint columns before injecting + the chosen window recipe; and +3. writes all evaluation outputs under the fresh V4-v2 run directory. + +The imported allocations remain frozen before outcomes. V4-v1 is preserved +as valid outcome-free provenance but is not an evaluable final run. Required +V4-v2 tag: +`protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v2`. diff --git a/docs/research/ijds_claim_concept_audit_2026-06-26.md b/docs/research/ijds_claim_concept_audit_2026-06-26.md deleted file mode 100644 index 2258b59..0000000 --- a/docs/research/ijds_claim_concept_audit_2026-06-26.md +++ /dev/null @@ -1,374 +0,0 @@ -# IJDS Claim Concept Audit: Alpha Grid, Robust Region, Bound, and Exact Frontier - -> **Historical only.** This pre-closeout analysis uses the former linear -> certificate shortcut. Current claims and corrected endpoint thresholds live -> in `active_claims_2026-07-04.md` and -> `pool93_certificate_semantics_v2_2026-07-09.md`. - -Date: 2026-06-26 - -This memo audits the concepts that are easy to take for granted in the CRPTO -paper narrative. The goal is not to weaken the paper, but to make the strongest -IJDS-facing claim precise enough that it survives reviewer scrutiny. - -## Current Run Snapshot - -Active branch: `codex/champion-reopen-2026-06`. - -Active pool93 run: - -- run tag: `champion-reopen-2026-06-19__pool93__ijds-local-refine-stage1` -- stage: local exact refinement around ranks 96, 219, and 223 -- total exact checks: 6,520 -- latest observed progress: about 47.6% -- latest ETA at that point: about 9.3 hours - -The current partial leaderboard is already informative: - -- 815 local policies have appeared in the leaderboard. -- 329 policies pass all currently evaluated alpha checks and exceed the previous - champion return reference. -- The current max-return local policy is around 222.6K realized return with - `V(0.01)=0.071075` and `Gamma_CP(0.01)=0.459280`. -- The rank-219/rank-223 anchors are less explosive in return, but much cleaner - for a bound-facing claim, with `Gamma_CP` around 0.205--0.223 and - `V(0.01)` around 0.0446--0.0456 while still exceeding the previous champion - return. - -Interpretation: pool93 is not just producing one candidate. It is producing a -claim frontier: max realized return, tighter conformal premium, lower weighted -miscoverage, and balanced return-bound trade-offs. - -Later in the same partial run, after the bound-efficient neighborhood started -arriving, the more defensible IJDS lens became clear. The paper-facing bound -quantity is not `Gamma_CP` alone; it is the endpoint budget implied by Theorem 1, - -`tau + (1 - gamma) * Gamma_CP`, - -and the corresponding Markov cap, - -`tau + (1 - gamma) * Gamma_CP + sqrt(alpha)`. - -Under that lens, the provisional `pool93` body-default claim should not be the -max-return endpoint. The better IJDS-facing candidate is the return-bound point -that improves realized return while tightening the theorem's endpoint budget. -At the latest audited partial snapshot, that point is: - -- `local_candidate_id = 462` -- `local_family = bound_efficient_local` -- `risk_tolerance = 0.1725`, `gamma = 0.50`, `uncertainty_aversion = 0.10` -- realized return: `$183,832.67` -- return surplus versus the previous champion reference: `$13,368.13` -- `Gamma_CP(0.01) = 0.176347` -- `V(0.01) = 0.041341` -- endpoint budget: `0.2606735` -- Markov loss cap: `0.3606735` -- all-alpha pass count: `8/8` on the finite alpha grid - -For comparison, the previous champion has endpoint budget -`0.175 + 0.55 * 0.187987 = 0.27839285` and Markov cap `0.37839285`. -Thus the provisional return-bound candidate is more modest than the max-return -frontier endpoint, but it gives a cleaner paper claim: higher realized return -and tighter theorem-facing endpoint budget. Its realized `V` is higher than the -previous champion's `0.028875`, so the claim should not say "lower realized -weighted miscoverage versus the old champion"; it should say "all-alpha-grid -safe under the Markov audit, with tighter endpoint budget and higher return." - -## 1. What `8/8` Really Means - -In code, the number eight comes from the current alpha sweep artifact and the -default alpha grid: - -```python -DEFAULT_ALPHAS = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] -``` - -These are the eight supported conformal levels in -`data/processed/alpha_sweep_pareto_mondrian.parquet`. Therefore `8/8` means: - -> The policy passed every alpha level in the pre-specified finite alpha grid -> A = {0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}. - -It does not mean: - -- a theorem requires exactly eight alpha values; -- all possible alpha values in (0, 1) were certified; -- the policy has universal conditional coverage; -- the policy is guaranteed for every future portfolio selection. - -The related literature supports pre-specified risk or coverage levels, not a -canonical eight-point alpha grid. Conformal Risk Control controls expected -monotone losses at user-chosen levels; Risk-Controlling Prediction Sets and -Learn-Then-Test calibrate predictive algorithms to satisfy explicit finite-sample -risk criteria; conformal robust optimization papers use conformal sets as -uncertainty sets at declared coverage levels. None of these sources says the -audit must contain eight alpha levels. - -Primary sources checked for this point: - -- Angelopoulos et al., Conformal Risk Control, ICLR 2024: - https://openreview.net/forum?id=33XGfHLtZg -- Bates et al., Distribution-Free, Risk-Controlling Prediction Sets: - https://arxiv.org/abs/2101.02703 -- Angelopoulos et al., Learn Then Test: - https://arxiv.org/abs/2110.01052 -- Johnstone and Cox, Conformal Uncertainty Sets for Robust Optimization: - https://proceedings.mlr.press/v152/johnstone21a.html -- Patel et al., Conformal Contextual Robust Optimization, AISTATS 2024: - https://proceedings.mlr.press/v238/patel24a.html -- Sun et al., Predict-then-Calibrate: - https://arxiv.org/abs/2305.15686 -- Bertsimas and Sim, The Price of Robustness: - https://pubsonline.informs.org/doi/10.1287/opre.1030.0065 - -Recommended paper language: - -> The promoted policy passes all evaluated levels in the pre-specified alpha -> grid, `8/8` over A = {0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}. - -Avoid: - -> The policy is robust for all alpha. - -If we want a stronger claim, the correct next step is to generate and freeze a -denser conformal alpha sweep, then rerun exact validation on that declared grid. -Interpolating unsupported alphas would be useful as a diagnostic curve, but not -as the primary certificate. - -## 2. What The Robust Region Really Means - -The existing `45/45` region means: - -> Every policy in a finite, declared local policy grid passed the exact alpha01 -> check. - -For the current paper champion this grid is described as the cross-product of -five risk-tolerance values, three gamma values, and three uncertainty-aversion -settings within the bound-aware family. - -This is valuable because it shows the promoted policy is not an isolated lucky -point. But it is not the same as a continuous robust feasible region in the -mathematical programming sense. Classical robust optimization discusses -uncertainty sets, robust counterparts, protection levels, and price of -robustness. CRPTO's "robust region" is better described as a finite policy-grid -stability surface. - -Recommended paper language: - -> The final finite policy-grid region contains 45 evaluated policies, and all -> 45 pass the alpha01 exact funded-set audit. - -For pool93 local refinement, the equivalent claim should be reported as: - -- exact evaluated policies; -- number and rate of all-alpha passers; -- number and rate of passers above champion-return reference; -- family-level pass rates; -- the alpha set used. - -Do not transfer the old `45/45` phrase to pool93 unless the final selected -pool93 region is explicitly defined and frozen with exactly 45 policies. A -larger and better pool93 region can be stronger, but it needs its own denominator. - -## 3. What The Bound Actually Certifies - -The current implementation checks each alpha by computing: - -- `weighted_miscoverage_V = sum_i w_i 1{Y_i > u_i(alpha)}` -- `gamma_cp = sum_i w_i clip(u_i(alpha) - p_hat_i, 0, 1)` -- `weighted_pd_true = sum_i w_i Y_i` -- `violation = max(0, weighted_pd_true - tau)` -- `all_bounds_hold = (violation <= alpha) and (V <= sqrt(alpha))` - -The strongest clean theoretical object is: - -1. A deterministic accounting identity: for fixed weights, realized loss is - bounded by the conformal upper-endpoint budget plus realized weighted - miscoverage. -2. A Markov bound under the explicit assumption - `E[V(alpha)] <= alpha`. -3. An exact frozen funded-set audit showing the selected policy's realized - `V`, `Gamma_CP`, violation, and pass/fail indicators. - -The manuscript is already honest on the key point: for the existing champion, -`V(0.01)=0.028875` is above `alpha=0.01`, so the paper must not claim nominal -funded-set alpha coverage. The operative audit is `V <= sqrt(alpha)` and zero -violation, together with the explicit weighted funded-set validity assumption. - -For IJDS this is a strength, not a weakness, if framed correctly: - -> CRPTO separates deterministic portfolio accounting, distribution-free -> first-moment risk control under a stated funded-set validity assumption, and a -> frozen exact empirical certificate. - -Avoid: - -> The conformal method guarantees the selected funded set has 99% coverage. - -## 4. What "Exact Frontier" Should Mean - -The exact refinement is exact in the allocation solve for a fixed policy and -fixed alpha. It is not a proof of global optimality over all possible policy -families or continuous hyperparameter values. - -Therefore: - -- "exact full-universe rerank" is correct; -- "exact policy-grid frontier" is correct; -- "global exact optimum" is too strong unless the full continuous search space is - formally restricted to the declared finite grid. - -The pool93 local refinement improves the IJDS position because it ranks using -metrics from the exact full-universe allocation itself, not the earlier -frontier/proxy score. That is a better evidence chain for the paper. - -Recommended default artifact for the paper: - -- do not promote a single max-return point alone; -- promote a frontier with three named points: - - economic max-return point; - - bound-efficient point, ranked by `tau + (1-gamma) Gamma_CP + sqrt(alpha)`; - - balanced IJDS point, ranked by a declared return-bound score. - -This gives reviewers a principled choice surface rather than asking them to -accept one lucky corner. If the final pool93 result is promoted, the artifact -should expose the alpha grid, all-alpha denominator, candidate region -denominator, and claim lens used to select the paper-facing point. - -## 5. The Biggest Remaining Conceptual Risk: Post-Selection - -The optimizer does not see OOT labels when solving a fixed policy. However, the -research process can still overfit the OOT backtest if the final policy is chosen -because it has the best realized OOT return after many policies have been tried. - -That does not invalidate the result, but it changes the claim: - -- safe claim: retrospective frozen backtest plus exact funded-set certificate; -- stronger claim: predeclared policy selection rule evaluated once on untouched - final OOT labels; -- strongest claim: nested temporal selection/evaluation or prospective replay. - -For the current pool93 local refinement, the safest IJDS framing is: - -> The search identifies a certified return-bound frontier on a frozen OOT -> backtest. Promotion should prefer a policy selected by a declared -> return-bound criterion, not solely by the realized-return maximum. - -This is why the rank-219/rank-223 anchors matter. They are not just backup -policies; they are selection-bias insurance for the paper narrative. - -## 6. Academic-Researcher Skill Audit - -The `academic-researcher` skill recommends exactly the discipline needed here: - -- define the claim target; -- define the evidence gate; -- define the artifact sink; -- define the stop rule; -- promote only what is supported by the artifact; park or archive the rest. - -That skill is valuable for CRPTO. The current paper already follows much of it -through claim ladders, assumption maps, frozen artifacts, and negative-result -boundaries. - -No installed skill named `crpto` was found in the visible skill registry for this -session. The CRPTO-specific source of truth is therefore the project itself: -scripts, frozen artifacts, Quarto manuscript, supplement, and research memos. - -## 7. Recommended Claim Hierarchy For IJDS - -Tier 1, strongest and safest: - -> A frozen PD-calibration-conformal-optimization pipeline maps calibrated credit -> uncertainty into an auditable robust portfolio decision, with exact funded-set -> accounting on a full OOT universe. - -Tier 2, empirical but defensible: - -> The selected pool93 policy lies on a finite exact policy-grid frontier where -> all evaluated alpha levels in the declared grid pass the exact audit. - -Tier 3, economic: - -> The frontier contains policies that exceed the previous champion return while -> preserving zero violation and all-alpha-grid pass status. - -Tier 4, optional if the final region supports it: - -> The result is not isolated: a declared local policy-grid region has a high -> all-alpha pass rate, and a material subset of that region exceeds the champion -> return reference. - -Do not make the primary paper claim: - -- AUC leaderboard dominance; -- universal coverage of adaptively selected portfolios; -- global optimality over all policies; -- continuous-region robustness unless we explicitly solve or prove it. - -## 8. Recommended Next Experimental Guardrail - -After stage1 completes: - -1. Freeze the exact alpha grid in the claim summary. -2. Report both count and denominator, never only `8/8`. -3. Choose the IJDS candidate from a predeclared claim score, for example: - - `pass_all_alpha_grid = true` - `champion_return_surplus > 0` - then maximize a balanced score over normalized return surplus, the inverse - Markov loss cap, and inverse `V`. - -4. Keep the pure max-return point as an economic frontier endpoint. -5. Keep the rank-219/rank-223 family as the bound-efficient endpoint. -6. If the local stage reveals a clean contiguous region, only then run expanded - exact refinement around that region. - -This makes the paper stronger because it shifts the story from "we found a big -number" to "we found a certified decision frontier and promoted the point whose -claim is hardest to attack." - -## 9. Final Pool93 Closure Update - 2026-07-02 - -The terminal pool93 search closed the concept audit with a stronger and cleaner -denominator than the earlier `45/45` language: - -- terminal endpoint search: 37,068 policies and 296,544 exact alpha checks; -- terminal all-alpha passers: 37,068/37,068; -- terminal all-alpha passers above the declared return floor: 14,814/37,068; -- consolidated semantic frontier: 50,010 deduplicated policies; -- consolidated eligible all-alpha and above-floor policies: 27,508; -- declared alpha grid remains the finite set - `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. - -The strongest IJDS claim is now a finite-grid return-bound frontier, not a -single champion and not a continuous robust region. The recommended paper-facing -roles are: - -| Role | Candidate | Return | Gamma_CP | V | Markov cap | Destination | -|---|---:|---:|---:|---:|---:|---| -| Body/default balanced point | 131 | 184,832.475845 | 0.162616 | 0.035350 | 0.345083740 | Body | -| Strict cap <= 0.345 proxy | 512 | 184,800.413581 | 0.162562 | 0.035350 | 0.344996495 | Body footnote or supplement | -| Above-floor minimum-cap endpoint | 10661 | 170,467.268819 | 0.095719 | 0.031875 | 0.273035950 | Body frontier sentence and supplement | -| Max-return endpoint | 4041 | 223,458.135875 | 0.457438 | 0.069575 | 0.510753090 | Supplement/frontier endpoint | - -Final claim wording: - -> CRPTO reports a frozen finite-grid exact return-bound frontier for credit -> allocation. On the Lending Club OOT universe, the selected pool93 body point -> earns 184.8K on a 1M budget, passes all eight declared alpha levels, and has -> Gamma_CP = 0.1626, V = 0.03535, and Markov cap = 0.3451. The same frontier -> contains an above-floor bound endpoint with Markov cap = 0.2730 and an -> economic endpoint above 223K, making the return-bound tradeoff explicit. - -What changes conceptually: - -- `8/8` remains a finite alpha-grid certificate; it is not universal alpha - coverage. -- "Robust region" should be replaced by "finite policy-grid robustness - surface" or "finite-grid return-bound frontier" with explicit denominators. -- The bound claim remains Theorem 1 plus Assumption 1 plus exact audit. It does - not become a post-selection conformal theorem. -- The terminal endpoint is valuable because it improves the bound cap; it does - not replace the body/default point, whose role is to be useful and defensible - as the manuscript's main economic decision. diff --git a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md b/docs/research/ijds_claim_maximization_analysis_2026-06-27.md deleted file mode 100644 index 6002fc4..0000000 --- a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md +++ /dev/null @@ -1,1131 +0,0 @@ -# IJDS Claim Maximization Analysis - 2026-06-27 - -> **Historical only.** This search diary predates the policy-aware certificate -> correction. Do not reuse its cap labels or endpoint thresholds; current A35 -> and A40 evidence is registered in `pool93_certificate_semantics_v2_2026-07-09.md`. - -This memo evaluates the strongest IJDS-facing claims for CRPTO after the pool93 -claim-governance and local exact-refinement work. It is an internal research -artifact, not manuscript prose. - -## Executive Decision - -The strongest paper claim is not "a better credit classifier" and not "a new -universal conformal theorem." The strongest claim is: - -> CRPTO is an auditable conformal-robust credit-portfolio decision certificate: -> it maps a frozen calibrated PD artifact into conformal upper endpoints, solves -> a robust funding policy, and reports an exact full-universe finite-grid -> return-bound frontier with explicit alpha-grid and Markov-bound semantics. - -This should replace the current body emphasis on a single promoted economic -policy and a 45/45 local region if pool93 is promoted. - -## Current Evidence Snapshot - -Stage1 pool93 exact refinement is final: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-local-refine-stage1` -- exact policies: 815 -- exact alpha checks: 6,520 -- alpha grid: `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}` -- all-alpha passers: 756/815 -- bound-efficient passers: 418/418 -- bottleneck: alpha = 0.01; all other alphas pass 815/815 - -Paper-body default candidate from governance: - -- local candidate: 462 -- family: `bound_efficient_local` -- policy: `blended_uncertainty` -- risk tolerance: 0.1725 -- gamma: 0.50 -- uncertainty aversion: 0.10 -- realized return: 183,832.670701 -- return-floor surplus: 13,368.130701 -- Gamma_CP: 0.176347 -- weighted miscoverage V: 0.041341 -- endpoint budget upper: 0.2606735 -- Markov loss cap at alpha01: 0.3606735 -- alpha pass count: 8/8 -- mean funded loans: 318.5 - -Frontier endpoints: - -- bound-tight: candidate 466, return 181,217.537003, Gamma_CP 0.176071, - Markov cap 0.3605355, 8/8. -- return-bound body/default: candidate 462, return 183,832.670701, - Markov cap 0.3606735, 8/8. -- economic endpoint: candidate 264, return 222,602.669743, - Markov cap 0.5106040, 8/8. - -The live `claim_expanded` refinement is running and is not yet claim-final. It -is useful because it can refine the bound ridge and economic endpoint, but the -existing stage1 evidence is already strong enough to redesign the claim surface. - -## Live Update - 2026-06-27 20:44 America/Bogota - -The expanded local refinement is materially underway but remains exploratory: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine` -- profile: `claim_expanded` -- completed exact checks: 10,105/26,064 -- completion: 38.77 percent -- observed throughput: about 6.77 exact checks/minute under concurrent HPO load -- ETA at current throughput: about 39.3 hours, around 2026-06-29 12:02 local -- current family: `bound_claim_refined_local` -- current anchor rank: 219 - -Interpretation: the run has reached the region that can improve the paper claim -because rank 219 is where the Stage1 bound-efficient body/default point came -from. Do not promote this expanded run yet, because no final expanded -leaderboard or governance artifact exists in the run directory. Treat it as a -live claim-improvement search whose evidence gate is a regenerated governance -JSON plus a frontier table after the run finishes or after a deliberately -declared early-stop checkpoint. - -The HPO wave is still running on `pooltop72_business80`; seven of eight cases -have completed seed-42 HPO. Among completed cases, test-set metrics are tightly -clustered: - -| case | features | AUC | Brier | ECE | PR-AUC | -|---|---:|---:|---:|---:|---:| -| `pooltop72_tab60` | 132 | 0.721661 | 0.153035 | 0.007982 | 0.412929 | -| `pooltop93_tab120` | 213 | 0.721641 | 0.153019 | 0.008258 | 0.413265 | -| `pool93_woe` | 106 | 0.721562 | 0.153022 | 0.008594 | 0.413707 | -| `pooltop80_business80` | 160 | 0.721438 | 0.153077 | 0.008186 | 0.412686 | -| `pool93` | 93 | 0.721356 | 0.153066 | 0.007821 | 0.412954 | -| `pool93_business80` | 173 | 0.721309 | 0.153120 | 0.008017 | 0.412025 | -| `pooltop80_tab90` | 170 | 0.721294 | 0.153108 | 0.008175 | 0.412365 | - -The HPO differences are small enough that paper-facing selection should not be -driven by seed-42 AUC alone. `pool93` remains the strongest body candidate if -its downstream chain dominates because it is compact, self-contained, and -already has the strongest exact portfolio-governance evidence. `pool93_woe` and -`pooltop72_tab60` are plausible appendix/downstream challengers because they -show marginally stronger predictive metrics, but they must earn promotion at -calibration, conformal, and portfolio layers. - -Expanded search is warranted only if it can change one of these manuscript -claims: - -- identify a higher-return point with Markov cap no worse than Stage1 candidate - 462; -- identify a lower-cap point with return still above the declared body-return - floor; -- show a wider, cleaner finite-grid robustness surface around the selected - point; -- provide an interpretable frontier figure/table that replaces single-point - reporting. - -Stop rule: if expanded search only creates more high-return/high-cap endpoints, -append it as frontier sensitivity and keep Stage1 candidate 462 as the body -default. If it finds a point that dominates 462 on return and cap, regenerate -governance and promote that point. - -## Engineering Update - 2026-06-27 21:20 America/Bogota - -The search was re-prioritized around pool93 because the other subsets are not -currently the best use of compute for the paper. HPO was paused with `SIGSTOP` -and remains resumable. Before pausing, HPO was using about 18.3 CPU cores and -19.1 GiB RAM, while the pool93 exact refinement used about one CPU core and -2.1 GiB RAM. - -Observed exact-refinement throughput: - -- with HPO running: about 6.8 checks/minute; -- with HPO paused, sequential exact runner: about 15.7 checks/minute; -- with independent exact solves in parallel, 6 workers: about 50-75 - checks/minute; -- with independent exact solves in parallel, 8 workers: active and stable at - eight near-full CPU workers; status ETA after corrected resume logic is about - 4.3 hours, pending a longer throughput sample. - -Technical change applied: - -- `scripts/search/run_pool93_ijds_local_refinement.py` now supports - `--parallel-workers`. -- The parallel path runs independent candidate-alpha exact solves in worker - processes and keeps checkpoint/leaderboard/governance writes centralized in - the parent process. -- Resume semantics remain keyed by `(local_candidate_id, alpha)`. -- The status ETA calculation was corrected for resumed runs by tracking - `completed_checks_at_start` and `completed_checks_this_run`. - -Partial expanded evidence is already paper-relevant. At the latest checkpoint, -the expanded `bound_claim_refined_local` region produced a candidate that -dominates Stage1 candidate 462 on the paper-facing return-bound lens: - -- local candidate: 1076 -- family: `bound_claim_refined_local` -- policy: `capped_blended_uncertainty` -- risk tolerance: 0.1705 -- gamma: 0.50 -- uncertainty aversion: 0.1125 -- realized return: 185,735.027517 -- Gamma_CP: 0.173059 -- weighted miscoverage V: 0.037750 -- endpoint budget upper: 0.2570295 -- Markov loss cap at alpha01: 0.3570295 -- alpha pass count: 8/8 - -This is not yet a final manuscript result because the expanded run has not -closed. But it changes the evidence gate: the run is no longer merely searching -for sensitivity. It is searching for the final body/default policy on a stronger -return-bound frontier. - -## Final Expanded Results - 2026-06-28 01:08 America/Bogota - -The expanded pool93 run closed successfully: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine` -- profile: `claim_expanded` -- exact policies: 3,258 -- exact alpha checks: 26,064 -- alpha grid: `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}` -- all-alpha passers: 3,071/3,258 -- all-alpha passers above declared return floor: 3,071/3,258 -- bottleneck: alpha = 0.01; all other alpha levels pass 3,258/3,258 -- full `bound_claim_refined_local` pass rate: 1,660/1,660 - -The final body/default point should move from Stage1 candidate 462 to the -expanded return-bound point: - -- local candidate: 1665/1667 equivalent allocation family -- family: `bound_claim_refined_local` -- policy: `blended_uncertainty` or equivalent capped variant -- risk tolerance: 0.1720 -- gamma: 0.55 -- uncertainty aversion: 0.05 -- realized return: 184,472.681224 -- Gamma_CP: 0.162673 -- weighted miscoverage V: 0.036017 -- endpoint budget upper: 0.24520285 -- Markov loss cap at alpha01: 0.34520285 -- alpha pass count: 8/8 -- mean funded loans: 320.625 - -This point is stronger than the Stage1 body/default on the intended -return-bound claim: it keeps a materially higher return than the declared -return floor, lowers Gamma_CP and the Markov cap versus Stage1 candidate 462, -and lies in a fully passing 1,660-policy bound-refined surface. It should be -presented as the manuscript body point if pool93 is promoted. - -Important caveat: relative to the old manuscript policy, this point does **not** -lower realized V; the old manuscript reported V = 0.028875. The correct -paper-facing claim is therefore not "lower V than the previous manuscript -policy." The correct claim is: - -> a selected body point on a predeclared exact finite-grid return-bound frontier -> realizes 184.5K return on a 1M budget while passing all eight alpha levels, -> with Gamma_CP = 0.1627, endpoint budget upper = 0.2452, and Markov cap = -> 0.3452. - -The final expanded frontier gives several defensible endpoints: - -| Role | Candidate | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|---:|---:| -| Bound-tight endpoint | 1206 | 181,242.759646 | 0.153611 | 0.038375 | 0.335785 | 8/8 | -| Body/default return-bound point | 1665 | 184,472.681224 | 0.162673 | 0.036017 | 0.345203 | 8/8 | -| Highest return under cap <= 0.36 | 1922 | 185,984.969939 | 0.174479 | 0.037750 | 0.358495 | 8/8 | -| Higher-return frontier under cap <= 0.45 | 979 | 198,693.277519 | 0.252323 | 0.045600 | 0.449010 | 8/8 | -| Economic endpoint under cap <= 0.50 | 3021 | 222,558.702500 | 0.459083 | 0.071075 | 0.489837 | 8/8 | -| Max-return endpoint | 2857/2777 | 223,346.552500 | about 0.4574 | 0.069575 | 0.5086--0.5127 | 8/8 | - -Best IJDS claim after this run: - -> CRPTO reports an exact finite-grid return-bound frontier for credit allocation: -> among 3,258 evaluated pool93 policies and 26,064 full-universe alpha checks, -> 3,071 policies pass every predeclared alpha level; the selected body point -> realizes 184.5K return with Markov cap 0.3452, while the frontier separately -> exposes bound-tight and economic endpoints. - -This is stronger and cleaner than the older single-policy `45/45` robust-region -framing. The robust-region language should become "finite policy-grid robustness -surface" with explicit denominators: 1,660/1,660 for the bound-refined local -surface and 3,071/3,258 for the expanded exact policy grid. - -## Final Micro-Refinement Results - 2026-06-28 06:19 America/Bogota - -The targeted `claim_micro` refinement closed successfully. It improves the -expanded body point and, more importantly for IJDS, gives a cleaner finite-grid -robustness surface around the claim-bearing neighborhoods discovered by the -expanded run. - -Final micro run: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine` -- profile: `claim_micro` -- workers: 10 independent exact-solve workers -- exact policies: 2,949 -- exact alpha checks: 23,592 -- all-alpha passers: 2,949/2,949 -- all-alpha passers above declared return floor: 2,949/2,949 -- bottleneck: none inside this micro surface; every evaluated policy passes - every alpha in the declared grid - -Final body/default micro point: - -- local candidate: 37 -- family: `claim_micro_body_low_v` -- policy: `blended_uncertainty` -- risk tolerance: 0.1715 -- gamma: 0.55 -- uncertainty aversion: 0.0375 -- realized return: 184,687.272568 -- Gamma_CP: 0.161995 -- weighted miscoverage V: 0.035350 -- endpoint budget upper: 0.24439775 -- Markov loss cap at alpha01: 0.34439775 -- alpha pass count: 8/8 -- mean funded loans: 320.75 - -This dominates the expanded body/default point 1665/1667 on the paper-facing -balanced lens: - -- return improves by 214.591344; -- Gamma_CP falls by 0.000678; -- weighted miscoverage V falls by 0.000667; -- Markov cap falls by 0.0008051. - -The improvement over the expanded point is modest, but it is directional on -all four body metrics. Relative to the current manuscript numbers, the final -micro body point raises return by 14,222.729640 (about 8.34%) and lowers -Gamma_CP by 0.025992 (about 13.83%). Its V is higher than the current -manuscript policy, so the body should not claim lower V relative to a historical -policy. The paper-facing claim should instead be a return-bound frontier claim. - -Final micro frontier roles: - -| Role | Candidate | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|---:|---:| -| Bound-tight endpoint | 949 | 179,763.276471 | 0.146997 | 0.038375 | 0.328799 | 8/8 | -| Body/default balanced point | 37 | 184,687.272568 | 0.161995 | 0.035350 | 0.344398 | 8/8 | -| Highest return under cap <= 0.345 | 205 | 184,770.883882 | 0.162402 | 0.035350 | 0.344831 | 8/8 | -| Highest return under cap <= 0.36 | 1975 | 186,046.522897 | 0.174593 | 0.037750 | 0.358678 | 8/8 | -| High-return endpoint under cap <= 0.50 | 2616 | 222,558.702500 | 0.459075 | 0.071075 | 0.491878 | 8/8 | -| Max-return economic endpoint | 2122 | 223,369.907874 | 0.457446 | 0.069575 | 0.512920 | 8/8 | - -Decision: - -- Promote candidate 37 as the paper body/default point if pool93 is promoted. -- Use candidate 205 as a near-equivalent "highest return under cap <= 0.345" - sensitivity in the frontier table. -- Use candidate 1975 as the "highest return under cap <= 0.36" endpoint. -- Keep candidate 2122 as an economic endpoint only. It is useful to show the - return-bound tradeoff, but it should not be the body policy because its Markov - cap is much larger. - -Manuscript consequence: - -> CRPTO selects a pool93 body policy from a predeclared finite-grid exact -> frontier. On the 276,869-loan OOT universe and a 1M budget, the selected point -> realizes about 184.7K return, passes all eight alpha levels in the declared -> grid, and reports Gamma_CP about 0.162, weighted miscoverage about 0.035, and -> Markov cap about 0.344. - -This wording should not compare against a previous champion. The comparison set -visible to the paper is the declared finite-grid frontier itself: the body point, -the bound-tight endpoint, and the economic endpoint. - -Search-extension gate: - -- A small follow-up search is scientifically justified only around exposed - boundaries that could change the frontier table: - - `claim_micro_bound_tight`: winner is at risk minimum, gamma maximum, and - aversion maximum. A narrower bound-tight extension could reduce Markov cap - below 0.3288 but would likely cost return. - - `claim_micro_high_return_cap036`: winner is at the risk upper boundary. A - small extension could improve the cap-0.36 endpoint by tens or hundreds of - dollars. - - `claim_micro_economic_endpoint`: winner is at gamma minimum. A small - endpoint extension could add small return, but this is not the body claim. -- A broad new search is not justified by the current evidence. The paper claim - is already supported by a completed governance sidecar and a complete - 2,949-policy all-alpha micro surface. - -## Surgical Micro-Extensions Launched - 2026-06-28 10:37 America/Bogota - -The final micro run exposed several grid edges that can still change the -frontier table. I therefore launched one bounded follow-up run rather than a -new broad portfolio search. - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- profile: `claim_micro_ext` -- workers: 12 independent exact-solve workers -- exact policies: 4,407 -- exact alpha checks: 35,256 -- alpha grid: `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}` - -Families: - -| Family | Policies | Claim target | Evidence gate | -|---|---:|---|---| -| `claim_micro_ext_body_cap345` | 660 | Improve the body/default or cap<=0.345 point. | Return above candidate 37 or 205 with Markov cap at or below about 0.345. | -| `claim_micro_ext_bound_tight` | 1,584 | Improve the bound-tight endpoint. | Lower Markov cap than 0.328799 while preserving all-alpha pass and nonnegative return-floor surplus. | -| `claim_micro_ext_cap036_return` | 1,080 | Improve highest return under cap<=0.36. | Return above 186,046.522897 with Markov cap <= 0.36. | -| `claim_micro_ext_economic_endpoint` | 1,080 | Stress the high-return endpoint. | Return above 223,369.907874 while keeping all-alpha pass; appendix/frontier only. | - -Initial checkpoint: - -- completed checks: 140/35,256 -- ETA at first stabilized checkpoint: about 8.2 hours -- first partial body candidate under cap<=0.345: - return 184,739.352839, Gamma_CP 0.162248, V 0.035350, - Markov cap 0.344667. - -Interpretation: the first partial body improvement is useful but not yet -frontier-changing relative to candidate 205. The run should continue because -the three edge families that motivated the extension have barely started. - -## Literature Landscape And CRPTO Position - -### 1. Conformal risk control and risk-controlling prediction sets - -Closest sources: - -- Bates et al. 2021, Risk-Controlling Prediction Sets. -- Angelopoulos et al. 2024, Conformal Risk Control. -- Angelopoulos et al. 2025, Learn Then Test. -- Angelopoulos et al. 2026, Non-Monotonic CRC. - -Their contribution: distribution-free or finite-sample calibration of predictive -sets or risk functions, often model-agnostic and post-hoc. - -CRPTO should not claim to improve those general guarantees. CRPTO uses this -family as the statistical substrate, then moves the certificate into a credit -portfolio decision. The IJDS contribution is the operational coupling: -calibrated PD -> conformal endpoints -> robust LP -> funded-set audit. - -Promote: - -- "decision-level audit of conformal uncertainty in credit allocation." - -Avoid: - -- "new conformal risk-control theorem." -- "selected funded set has nominal alpha coverage." - -### 2. Conformal robust optimization and contextual robust optimization - -Closest sources: - -- Johnstone and Cox 2021, conformal uncertainty sets for robust optimization. -- Patel et al. 2024, Conformal Predict-Then-Optimize. -- Sun et al. 2024, Predict-then-Calibrate for robust contextual LP. -- Hu et al. 2026, Conformal Robustness Control. -- Zhao et al. 2026, CRO and satisficing. -- Bao et al. 2025, CROMS model selection. -- Zhou et al. 2025, CREDO. -- Zhou and Zhu 2025, inverse conformal risk control for robustness levels. - -Their contribution: broad methods for CP-based uncertainty sets, robustness -control, model selection, decision reliability, and robustness-level calibration. - -CRPTO's best position is narrower and more empirical: a credit-specific, -artifact-backed, full-universe certificate that explicitly separates endpoint -budget, Markov cap, realized V, and return. This is more IJDS-compatible than -claiming theoretical dominance over CRC/CROMS/CREDO. - -Promote: - -- "finite-grid exact return-bound frontier for a real credit portfolio." -- "governed robustness-level selection, not an ad hoc gamma choice." -- "auditability as the contribution: a selected policy plus interpretable - frontier endpoints." - -Avoid: - -- "globally optimal robust policy over continuous parameters." -- "better than CRC/CROMS/CREDO." - -### 3. Robust optimization and price of robustness - -Closest sources: - -- Bertsimas and Sim 2004, price of robustness. -- Bertsimas et al. 2018, data-driven robust optimization. -- Goldfarb and Iyengar 2003, robust portfolio selection. -- Delage and Ye 2010, DRO under moment uncertainty. - -Their contribution: uncertainty sets, tractable robust counterparts, and -explicit conservatism-performance tradeoffs. - -CRPTO should reuse that language. The new pool93 evidence makes this stronger: -instead of one policy plus a heatmap, we can report a return-bound frontier with -the endpoint budget and Markov cap visible. - -Promote: - -- "price of robustness is measured as a frontier, not asserted." -- "the selected policy is the body point on a declared finite frontier." - -Avoid: - -- "robustness is free" or "robustness always improves return." - -### 4. Predict-then-optimize and decision-focused learning - -Closest sources: - -- Bertsimas and Kallus 2020, from predictive to prescriptive analytics. -- Elmachtoub and Grigas 2022, SPO/SPO+. -- Liu and Grigas 2021, risk bounds for SPO+. -- Donti et al. 2017, task-based end-to-end learning. -- Mandi et al. 2024, DFL survey. -- Schutte et al. 2024, robust losses for DFL. - -Their contribution: train models for decision quality or regret, not just -prediction error. - -CRPTO's distinction is that it remains post-hoc and governance-friendly: the -PD model is frozen, and the certificate is built around the decision layer. This -is weaker than end-to-end optimality but stronger for auditability and model-risk -management. - -Promote: - -- "post-hoc decision certificate for frozen ML artifacts." -- "regret-auditability frontier: SPO+ is a comparator, CRPTO is the auditable - risk-control corner." - -Avoid: - -- "CRPTO minimizes regret better than SPO+." - -### 5. P2P lending and credit portfolio decision support - -Closest sources: - -- Guo et al. 2016, instance-based credit-risk assessment and portfolio - allocation. -- Serrano-Cinca and Gutierrez-Nieto 2016, profit scoring. -- Zhao et al. 2016, multi-objective P2P portfolio selection. -- Chi et al. 2019, data-driven robust credit portfolio optimization in P2P. -- Babaei and Bamdad 2020, multi-objective P2P recommendation. -- Torkian et al. 2025/2026, AI plus OR for digital lending. - -Their contribution: P2P loan selection, risk-return optimization, multi-objective -portfolio recommendation, robust P2P portfolio models. - -CRPTO's gap is not "we optimize P2P portfolios"; that exists. The gap is: -portfolio selection with conformal PD uncertainty, exact funded-set -miscoverage accounting, and an explicit paper-governed return-bound frontier. - -Promote: - -- "credit portfolio decision certificate, not only P2P recommendation." -- "exposure-weighted funded-set audit with interpretable grade/risk - decomposition." - -Avoid: - -- "first portfolio optimizer for P2P lending." - -### 6. Credit scoring, calibration, and fairness context - -Closest sources: - -- Jagtiani and Lemieux 2019, alternative data and fintech lending. -- Albanesi and Vamossy 2024, credit scores performance and equity. -- Kawasumi et al. 2026, conformal prediction for ordinal credit scoring. -- Yang and Bi 2025, cost-aware calibration. - -Their contribution: predictive performance, calibration, equity/fairness, and -credit-score uncertainty. - -CRPTO can use these sources to justify why calibrated PD matters, but should not -turn the paper into a fairness or underwriting-credit-score paper. - -Promote: - -- "calibrated probabilities are necessary but insufficient; the paper studies - how they shape a funding decision." - -Avoid: - -- statutory fair-lending proof; -- equity improvement claims without direct protected attributes. - -### 7. Limits of conformal validity - -Closest sources: - -- Barber et al. 2021, limits of conditional predictive inference. -- Barber et al. 2023, beyond exchangeability. -- Gibbs and Candes 2021, adaptive conformal inference. -- Gibbs et al. 2025, conditional guarantees. -- Bhattacharyya and Barber 2026, group-weighted CP. -- Yang and Jin 2026, multi-distribution robust CP. - -Their contribution: they define what is and is not possible under -distribution-free conformal inference, especially under conditional coverage, -shift, group weighting, and multi-source deployment. - -CRPTO should use this literature as a guardrail. The paper can say it audits -weighted funded-set validity and reports group/multi-distribution diagnostics, -not that it has exact conditional validity for selected portfolios. - -Promote: - -- "explicit validity ladder." -- "finite-grid and retrospective exact audit." - -Avoid: - -- universal conditional coverage; -- future live-deployment coverage without online protocol. - -## Proposed Claim Hierarchy - -### Main Claim - -CRPTO provides a reproducible conformal-robust decision certificate for credit -portfolio selection by connecting frozen calibrated PD artifacts, conformal -uncertainty, robust LP decisions, and exact full-universe funded-set audits. - -Evidence: - -- final stage1 pool93 governance sidecar; -- 276,869-row full-universe OOT evaluation; -- all-alpha finite-grid audit; -- claim-selection protocol and frozen artifact paths. - -Destination: abstract, introduction, method, results. - -### Result Claim - -On the pool93 finite-grid exact refinement, the paper-body policy sits on a -return-bound frontier: it preserves all-alpha-grid safety while giving a -cleaner endpoint budget/Markov cap than the old manuscript policy and materially -higher realized return relative to the declared return floor. - -Paper-facing phrasing should not mention an old champion. Use: - -> relative to the declared return floor used in the selection protocol - -or avoid the comparison entirely: - -> the selected body point realizes 183.8K return at a 0.3607 Markov cap and -> passes all eight predeclared alpha checks. - -Destination: results table and frontier figure. - -### Theory Claim - -The theorem is a first-moment Markov decision certificate under weighted -funded-set validity. The exact audit verifies deterministic accounting and the -observed finite-grid pass/fail indicators. It is not a new conditional-coverage -theorem. - -Destination: theory section, assumption-to-evidence table. - -### Robust Region Claim - -Replace generic "robust region" language with: - -> finite policy-grid robustness surface. - -For pool93, the stronger summary is: - -- 756/815 all-alpha passers in stage1; -- 418/418 bound-efficient passers; -- claim-expanded refinement pending. - -Destination: results and supplement. - -### Frontier Claim - -The most IJDS-friendly presentation is a three-point frontier: - -1. Bound-tight endpoint: candidate 466. -2. Body/default return-bound point: candidate 462. -3. Economic endpoint: candidate 264. - -This prevents the paper from overselling max return while still showing the -economic opportunity. - -Destination: main results figure/table. - -## What Can Still Improve - -1. Finish `claim_expanded`. - Evidence gate: a policy that dominates candidate 462 on return at no worse - Markov cap, or a cleaner/fatter safe ridge around the same cap. - -2. Generate a manuscript-ready frontier artifact. - Needed columns: candidate role, family, tau, gamma, aversion, return, - Gamma_CP, V, endpoint budget, Markov cap, funded loans, alpha pass count. - -3. Update manuscript numbers only after promotion decision. - The current manuscript still reports 170.5K, V 0.028875, Gamma_CP 0.187987, - and 45/45. These are now weaker than the pool93 stage1 frontier for the - return-bound claim. - -4. Replace "champion" language in paper-facing sections. - Use "selected policy", "body point", "economic endpoint", "declared return - floor", and "finite-grid certificate". - -5. Add a selection-protocol paragraph. - Explain that CRPTO promotes a body point by declared return-bound criteria, - while max-return and bound-tight points are appendix/frontier endpoints. - -6. Keep end-to-end learning and stronger CRC methods as future work. - They are relevant but would change the protocol. Do not reopen the paper - around them unless a new theorem or new prospective evaluation is added. - -## Scenario Ladder - -### Minimum Publishable Improvement - -Stage1 candidate 462 remains body default. The paper updates its claim from -45/45 local champion to 815-policy finite-grid return-bound frontier. This is -already stronger and more IJDS-facing than the current manuscript. - -### Strong Improvement - -`claim_expanded` finds a point with return >= 183.8K and Markov cap <= 0.3607, -or a larger clean region around cap <= 0.37. The paper can then state a more -defensible "highest return under declared Markov-cap lens" result. - -### Ideal Scenario - -The final promoted pool93 point satisfies: - -- return >= 190K; -- Markov cap <= 0.3607; -- Gamma_CP <= 0.1763; -- V <= 0.0413 or still clearly below sqrt(0.01); -- 8/8 alpha pass; -- belongs to a broad interpretable bound-refined region with no alpha01 - failures. - -This would give the strongest IJDS claim: - -> A predeclared return-bound selection rule identifies a high-return policy on -> an exact full-universe conformal-robust frontier, while a surrounding -> finite-grid surface verifies that the decision is not an isolated point. - -### Unrealistic Or Unsafe Scenario - -Trying to claim: - -- global optimality over continuous policy space; -- nominal 99 percent funded-set coverage; -- universal conditional coverage; -- fairness certification; -- live sequential validity; -- superiority over CRC/CROMS/CREDO as theory. - -These would invite reviewer rejection because the evidence does not support -them. - -## Recommended Paper Rewrite Direction - -In the abstract and introduction, replace: - -> promoted economic policy earns 170.5K ... final region 45/45 - -with: - -> a selected return-bound policy from a predeclared finite-grid exact audit -> realizes 183.8K on a 1M budget, passes all eight alpha levels in the declared -> grid, and sits on a full-universe return-bound frontier whose bound-tight and -> economic endpoints are reported separately. - -In the theory section, keep Markov but update the numeric example to candidate -462 or the final claim-expanded winner. - -In the supplement, preserve old champion history only as archived/internal -decision provenance if needed. The submitted paper should not explain a -succession of champions. - -## Operational Next Steps - -1. Let `claim_expanded` finish or reach the bound-refined families before final - promotion. -2. Build a `pool93_ijds_frontier_table` from the final selected run. -3. Update `paper/CRPTO_ijds.qmd`, `paper/supplement_ijds.qmd`, and - `paper/submission/CLAIM_AUDIT_MATRIX.md` using the new neutral vocabulary. -4. Render/validate the paper after replacing old numbers. -5. Archive old champion-comparison language in internal docs only. - -## Final Micro-Extension And Consolidated Frontier - 2026-06-28 19:35 America/Bogota - -The surgical `claim_micro_ext` refinement closed successfully and should be -treated as the strongest completed pool93 evidence set before the final -bound-only closure: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- profile: `claim_micro_ext` -- exact policies: 4,407 -- exact alpha checks: 35,256 -- all-alpha passers: 4,406/4,407 -- all-alpha passers above the declared return floor: 4,406/4,407 -- only failure: one high-return endpoint policy at the strictest alpha gate; the - body and bound-facing neighborhoods remain clean. - -Final `claim_micro_ext` governance roles: - -| Role | Candidate | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|---:|---:| -| Minimum Markov-cap endpoint | 856/857/861 equivalent | 178,639.843939 | 0.136322 | 0.035875 | 0.316713 | 8/8 | -| Body/default balanced point | 131 | 184,832.475845 | 0.162616 | 0.035350 | 0.345084 | 8/8 | -| Strict cap<=0.345 body proxy | 511/512/513 equivalent | 184,800.413581 | 0.162562 | 0.035350 | 0.344996 | 8/8 | -| Highest return under cap<=0.36 | 3211/3212 equivalent | 186,050.727749 | 0.174600 | 0.037750 | 0.358685 | 8/8 | -| Max-return economic endpoint | 4041/4042/4043 equivalent | 223,458.135875 | 0.457438 | 0.069575 | 0.510753 | 8/8 | - -Interpretation: - -- Candidate 131 is still the best protocol-selected body point because it - maximizes the balanced return-bound score and shares the lowest observed - realized V among return-floor-safe policies. -- Candidate 511/512/513 is slightly cleaner for manuscript language if the - paper wants a strict `Markov cap <= 0.345` sentence. It gives up only about - 32 dollars of realized return relative to candidate 131. -- Candidate 4041 is a useful economic endpoint, not the paper body point. It - shows the return-bound tradeoff, but its Markov cap is much looser. - -I built a consolidated frontier sidecar across the completed `expanded`, -`claim_micro`, and `claim_micro_ext` runs: - -`models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated/portfolio/pool93_ijds_consolidated_frontier.json` - -Consolidated counts: - -- raw evaluated rows: 10,614 -- deduplicated semantic policies: 9,870 -- duplicate semantic rows removed: 744 -- eligible all-alpha, above-return-floor policies: 9,682 -- nonpass or below-floor policies: 188 - -The consolidated table is the current best paper-facing artifact because it -does not overfit to whichever local run happened to produce a role. It preserves -the source run for every selected point and uses the rule: - -> eligible = all-alpha pass and nonnegative return-floor surplus; dedupe by -> semantic policy; body score = 0.40 return + 0.40 inverse Markov cap + -> 0.20 inverse V; cap-frontier rows maximize return under fixed Markov-cap -> thresholds. - -The consolidated frontier selected: - -| Role | Source | Candidate | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---|---:|---:|---:|---:|---:|---:| -| Minimum Markov-cap endpoint | micro_ext | 861 | 178,639.843939 | 0.136322 | 0.035875 | 0.316713 | 8/8 | -| Body/default balanced point | micro_ext | 131 | 184,832.475845 | 0.162616 | 0.035350 | 0.345084 | 8/8 | -| Strict cap<=0.345 point | micro_ext | 512 | 184,800.413581 | 0.162562 | 0.035350 | 0.344996 | 8/8 | -| Highest return under cap<=0.36 | micro_ext | 3212 | 186,050.727749 | 0.174600 | 0.037750 | 0.358685 | 8/8 | -| Highest return under cap<=0.45 | expanded | 979 | 198,693.277519 | 0.252323 | 0.045600 | 0.449010 | 8/8 | -| Highest return under cap<=0.50 | micro | 2840 | 222,558.702500 | 0.459089 | 0.071075 | 0.487795 | 8/8 | -| Max-return economic endpoint | micro_ext | 4041 | 223,458.135875 | 0.457438 | 0.069575 | 0.510753 | 8/8 | - -Paper-facing decision: - -- Promote the selected policy as a finite-grid return-bound point, not as a - historical champion replacement story. -- In the paper body, use either candidate 131 for the protocol-selected body - point or candidate 512 if the prose needs the cleaner strict cap threshold. - The table can show both roles without making them compete. -- Replace `45/45 robust region` with `finite policy-grid robustness surface`. - The completed pool93 surface is much stronger: 9,682 eligible deduplicated - policies across the consolidated refinements, with explicit alpha-grid - semantics and no continuous-region overclaim. -- Keep the theorem claim at the Markov/weighted-funded-validity level. The - exact full-universe audit is deterministic accounting plus finite-grid - evidence, not a new conditional-coverage theorem. - -Best manuscript claim after completed evidence: - -> CRPTO maps a frozen calibrated credit-risk artifact into conformal robust -> funding decisions and reports an exact full-universe finite-grid -> return-bound frontier. On the Lending Club OOT universe, the selected pool93 -> body point realizes about 184.8K on a 1M budget, passes all eight predeclared -> alpha levels, and sits on a frontier whose bound-tight and economic endpoints -> are reported separately rather than collapsed into a single leaderboard -> winner. - -Use this as manuscript logic, not verbatim final prose. - -### Final Bound-Closure Search - -The only additional search that is still scientifically justified is a -bound-only closure, because `claim_micro_ext` showed a monotone reduction in -Markov cap as gamma increased up to 0.65 while uncertainty aversion was already -mostly saturated. This can change the appendix/frontier endpoint and the -theory-facing bound language, but it is not expected to change the body/default -policy. - -Launched run: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure` -- profile: `claim_bound_closure` -- exact policies: 1,653 -- exact alpha checks: 13,224 -- workers: 12 -- evidence gate: Markov cap below 0.316713, nonnegative return-floor surplus, - and 8/8 alpha pass. -- artifact sink: appendix/frontier endpoint and consolidated frontier sidecar, - not paper-body policy unless it unexpectedly dominates the body point. - -Postprocessing is automated. When the run finishes, it will generate the run -governance/frontier sidecars and a four-run consolidated frontier: - -`models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-with-bound-closure/portfolio/pool93_ijds_consolidated_frontier.json` - -Stop rule: - -- If the minimum Markov cap moves materially below 0.316713 while keeping - 8/8 and return above the declared floor, promote it as a stronger - bound-tight endpoint. -- If the cap improvement is tiny or the endpoint loses the return floor, close - the search as negative evidence and keep the completed consolidated frontier - above as the paper-facing result set. - -## Bound-Closure Final And Last Floor-Threshold Check - 2026-06-28 23:20 America/Bogota - -The `claim_bound_closure` run completed and materially strengthened the -bound-tight endpoint: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure` -- exact policies: 1,653 -- exact alpha checks: 13,224 -- all-alpha passers: 1,653/1,653 -- all-alpha passers above declared return floor: 1,653/1,653 -- elapsed wall time: about 3.23 hours with 12 workers - -Best bound-tight endpoint from this closure: - -- local candidate: 166/167/168 equivalent -- family: `claim_bound_closure_low_cap` -- risk tolerance: 0.1685 -- gamma: 0.75 -- uncertainty aversion: 0.35 -- realized return: 174,136.767088 -- return-floor surplus: 3,672.227088 -- Gamma_CP: 0.119477 -- weighted miscoverage V: 0.035875 -- endpoint budget upper: 0.19836925 -- Markov cap: 0.29836925 -- alpha pass count: 8/8 - -This improves the previous minimum-cap endpoint from the consolidated -`micro_ext` surface: - -| Endpoint | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|---:| -| Before bound closure | 178,639.843939 | 0.136322 | 0.035875 | 0.316713 | 8/8 | -| After bound closure | 174,136.767088 | 0.119477 | 0.035875 | 0.298369 | 8/8 | - -The price of tightening from the body/default point is now explicit: - -| Role | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|---:| -| Body/default balanced point | 184,832.475845 | 0.162616 | 0.035350 | 0.345084 | 8/8 | -| Bound-tight endpoint | 174,136.767088 | 0.119477 | 0.035875 | 0.298369 | 8/8 | -| Economic endpoint | 223,458.135875 | 0.457438 | 0.069575 | 0.510753 | 8/8 | - -Interpretation: - -- The body/default point remains candidate 131 from `micro_ext`. The - bound-closure result is not a better body policy; it is a sharper endpoint - for the return-bound frontier. -- The paper can now state a cleaner frontier story: one selected body point - around 184.8K, one bound-tight endpoint below Markov cap 0.30, and one - economic endpoint above 223K. -- This is stronger than the former `45/45` language because it gives a - multi-point exact frontier with explicit denominators and tradeoffs. - -The consolidated-with-bound-closure sidecar was generated at: - -`models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-with-bound-closure/portfolio/pool93_ijds_consolidated_frontier.json` - -Consolidated counts after adding the bound closure: - -- raw evaluated rows: 12,267 -- deduplicated semantic policies: 11,472 -- eligible all-alpha, above-return-floor policies: 11,284 -- nonpass or below-floor policies: 188 - -### Last Floor-Threshold Check - -The minimum-cap policy still lies on the boundary of the closure grid -(`tau=0.1685`, `gamma=0.75`, `aversion=0.35`) and retains a positive surplus -of about 3.7K over the declared return floor. Therefore, one last bounded check -is justified only for the endpoint claim: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure` -- profile: `claim_bound_floor_closure` -- exact policies: 2,343 -- exact alpha checks: 18,744 -- target: test whether the bound-tight endpoint can cross `Markov cap < 0.29` - while preserving 8/8 alpha pass and nonnegative return-floor surplus. -- artifact sink: appendix/frontier endpoint and final consolidated frontier - only; it should not replace the paper body/default point. - -Early checkpoint: - -- first completed endpoint already improved Markov cap from 0.298369 to - 0.297442 at return 173,851.453513, Gamma_CP 0.118766, V 0.035875, 8/8. -- Higher-gamma candidates are still pending, so the final endpoint is not yet - claim-final. - -Stop rule for the whole pool93 portfolio-claim search: - -- If `claim_bound_floor_closure` crosses cap 0.29 with return above the - declared floor, promote that as the final bound-tight endpoint. -- If it does not cross cap 0.29, keep cap 0.298369 as the final bound-tight - endpoint and close the search. A further wave would be threshold chasing - rather than a new manuscript claim. - -## Terminal Bound Search Design - 2026-06-29 03:10 America/Bogota - -At 87 percent completion, `claim_bound_floor_closure` had already crossed the -intended paper threshold: - -- best observed Markov cap: 0.284839 -- return: 171,399.674021 -- return-floor surplus: 935.134021 -- Gamma_CP: 0.106806 -- V: 0.034875 -- alpha pass: 8/8 - -This is already sufficient for the paper phrase "bound-tight endpoint below -0.29 Markov cap." However, the best endpoint still sits on the high-gamma edge -of the current grid (`gamma=0.84`) and retains a small positive return surplus. -One final terminal search is therefore justified, but only as an endpoint -closure. It is not a body-policy search and it must be the last search wave. - -Terminal run: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- profile: `claim_bound_terminal` -- exact policies: 37,068 -- exact alpha checks: 296,544 -- workers: 12 -- launch mode: waits for - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure` to - complete, then starts automatically. -- postprocessing: governance, frontier table, and consolidated definitive - frontier generated automatically. - -Search blocks: - -| Block | Purpose | Grid | -|---|---|---| -| `claim_bound_terminal_ultra_low_cap` | Test whether cap can cross cleaner thresholds such as 0.280 or 0.275 while staying above the return floor. | tau 0.16675--0.17025, gamma 0.84--0.99, aversion 0.40--0.70 | -| `claim_bound_terminal_return_recovery` | Preserve the best return under low-cap thresholds instead of only minimizing cap. | tau 0.16800--0.17150, gamma 0.80--0.92, aversion 0.35--0.60 | - -Evidence gates: - -- primary: minimum Markov cap with 8/8 alpha pass and nonnegative - return-floor surplus; -- secondary: highest return under cap thresholds 0.275, 0.280, 0.285, 0.290, - 0.295, and 0.300; -- tertiary: verify that the selected body/default point from `micro_ext` - remains unchanged. - -Absolute stop rule: - -- After `claim_bound_terminal` completes, stop portfolio-claim searching. -- If it improves only the bound endpoint, promote only the endpoint in the - frontier table. -- If it fails to improve the current 0.284839 endpoint, close it as negative - evidence and retain `claim_bound_floor_closure`. -- Do not launch another wave merely because the minimum-cap point lands on a - new grid boundary. At that point, further reduction would be threshold - chasing and would not change the manuscript's main claim. - -Definitive artifact target: - -`models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json` - -## Final Terminal Closure - 2026-07-02 America/Bogota - -`claim_bound_terminal` completed and the postprocessors generated the -governance, frontier-table, and consolidated-definitive sidecars. This closes -the pool93 portfolio-claim search under the declared stop rule. - -Final terminal run: - -- run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- exact policies: 37,068 -- exact alpha checks: 296,544 -- all-alpha passers: 37,068/37,068 -- all-alpha passers above the declared return floor: 14,814/37,068 -- alpha grid: `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}` -- final status artifact: - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/runtime_status.json` - -Definitive consolidated frontier: - -- raw evaluated rows: 51,678 -- deduplicated semantic policies: 50,010 -- duplicate rows removed: 1,668 -- eligible all-alpha and above-floor policies: 27,508 -- consolidated sidecar: - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json` -- manuscript/supplement table: - `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` - -Final claim roles: - -| Role | Source | Candidate | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---|---:|---:|---:|---:|---:|:---:| -| Minimum Markov-cap endpoint above floor | terminal | 10661 | 170,467.268819 | 0.095719 | 0.031875 | 0.273035950 | 8/8 | -| Low-cap balanced endpoint | terminal | 5504 | 171,006.195983 | 0.097190 | 0.031875 | 0.274789250 | 8/8 | -| Highest return under cap <= 0.30 | terminal | 36412 | 173,314.040806 | 0.115400 | 0.035875 | 0.294580000 | 8/8 | -| Strict cap <= 0.345 body proxy | micro_ext | 512 | 184,800.413581 | 0.162562 | 0.035350 | 0.344996495 | 8/8 | -| Body/default balanced point | micro_ext | 131 | 184,832.475845 | 0.162616 | 0.035350 | 0.345083740 | 8/8 | -| Highest return under cap <= 0.36 | micro_ext | 3212 | 186,050.727749 | 0.174600 | 0.037750 | 0.358685000 | 8/8 | -| Highest return under cap <= 0.45 | expanded | 979 | 198,693.277519 | 0.252323 | 0.045600 | 0.449009950 | 8/8 | -| Max-return economic endpoint | micro_ext | 4041 | 223,458.135875 | 0.457438 | 0.069575 | 0.510753090 | 8/8 | - -Interpretation: - -- The terminal search materially improves the bound endpoint, not the body - point. It lowers the above-floor Markov-cap endpoint to 0.2730 while keeping - a nonnegative return-floor surplus. -- The body/default point remains micro_ext candidate 131 because it is a more - useful manuscript tradeoff: materially higher return, all-alpha pass, zero - violation, and a clean Markov cap around 0.345. -- Candidate 512 is the strict wording alternative if the paper wants the exact - phrase "Markov cap <= 0.345"; it gives up only about 32 dollars versus the - body/default point. -- Candidate 4041 is the economic endpoint and should be shown as a frontier - endpoint, not as the primary theorem-facing policy. - -Manuscript claim hierarchy: - -1. Promote in the body: a finite-grid exact return-bound frontier on the - Lending Club OOT universe, with the body/default point earning 184.8K on a - 1M budget and passing all eight declared alpha levels. -2. Promote in the body as the conservative endpoint: the frontier also contains - an above-floor policy with Markov cap 0.2730. -3. Append in the supplement: the full cap frontier, including 0.30, 0.345, - 0.36, 0.45, 0.50, and max-return endpoints. -4. Closed on 2026-07-02 for the submitted pool93 claim: grade composition - (A36), tail-risk repricing (A37), concentration/cluster-bound sensitivity - (A38), and fixed-allocation bootstrap metrics (A39) now come from the - promoted pool93 allocation. A39 bootstraps funded-loan contributions only; it - does not resample solver inputs, the PD model, calibration data, conformal - intervals, or policy search. - -Final stop decision: - -- Close portfolio-claim searching for pool93. -- Do not launch another threshold chase solely because the minimum-cap policy - lies near a new grid boundary. -- Reopen only if a reviewer requests a denser alpha grid, row-level pool93 - composition/tail-risk artifacts, or a formal continuous-region theorem. diff --git a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md deleted file mode 100644 index 0333702..0000000 --- a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md +++ /dev/null @@ -1,643 +0,0 @@ -# IJDS Corpus, Claims, and Improvement Plan - 2026-07-07 - -> **Superseded metric vocabulary (2026-07-09).** This memo is retained as the -> corpus-reading and editorial decision trail. Its old "Markov cap" and -> preliminary frontier language must not be used as current evidence. The -> policy-aware A35 correction and matched A40 baseline in -> `pool93_certificate_semantics_v2_2026-07-09.md` are authoritative. - -Scope: analyze the current CRPTO body paper, official IJDS submission PDF/source, -online supplement, frozen metrics/artifacts, and the local `Papers_tesis` corpus -using the global `academic-pdf-intake` skill outputs. - -This memo does **not** reopen the champion, does **not** modify -`EXTRACTION_MANIFEST.json`, and does **not** recommend rerunning protected DVC -stages. The active paper claim remains the finite-grid decision certificate -registered in `docs/research/active_claims_2026-07-04.md`. - -## Implementation Status - -Status on 2026-07-07: the actionable P0/P1/P2 editorial recommendations in this -memo have been applied to the manuscript, official IJDS `.tex` handoff, -supplement, submission checklist, bibliography, and reviewer-defense matrix. -This file is retained as traceability for the corpus/IJDS analysis and parser -evidence, not as an open TODO list. Future edits should use -`paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` and -`docs/research/active_claims_2026-07-04.md` as the active operating gates. - -## Inputs Used - -Skill / benchmark outputs: - -- Full local inventory: - `.tmp_pdf_intake_benchmark/run_20260707_1715/manifest.jsonl` -- Full parser benchmark: - `.tmp_pdf_intake_benchmark/run_20260707_1715/runs.jsonl` -- Current literature matrix generated from extracted text: - `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` -- IJDS venue snippets: - `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/ijds_venue_snippets.csv` -- Parser summary for active CRPTO PDFs: - `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/active_crpto_parser_benchmark.csv` -- MinerU CUDA follow-up for active CRPTO PDFs: - `.tmp_pdf_intake_benchmark/run_20260707_active_mineru_cuda/runs.jsonl` - -CRPTO sources: - -- `paper/CRPTO_ijds.qmd` -- `paper/submission/CRPTO_ijds_submission.tex` -- `paper/submission/CRPTO_ijds_submission.pdf` -- `paper/supplement_ijds.qmd` -- `paper/CRPTO_ijds.pdf` -- `paper/supplement_ijds.pdf` -- `reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv` -- `reports/crpto/tables/crpto_tableA25_external_replication_gate.csv` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` -- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `models/experiments/champion_reopen/.../pool93_ijds_claim_governance.json` -- `models/experiments/champion_reopen/.../pool93_ijds_consolidated_governance.json` - -IJDS public sources checked on 2026-07-07: - -- Submission guidelines: - -- Reviewer guidelines: - -- Editorial statement: - -- Data and Code Disclosure Policy: - - -## Executive Diagnosis - -CRPTO is well matched to IJDS when written as **data science for decisions**: -real credit data, a methodological bridge from calibrated prediction to robust -optimization, managerial/model-risk relevance, practical implications, and -reproducible computational evidence. The strongest submission story is not -"better credit scoring." It is: - -> CRPTO converts a frozen calibrated PD model into an auditable robust funding -> decision by carrying conformal uncertainty into a finite-grid portfolio -> frontier, exposing the return-bound trade-off and validating the selected -> funded set from frozen evidence. - -The paper is already close to this framing. The main improvement opportunity is -reader focus: reduce the cognitive load around internal run history, make the -closest IJDS precedents visible, and make the A35 frontier the unmistakable -decision object. - -The main risk is overclaiming by shorthand. In particular, the current body and -submission use "0.345 Markov cap" as a rounded lens. The active body/default -point has `Markov_cap = 0.345083740`; the strict `<= 0.345` frontier row is a -neighboring policy with return `$184,800.41`, not the body/default return -`$184,832.48`. This is fixable in prose by saying "approximately 0.3451" or -"the declared 0.345 return-bound lens" and by avoiding exact `<= 0.345` language -for the body/default row. - -## Official IJDS Fit - -The IJDS submission guidelines say IJDS publishes data science methodologies -for decision-making environments and expects four components: data, innovative -model/algorithm/approach, managerial/engineering/industrial relevance, and -implications. CRPTO has all four: - -| IJDS component | CRPTO evidence | Current strength | Improvement | -|---|---|---|---| -| Data | Lending Club OOT panel; Prosper and Freddie/Mendeley external frozen applications. | Strong. | Keep external results as recipe transfer, not as extra certificates. | -| Model / algorithm | Calibrated PD -> Mondrian conformal intervals -> robust LP -> exact funded-set audit. | Strong. | Name the methodological unit earlier as a "decision certificate." | -| Decision relevance | `$1M` budgeted credit funding with return/risk trade-off. | Strong. | Make A35/A19 the reader-facing proof of relevance. | -| Implications | Reproducible model-risk audit surface; explicit limits on live/conditional claims. | Strong. | Put implications in abstract/conclusion as knowledge gained, not only actions performed. | - -Venue constraints that matter: - -- Initial submissions should be at most 25 IJDS-style pages excluding references - and appendices. -- IJDS uses double-anonymous review for submissions on/after 2025-01-01. -- Abstract must be one paragraph and <=300 words; IJDS strongly encourages it - to answer problem/relevance, methods/results, and insights/implications. -- Tables/figures should appear near first mention. -- Data/code disclosure is required at submission, and accepted computational - papers are expected to upload data/code or an approved alternative plan. - -The current official-template build is 26 pages total, with conclusion and -references starting on page 22, so the body appears inside the IJDS page budget -when references are excluded. The reproducibility package plan is aligned with -the IJDS policy, but review-stage path/identity sanitization remains essential. - -## Current CRPTO Claim Stack - -Active body point: - -| Quantity | Current value | -|---|---:| -| OOT Lending Club universe | `276,869` loans | -| Budget | `$1,000,000` | -| Selected-policy realized return | `$184,832.48` | -| Return-floor surplus | `$14,367.94` | -| `V(alpha=0.01)` | `0.035350` | -| `Gamma_CP(alpha=0.01)` | `0.162616` | -| Endpoint budget upper | `0.245083740` | -| Markov cap | `0.345083740` | -| Exact alpha violation | `0.0` | -| Alpha-grid pass | `8/8` | -| Bootstrap return interval | `$167,963.20` to `$198,650.47` | - -Finite-grid denominators: - -| Surface | Denominator | Result | -|---|---:|---| -| Consolidated semantic policies | `50,010` | `27,508` pass every declared alpha and exceed return floor | -| Raw consolidated rows | `51,678` | `1,668` duplicate semantic rows removed | -| Terminal endpoint policies | `37,068` | `37,068/37,068` all-alpha passers | -| Terminal alpha checks | `296,544` | `296,544/296,544` completed checks | - -Main claim boundary: - -- Claim **finite-grid decision certificate**, not continuous global optimality. -- Claim **weighted funded-set validity plus Markov bound**, not universal - conditional coverage. -- Claim **frozen external economic recipe transfer**, not Prosper/Freddie exact - Lending Club-style certificates. -- Claim **regret-auditability trade-off**, not dominance over SPO+ or other - decision-focused learners on every regret metric. -- Claim **fixed-allocation bootstrap diagnostics**, not a conformal/bootstrap - guarantee over model retraining, solver inputs, or search. - -## Paper / Submission / Supplement Assessment - -### Body and official submission - -Strengths: - -- The abstract already starts with the correct IJDS premise: credit allocation is - a data-science-for-decisions problem. -- The introduction explicitly says the research question is not a better - classifier, but whether predictive uncertainty can be carried into a robust - auditable portfolio decision. -- The contribution list separates construction, theorem, literature positioning, - frozen evidence, and evidence ladder. -- The official submission uses the IJDS `informs4` class with `dblanonrev`, and - the build instructions now document both `latexmk` and the robust - `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback. -- The body has the right core tables: certificate metrics, finite-grid frontier, - reviewer-question boundary table, regret-auditability table, and supplement - map. - -Risks / repairs: - -- **Cap wording repair:** replace exact-sounding "under the declared 0.345 - Markov cap" for the body/default point with "near the declared 0.345 - return-bound lens" or "Markov cap 0.345084." Keep the strict `<=0.345` row as - a neighboring frontier endpoint, not the body point. -- **A35 should be the first durable result object.** The paper currently explains - many pieces before the reader sees the frontier. A reviewer should encounter - the frontier logic as soon as possible after the certificate table. -- **IJDS precedent paragraph is thin.** The body cites relevant work, but the - venue-specific story can be sharper: IJDS has already published credit ML, - cost-aware calibration, causal decision framing, and reproducibility-oriented - data-science-for-decisions papers. CRPTO extends that line into a portfolio - decision certificate. -- **External replications are valuable but slightly loud in the abstract.** They - support transfer of the recipe; they should not compete with the Lending Club - certificate as the abstract's main result. -- **SPO+ comparison is strong but needs a crisp takeaway sentence.** The current - text is accurate: SPO+ wins synthetic regret; CRPTO buys auditable risk - controls and a dollar-funded set. Put that exact contrast in one memorable - sentence before the table. - -### Supplement - -Strengths: - -- A35--A39 are exactly the right selected-policy closure block. -- A37--A39 correctly close tail, concentration, and empirical contribution - objections without changing the selector. -- A25--A34 answer the single-dataset concern through external economic recipe - transfer and exhaustiveness checks. -- Appendix E makes routine reproduction distinct from protected champion/search - reruns. - -Risks / repairs: - -- The supplement is rich enough that a reader may miss the hierarchy. Add a - short "how to read this supplement" map at the top that says: theory -> active - certificate -> diagnostics -> external transfer -> reproduction. -- A35 should be introduced as the supplement's active frontier, not as another - appendix table among many. -- A25--A34 should keep the words "economic replication" and "recipe transfer" - visible. Avoid "external validation" unless it is immediately scoped. -- In A38, the fact that every cluster-aware threshold is looser than Markov is a - strength: it explains why the body does not chase a more fragile bound. - -## Parser / Skill Findings - -The `academic-pdf-intake` skill routing is sensible for this repo: - -- **Docling** remains the primary parser for born-digital academic PDFs and - clean Markdown/JSON extraction. -- **OpenDataLoader hybrid** is the best comparison/fallback when bounding boxes, - reading-order traceability, hidden-text safety, and table provenance matter. -- **MinerU CUDA hybrid-engine** is now viable on the local RTX/CUDA setup for - the active CRPTO PDFs and is best kept for formula-heavy, OCR-heavy, scanned, - or visual-QA cases. -- **Codex PDF / MarkItDown** are useful as fast baselines and smoke tests, not - as the final source for complex academic extraction. - -Active PDF benchmark: - -| PDF | Pages | Docling | OpenDataLoader | MinerU CUDA | -|---|---:|---:|---:|---:| -| `paper/CRPTO_ijds.pdf` | 27 | 54.98s | 57.09s | 60.41s | -| `paper/submission/CRPTO_ijds_submission.pdf` | 26 | 46.14s | 35.00s | 54.47s | -| `paper/supplement_ijds.pdf` | 32 | 78.41s | 70.30s | 74.20s | - -Operational recommendation: - -- For day-to-day paper analysis: Docling first, ODL for table/traceability - comparison, Codex PDF for fast diffable text. -- For final figure/table/formula QA: add MinerU on the active PDF(s), then check - `layout.pdf` / visual outputs when extraction disagreement is material. -- For the full 81-PDF literature corpus: do not run all three heavy parsers - routinely. Use fast baseline + Docling on close papers, then ODL/MinerU only - on candidates with tables, equations, or layout ambiguity. - -## `Papers_tesis` Corpus Summary - -Current corpus inventory: - -- `81` local literature PDFs in `Papers_tesis` -- `3,404` literature pages -- `3` active CRPTO PDFs separately, `85` pages -- Full generated matrix: - `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` - -Topic signals from extracted text: - -| Topic signal | PDFs | -|---|---:| -| Tables / metrics / experiments | 78 | -| Tail risk / CVaR / OCE / loss | 76 | -| Portfolio / optimization / decision | 68 | -| Robust optimization / uncertainty sets | 67 | -| Conformal / coverage / calibration | 57 | -| Fairness / governance / explainability | 57 | -| Source shift / weighted / multi-source | 55 | -| Credit / lending / default | 38 | -| Causal decision | 24 | -| Decision-focused / SPO / PTO | 22 | -| Online / adaptive conformal | 20 | - -These counts are keyword/topic signals, not claims that every paper is equally -central. The editorial use is by cluster. - -## IJDS Papers in or Adjacent to the Local Corpus - -Confirmed IJDS papers with local or official evidence: - -| Paper | Evidence | What it contributes | CRPTO use | Boundary | -|---|---|---|---|---| -| Das et al. (2023), "Credit Risk Modeling with Graph Machine Learning" | Local PDF header and DOI `10.1287/ijds.2022.00018`; official IJDS page. | Extends tabular credit scoring with corporate graph features and GNN/AutoML ensembles; includes reproducibility capsule. | Shows IJDS accepts credit-risk ML when data/method/reproducibility are clear. Position CRPTO as a **decision certificate after scoring**, not a richer scorer. | Corporate credit ratings, not consumer loan portfolio funding; no conformal/robust funded-set certificate. | -| Yang and Bi (2025), "Cost-Aware Calibration of Classifiers" | Official IJDS DOI `10.1287/ijds.2024.0038`; cited in CRPTO `.bbl`. | Defines cost-aware calibration, expected calibration cost, and MetaCal; emphasizes downstream costs of miscalibration. | Strongest IJDS calibration precedent. CRPTO extends calibration into **portfolio allocation and funded-set audit**. | Classifier calibration problem, not robust portfolio optimization. | -| Fernandez-Loria and Provost (2022), "Causal Decision Making and Causal Effect Estimation Are Not the Same..." | Official IJDS DOI `10.1287/ijds.2021.0006`; cited in CRPTO `.bbl`. | Separates decision quality from effect-estimation accuracy. | Use to sharpen the intro: CRPTO is a decision object, not a prediction leaderboard. | Causal treatment assignment framing, not credit PD/conformal portfolio. | -| Fernandez-Loria and Provost (2025), "Observational vs. Experimental Data When Making Automated Decisions Using Machine Learning" | Local PDF in supplement; official DOI `10.1287/ijds.2023.0012`. | Shows observational data can sometimes support automated decisions when the decision target is ranking/thresholding rather than unbiased effect estimation. | Supports CRPTO's observational-panel boundary and the claim that decision metrics differ from estimation metrics. | Causal/automated intervention setting; CRPTO should cite it as a limitation/future protocol, not as causal validity. | -| Falconer, Kazempour, and Pinson (2026), "Toward Replication-Robust Analytics Markets" | Local PDF header and DOI `10.1287/ijds.2025.0075`; official IJDS page. | Builds an analytics market robust to strategic data replication; emphasizes reproducibility and strategic robustness. | Useful venue signal: IJDS values robust/reproducible analytics systems. Use only as a light reproducibility/robustness cousin. | Market design/collaborative analytics, not credit, conformal prediction, or portfolio funding. | - -The local corpus also references Morucci et al. (2022), an IJDS causal -uncertainty paper, but the PDF is not in `Papers_tesis`; do not count it as -local corpus evidence unless it is added. - -## Closest Non-IJDS Literature Clusters - -### 1. Conformal foundations and risk control - -Key local papers: - -- Angelopoulos and Bates (2023), gentle introduction. -- Angelopoulos et al. (2024), conformal risk control. -- Angelopoulos et al. (2025), Learn Then Test. -- Bates et al. (2021), risk-controlling prediction sets. -- Barber et al. (2021), conditional coverage limits. -- Angelopoulos et al. (2026), non-monotonic CRC. -- Gibbs/Candes, Lekeufack et al., Kiyani et al., Zhou/Orfanoudaki/Zhu. - -What they give CRPTO: - -- Validity language and finite-sample discipline. -- Justification for risk-control framing. -- Limits on conditional/group/live claims. - -How to improve paper: - -- Keep them as theory lineage, but do not over-expand. -- Use them to justify why CRPTO reports a bound and exact audit rather than only - nominal coverage. - -### 2. Conformal robust optimization / predict-then-calibrate - -Key local papers: - -- Johnstone and Cox (2021), conformal uncertainty sets for robust optimization. -- Patel et al. (2024), conformal contextual robust optimization. -- Sun et al. (2024), predict-then-calibrate. -- Zhao et al. (2026), conformal robust optimization and satisficing. -- Bao et al. (2025), CROMS model selection. -- Yeh et al. (2025/2026), conformal risk training / end-to-end calibration. -- Zhou and Zhu (2025), inverse conformal risk control. - -What they give CRPTO: - -- The nearest methodological neighborhood. -- A natural "what CRPTO adds" contrast: real credit payoff, funded-set weights, - exact portfolio audit, finite frontier, and reproducibility harness. - -How to improve paper: - -- Add a compact contrast table: abstract CRO/LP papers vs. CRPTO's credit - funded-set certificate. -- Say explicitly that CRPTO is post-hoc over a frozen PD system; end-to-end - variants are future work. - -### 3. Decision-focused learning and SPO+ - -Key local papers: - -- Elmachtoub and Grigas (2022), SPO+. -- Donti et al. (2017), task-based end-to-end learning. -- Mandi et al. (2024), DFL survey. -- Liu and Grigas (2021), risk bounds/calibration for SPO. -- Schutte et al. (2024), robust losses for DFL. - -What they give CRPTO: - -- The main alternative methodological family. -- A strong reviewer question: "Why not train through the optimizer?" - -How to improve paper: - -- Keep A19/Figure 15 central. -- State the contrast in one sentence: - "SPO+ is the low-regret corner; CRPTO is the auditable-risk-control corner - with a funded-set dollar certificate." -- Do not apologize for higher synthetic regret; explain that the metric is - different from the funded-set economic certificate. - -### 4. Credit / P2P / fairness / governance - -Key local papers: - -- Jagtiani and Lemieux (2019), fintech Lending Club context. -- Serrano-Cinca and Gutierrez-Nieto (2016), profit scoring in P2P lending. -- Guo et al. (2016), instance-based P2P credit investment. -- Zhao et al. (2016), P2P portfolio selection. -- Chi, Ding, and Peng (2019), data-driven robust P2P credit portfolio. -- Das et al. (2023), IJDS graph ML credit risk. -- Albanesi and Vamossy (2024), score performance and equity. -- Fuster et al. (2022), unequal ML credit-market effects. -- Blattner and Nelson (2021), noisy data and consumer credit disparities. -- FinRegLab (2023), explainability and fairness in credit underwriting. -- CFPB (2014), proxy race/ethnicity methods. - -What they give CRPTO: - -- Domain legitimacy and governance boundaries. -- Support for reporting economic return, risk, calibration, and governance - together. - -How to improve paper: - -- Keep fairness/proxy material bounded. CRPTO does not have protected labels or a - legal fair-lending protocol. -- Use credit/P2P papers to motivate why classification metrics alone are - insufficient for investment decisions. - -### 5. Robust optimization, DRO, tail risk, and concentration - -Key local papers: - -- Bertsimas and Sim (2004), price of robustness. -- Ben-Tal, El Ghaoui, and Nemirovski (2009), robust optimization. -- Bertsimas, Gupta, and Kallus (2018), data-driven robust optimization. -- Bertsimas and Kallus (2020), predictive to prescriptive analytics. -- Delage and Ye (2010), moment DRO. -- Goldfarb and Iyengar (2003), robust portfolios. -- Rockafellar and Uryasev (2000), CVaR. -- Ben-Tal and Teboulle (2007), OCE. -- Hoeffding, Bennett, Freedman, Fuk-Nagaev for concentration context. - -What they give CRPTO: - -- The language for price of robustness and uncertainty budgets. -- Tail-risk diagnostics and the reason to keep Markov as the weakest defensible - body-level statement. - -How to improve paper: - -- A37/A38 should be discussed as "assumption-priced sensitivity." -- Avoid making CVaR/OCE sound like promoted selectors. - -### 6. Source shift, multi-distribution, online conformal - -Key local papers: - -- Tibshirani et al. (2019), conformal prediction under covariate shift. -- Barber, Candes, Ramdas, Tibshirani (2023), beyond exchangeability. -- Bhattacharyya and Barber (2026), group-weighted conformal prediction. -- Guan (2023), localized conformal prediction. -- Liu, Levis, Normand, Han (2024), multi-source conformal inference. -- Yang and Jin (2026), multi-distribution robust conformal prediction. -- Gibbs and Candes (2021), adaptive conformal inference. -- Liu et al. (2026), online conformal prediction via universal portfolios. - -What they give CRPTO: - -- A future-work lane and reviewer caveats for group/source/live deployment. - -How to improve paper: - -- Keep A23/A24 as diagnostics. -- Do not promote multi-distribution or online validity without a new protocol. - -## High-Priority Improvement Plan - -### P0: repair precision in cap wording - -Change any exact-sounding text that says the body/default point is under a -`0.345` cap. The exact body/default cap is `0.345083740`; the strict -`<=0.345` row is a neighboring frontier point. - -Recommended wording: - -- "the selected policy sits at the declared approximately 0.345 return-bound - lens, with Markov cap 0.345084" -- "the strict `<=0.345` endpoint earns `$184,800.41`; the body/default balanced - point earns `$184,832.48` with Markov cap `0.345084`" - -Avoid: - -- "highest-return point under cap `<=0.345`" for the body/default row. -- "declared `0.345` Markov cap" unless the next words clarify rounding. - -### P1: add a venue-specific IJDS precedent table - -Add a compact body table or paragraph after the related-work overview: - -| IJDS precedent | Lesson for CRPTO | CRPTO extension | -|---|---|---| -| Das et al. (2023) credit graph ML | IJDS accepts reproducible credit-risk ML. | CRPTO turns credit risk scores into a funded portfolio certificate. | -| Yang and Bi (2025) cost-aware calibration | Calibration matters because downstream costs are asymmetric. | CRPTO prices uncertainty inside a budgeted allocation. | -| Fernandez-Loria and Provost (2022/2025) decision vs estimation | Decision quality is not the same as estimation/prediction quality. | CRPTO evaluates the funded decision, not only PD quality. | -| Falconer et al. (2026) replication-robust analytics | IJDS values robust/reproducible analytics systems. | CRPTO supplies frozen evidence, exact checks, and a reproducibility harness. | - -This helps the editor see fit immediately and helps reviewers place the paper -inside IJDS rather than only OR/ML. - -### P1: make A35 the central result object - -Move the reader quickly from method to A35: - -1. Exact certificate table: what the selected policy achieved. -2. A35 frontier: why this is not a cherry-picked singleton. -3. A19 regret-auditability: why CRPTO is not trying to beat SPO+ on its own - synthetic regret metric. -4. A25 external recipe transfer: why the method is not a Lending Club-only - curiosity. - -The current paper has these pieces; the improvement is ordering and signposting. - -### P1: sharpen abstract to IJDS's three-question template - -Current abstract is good but can be more IJDS-aligned: - -1. Problem/relevance: - "Credit allocation decisions need calibrated probabilities only insofar as - they change funding choices under risk appetite." -2. Methods/results: - "CRPTO maps frozen PD predictions through Mondrian conformal intervals into a - robust LP and finite-grid funded-set audit; on 276,869 OOT loans it earns - `$184.8K` on `$1M` with `V=0.035350`, `Gamma_CP=0.162616`, and Markov cap - `0.345084`." -3. Insight/implication: - "The insight is that uncertainty should be reported as a return-bound - frontier, not as a post-hoc calibration table; reproducible decision - certificates can be audited without retraining a production-style PD model." - -Reduce abstract space devoted to external replications unless needed for -single-dataset defense. - -### P1: make the baseline story a reviewer checklist - -IJDS reviewers will ask whether the paper uses reasonable baselines and -quantifies improvement. CRPTO can answer with a table: - -| Baseline / family | What it optimizes | CRPTO comparison | -|---|---|---| -| Two-stage baseline | Predict then optimize without conformal robust certificate. | CRPTO adds exact funded-set bound and frontier. | -| SPO+ / DFL | Synthetic regret / task-aligned training loss. | SPO+ has lower mean regret; CRPTO has funded-set dollar value and 3/3 verifiable risk controls. | -| P2P profit scoring | Economic loan selection. | CRPTO adds conformal premium and exact alpha-safe audit. | -| P2P robust portfolio | Robust credit allocation. | CRPTO calibrates uncertainty with conformal intervals and exposes finite-grid denominators. | -| Cost-aware calibration | Probability calibration under asymmetric costs. | CRPTO carries calibrated uncertainty into a portfolio decision. | - -The goal is not to claim universal dominance; it is to make the trade-off -impossible to miss. - -### P2: strengthen supplement navigation - -Add a short supplement reader map: - -- Appendix A: proof and Markov boundary. -- Appendix B: robustness/challenger diagnostics. -- Appendix C: active A35--A39 selected-policy closure. -- Appendix D: external recipe transfer. -- Appendix E: reproducibility, DVC, protected stages, and anonymization. - -Then repeat the claim hierarchy in one table: - -| Evidence | Promoted? | Why | -|---|---:|---| -| A35 | Yes | Active finite-grid frontier. | -| A36--A39 | Support | Selected-policy composition/tail/concentration/bootstrap diagnostics. | -| A19 | Support | Regret-auditability contrast. | -| A25--A34 | Support | External economic recipe transfer. | -| A20--A24 | Diagnostics | Tail/source/online objections, not selector changes. | - -### P2: citation hygiene before freeze - -The previous citation audit flagged several body references as partial or -citation-only. This pass reduces risk for `das2023creditgraph`, -`yang2025costaware`, and the Fernandez-Loria/Provost IJDS papers. Still -spot-check before final freeze: - -- `hoeffding1963`, `boucheron2013concentration`, `ghosh2002` -- `goldfarb2003robustportfolio`, `delage2010dro` -- `serrano2016profitscoring`, `zhao2016p2pportfolio` -- any recent credit/IJDS references added after 2026-06-14 - -Do this as source verification, not as a broad literature expansion. - -### P2: prepare response-ready reviewer objections - -Prewrite one paragraph each: - -- "Why not SPO+?" -- "Why not CVaR/OCE as the selector?" -- "Is the result cherry-picked?" -- "What happens under dependence?" -- "Is this a live-production guarantee?" -- "What exactly can be reproduced under double anonymous review?" - -Most answers already exist in the body/supplement; the improvement is to make -them short and reusable. - -## Concrete Editing Checklist - -Before submission: - -1. Replace exact-sounding `0.345` cap wording for the body/default point. -2. Add IJDS precedent paragraph/table with Das, Yang/Bi, Fernandez-Loria/Provost, - and Falconer/Kazempour/Pinson. -3. Move or signpost A35 so the finite-grid frontier appears as the main results - object, not only as a supporting table. -4. Add one compact "what the reviewer should remember" paragraph before the - A19 regret table. -5. Tighten abstract external-replication language to "recipe transfer." -6. Add supplement reader map and promoted/support/diagnostic hierarchy. -7. Check all anonymous review paths: no repo URLs, local paths, author names, - affiliations, or identifying metadata in body/supplement PDFs. -8. Run `just smoke` and `just validate-champion` after any paper edits. -9. Rebuild official submission PDF and check page count/log. - -Do **not** do before submission unless a reviewer or explicit research decision -requires it: - -- Rerun protected champion/search/conformal stages. -- Promote CVaR/OCE, multi-distribution, online, causal, or end-to-end DFL - variants. -- Turn external Prosper/Freddie replications into new exact funded-set - certificates. -- Expand the body with a long literature review. - -## Best Current IJDS Story - -The paper should read like this: - -1. Credit allocation is a decision problem; PD calibration alone is insufficient. -2. Existing IJDS work shows calibration, credit ML, and causal decision framing - matter for data-science decisions. -3. CRPTO contributes the missing bridge: a frozen predictive model becomes a - robust funding decision with a conformal premium and exact finite-grid audit. -4. The selected policy earns `$184,832.48` on `$1M`, with `V=0.035350`, - `Gamma_CP=0.162616`, and Markov cap `0.345084`. -5. The result is not a singleton: A35 exposes 50,010 semantic policies and a - return-bound frontier. -6. The method does not beat SPO+ at SPO+'s own regret target; instead, it buys - auditability, risk controls, and a funded-set dollar certificate. -7. The supplement shows tail, concentration, bootstrap, source, online, and - external-recipe checks without changing the body claim. -8. The reproducibility package is part of the scientific contribution, not just - administration. - -That is the IJDS version of CRPTO: **a reproducible decision certificate for -credit portfolio allocation under conformal uncertainty**. diff --git a/docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md b/docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md new file mode 100644 index 0000000..ca25334 --- /dev/null +++ b/docs/research/ijds_credit_risk_controls_protocol_2026-07-13.md @@ -0,0 +1,64 @@ +# IJDS Credit-Risk Learner Controls - Locked V1 Protocol + +## Purpose + +This retrospective, previously inspected-archive audit asks whether the active +V4 candidate-coverage result depends on one learner, on unconstrained boosting, +or on LendingClub's own pricing and grade signals. It is a robustness control, +not a learner competition and not a new portfolio search. + +## Closed Model Family + +Five learners are co-primary and all results must be reported: + +1. the active CatBoost plus independent 2011 Platt calibration; +2. the active numeric-logistic coverage control plus its own Platt map; +3. CatBoost with the same feature contract and domain-safe monotonic + constraints, plus its own Platt map; +4. an OptBinning WOE/IV scorecard using borrower, contract, platform grade, and + pricing signals, followed by regularized logistic regression and Platt; +5. the same scorecard protocol without grade, subgrade, interest, installment, + or their derived platform-price signals. + +The existing active scores are refit and must reproduce the hashed V4 score +artifact to `1e-12`; the referenced values are then used. Every model uses the +same status-independent 36-month universe, temporal train tail, 2011 +probability-calibration block, eight six-month residual windows, taxonomies +`1/2/5/10`, and 2016--2017 evaluation menus. + +## Data Contract + +- The full raw file is scanned; no row sample is permitted. +- Model fitting uses every label-available row in its declared temporal role. +- The 60-month population is outside the common payoff and maturity horizon. +- The 2014--March 2016 gap is not relabeled from early resolutions; doing so + would induce duration selection. A survival target would be a different + estimand and paper scope. +- Late-schema bureau variables absent from 2007--2011 cannot enter these + controls. Missing-by-era imputation is not treated as temporal validation. + +## WOE, IV, Monotonicity, And Stability + +OptBinning is fit on the PD-development training block only, with automatic +monotonic binning, two to eight bins, and a five-percent minimum bin share. +All bin tables, IV values, logistic coefficients, missing handling, and +feature-level population-stability indices are persisted. The borrower-only +scorecard estimates dependence on incumbent underwriting signals; it does not +claim those signals are illegitimate. + +Monotonic constraints are imposed only where the direction has a defensible +credit-risk interpretation. Ambiguous effects such as loan amount, open-account +count, employment length, and interactions remain unconstrained. + +## Stop And Interpretation Rules + +- No OOT outcome may select, tune, remove, promote, or weight a learner. +- No scorecard, constraint, feature, bin, taxonomy, residual window, or origin + is changed after OOT evaluation. +- All eight windows and all five learners are reported, including failures. +- The controls never enter portfolio optimization in this protocol. +- A result may strengthen model-class robustness of the observed coverage + failure. It cannot establish universal nontransport, causal validity, + deployment readiness, scorecard superiority, or a better policy. +- Prediction metrics, calibration slopes, PSI, IV, and WOE are diagnostics; + none is a new IJDS novelty claim by itself. diff --git a/docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md b/docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md new file mode 100644 index 0000000..0dcd883 --- /dev/null +++ b/docs/research/ijds_credit_risk_controls_v1_stop_2026-07-13.md @@ -0,0 +1,24 @@ +# IJDS Credit-Risk Controls V1 Technical Stop - 2026-07-13 + +## Status + +V1 is stopped and cannot support a scientific result. No primary-OOT outcome +evaluation was run and no partial score, IV, PSI, coverage, or model metric was +inspected. + +## Failure + +The tagged V1 implementation completed model fitting and reached artifact +serialization. PyArrow rejected the OptBinning aggregate `Totals` row because +the `WoE` column mixed numeric bin values with an empty string. A valid +`protocol_freeze.json` was never written. + +## Recovery Boundary + +V1b changes only the diagnostic table representation: known numeric bin-table +columns are coerced with invalid aggregate placeholders represented as missing +numeric values. A synthetic test must serialize the complete table to Parquet. +The five learners, feature lists, monotonic constraints, OptBinning options, +row universe, temporal splits, taxonomies, residual windows, and stop rules are +unchanged. V1 partial directories are removed after recording this stop and +must never be resumed or evaluated. diff --git a/docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md b/docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md new file mode 100644 index 0000000..166cd1e --- /dev/null +++ b/docs/research/ijds_credit_risk_controls_v2_evaluation_2026-07-13.md @@ -0,0 +1,32 @@ +# IJDS Credit-Risk Controls V2 Evaluation Protocol - 2026-07-13 + +## Frozen Source + +V2 imports the V1b outcome-free score and residual-recipe freeze by run tag, +protocol tag, protocol commit, artifact descriptors, and freeze SHA-256. V2 may +not refit a learner, alter a bin, change a constraint, or regenerate a score. + +## One Evaluation + +After verification, V2 joins the September 2020 archive endpoint once and +reports every learner, residual window, taxonomy in `1/2/5/10`, temporal role, +and stratum. The canonical paper-facing robustness diagnostic is all-candidate +coverage for five groups over all eight windows; other taxonomies and strata +remain closed diagnostics. + +For each learner and temporal role, V2 reports candidate, resolved, and +unresolved rows; ROC AUC, Gini, KS, average precision, Brier score, log loss, +ten-bin ECE, calibration-in-the-large, calibration intercept, and calibration +slope. These metrics describe model transport and do not select a learner. + +## Interpretation + +- All five learners and all eight residual windows are reported. +- No learner ranking, winner, ensemble, retuning, or feature removal is + authorized from V2 outcomes. +- Platform-versus-borrower and monotonic-versus-active differences are + dependence diagnostics, not superiority tests. +- The controls remain outside portfolio optimization. +- Robustness across these five specifications can strengthen the archive- + specific model-class boundary of the coverage result. It cannot prove + universal nontransport, deployment validity, or scorecard superiority. diff --git a/docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md b/docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md new file mode 100644 index 0000000..230434e --- /dev/null +++ b/docs/research/ijds_credit_risk_controls_v2_stop_2026-07-13.md @@ -0,0 +1,23 @@ +# IJDS Credit-Risk Controls V2 Numerical Stop - 2026-07-13 + +## Status + +V2 completed the frozen outcome join and its coverage, discrimination, loss, +and censoring outputs are valid. It is not promoted to paper-facing evidence +because BFGS returned `success=false` for 17 of 30 otherwise finite calibration +intercept/slope fits under an unnecessarily strict gradient tolerance. + +## Preserved Result + +The 3,520-row temporal-coverage artifact is frozen under SHA-256 +`956bdc9880c80cebc1f48fc2cdf57688dfff7dddd3939b4fd972413d83039767`. +V2b must reproduce this frame exactly, including row order, values, dtypes, and +shape. No score, recipe, outcome, coverage cell, learner, or window may change. + +## Recovery + +V2b replaces only the two-parameter calibration diagnostic solver with +unpenalized scikit-learn logistic regression and fails on nonconvergence. It +also preindexes the immutable fit audit to remove repeated full-frame scans. +The optimization is accepted only after exact V2 coverage equivalence; it does +not alter the scientific model or react to result direction. diff --git a/docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md b/docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md new file mode 100644 index 0000000..97fc574 --- /dev/null +++ b/docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md @@ -0,0 +1,38 @@ +# IJDS Endpoint-Availability Sensitivity Protocol + +## Question + +Do the active candidate-coverage and comparator-identification statements +survive alternative administrative delays for observing a terminal charged-off +status at the fixed September 30, 2020 evaluation cutoff? + +## Retrospective Boundary + +The archive, the six-month endpoint, and exploratory lag calculations were +already inspected. This protocol therefore controls complete reporting and +reproducibility; it is not preregistration, an untouched holdout, prospective +confirmation, or a basis for choosing an endpoint. + +## Locked Design + +- Preserve the 640,543-row status-independent universe, all frozen scores, + residual recipes, allocations, rulers, coordinates, and comparator supports. +- Reconstruct outcomes at charged-off lags 0, 3, 6, 8, and 12 months. Fully + Paid retains the declared zero administrative lag. +- Report policy-development, primary-OOT, and censored-extension coverage for + all five learners, eight windows, and the canonical five-stratum taxonomy. +- Re-evaluate all 48 two-ruler endpoint cells and all V4 exact-support + envelopes at every lag using one common unresolved-outcome assignment per + funded union. +- Do not refit scores, conformal recipes, allocations, rulers, coordinates, or + supports. Do not select a lag, model, window, metric, or policy. + +## Interpretation Rule + +The paper may state the 40-cell coverage result only under endpoint contracts +for which every upper bound is below 0.90. Any exception must be reported. A +directional portfolio conclusion may be called endpoint-robust only if it has +the same nonzero direction at every lag; crossing zero is retained as partial +identification, not failure of computation. + +Required tag: `protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1`. diff --git a/docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md b/docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md new file mode 100644 index 0000000..33c16fe --- /dev/null +++ b/docs/research/ijds_endpoint_reason_recovery_v5_erratum_2026-07-15.md @@ -0,0 +1,74 @@ +# IJDS Endpoint-Reason Recovery V5 Erratum + +## Status + +The V4 evaluation +`ijds-binary-geometry-frontier-v4-2026-07-15-v4` stopped before writing any +evaluation artifact. Its immutable failure receipt is +`models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v4/evaluate_failure_receipt.json` +(5,179 bytes; SHA-256 +`f29ccaeb6341fe7c40e7826f0f40ba7fdc453adf680e4f20808c474715bfcc02`). +No protected stage or protected artifact was run or written. + +The stop occurred in the V3 reference reconciliation. The checker required +bit-exact equality after recomputation. IEEE-754 roundoff changed values such +as `0.9739145159564250` to `0.9739145159564248`; Pandas therefore flagged +30.53977% of the `mean_width` cells despite differences at machine precision. +This is a validation-implementation defect, not an endpoint or result change. + +## Locked V5 Repair + +V5 makes exactly two implementation changes. + +1. Reference reconciliation remains exact for IDs, strings, booleans, + integers, datetimes, row order, column order, dtypes, and missingness masks. + Floating columns use fixed absolute and relative tolerances of `5e-14`. + This is below the protocol ceiling of `1e-12`; every frame records the + maximum absolute and relative drift by floating column. Any larger drift + stops the run before outputs are written. +2. The exact point-cap contrast loop uses the already tested + `PolicyContrastIndex`. It validates loan facts and allocation IDs once per + window, then evaluates the same policy pairs through the same sharp-bound + array oracle. The public slow pairwise implementation remains the test + oracle. This changes indexing cost, not arithmetic definitions or row +inventory. + +A full pre-tag preflight rebuilt all 221,040 V3 contrast rows through the +indexed path in 196.679 seconds. Every inherited column matched the slow V3 +artifact with maximum absolute floating drift `0.0` under the locked +tolerance. This benchmark is implementation validation, not paper evidence. + +The benchmark preflight also confirmed that C2 has one matched point-score cap +per month, not one cap per window. The inherited window-level `frontier_cap` +field retains its deterministic first-month value solely for V3 column +reconciliation; C2 bounds always aggregate the full monthly sequence. Unique +window-level cap validation applies only to C0 and C1. No C2 result treats the +legacy scalar as its estimand. + +The V5 repair does not alter the raw universe, endpoint cutoff, six-month +charge-off lag, endpoint reason taxonomy, source freezes, score vectors, +residual recipes, taxonomies, allocation panels, comparator support, rulers, +coordinates, LGD, simulation design, or claim boundary. No outcome may select +an implementation, model, window, policy, or comparator. + +## Fresh Runs and Stop Rules + +Required fresh tags are: + +- `protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5`; +- `protocol/ijds-credit-risk-controls-2026-07-15-v5`; +- `protocol/ijds-normalized-objective-frontier-2026-07-15-v5`; +- `protocol/ijds-missingness-sensitivity-2026-07-15-v2`; and +- `protocol/ijds-rolling-origin-stability-2026-07-15-v4`. + +The binary, credit-control, and two-ruler runs must reconcile every V3 +reference column under the locked equivalence contract. The binary run may +append only endpoint-reason and identification-width diagnostics. Missingness +and rolling-origin retain their previously declared complete specification +families; their tags change only because implementation provenance changed. + +Stop without adaptation if a reference frame changes inventory, a non-floating +column differs, a floating difference exceeds either tolerance, the optimized +grid differs from the slow oracle beyond `1e-15` in focused tests, or any +protected artifact would be touched. Failed directories remain immutable and +all retries require a fresh run tag. diff --git a/docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md b/docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md new file mode 100644 index 0000000..0ad2c7a --- /dev/null +++ b/docs/research/ijds_endpoint_reason_taxonomy_v4_protocol_2026-07-15.md @@ -0,0 +1,72 @@ +# IJDS Endpoint-Reason Taxonomy V4 Recovery Protocol + +## Status + +This protocol corrects one explanatory defect in the V3 reconstructed endpoint. +The archive and all V3 results have already been inspected. V4 is therefore a +retrospective reason-taxonomy recovery, not preregistration, confirmation, a new +endpoint choice, or submission freeze. + +The outcome-free V4-v1, credit-controls-v1b, and two-ruler-v1c freezes remain +immutable. V4 may import them by hash and may change only the reason assigned to +an archive-terminal row that is unresolved at September 30, 2020. It cannot +refit scores or residual recipes, modify an allocation, select a result, or +overwrite V3. + +Required tags: + +- `protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v4`; +- `protocol/ijds-credit-risk-controls-2026-07-15-v4`; and +- `protocol/ijds-normalized-objective-frontier-2026-07-15-v4`. + +## Defect + +V3 correctly required a nonmissing reconstructed availability date no later +than the cutoff before retaining a terminal label. Its prose and one resolution +string nevertheless described every reclassified terminal row as having a date +after the cutoff. Independent reconciliation found two distinct reasons among +the 525 primary rows: + +- 478 `Charged Off` rows have no parseable `last_pymnt_d`, so no reconstructed + availability date exists; and +- 47 terminal rows have a reconstructed availability date after the cutoff. + +Both groups were already unresolved in V3. The defect is explanatory, not +numeric. + +## Exhaustive Reason Taxonomy + +For a configured cutoff, every candidate must receive exactly one reason: + +1. `fully_paid_by_reconstructed_cutoff`; +2. `charged_off_by_reconstructed_cutoff`; +3. `terminal_availability_date_missing`; +4. `terminal_after_reconstructed_cutoff`; or +5. `nonterminal_or_unresolved_status`. + +Only the first two reasons have an observed binary endpoint. A missing date is +never treated as a post-cutoff date. Exact `Default` remains in the fifth class +because the terminal charged-off event is not established by that status alone. + +## Required Reconciliation + +1. Candidate, resolved, and unresolved counts must remain exactly + 376,890, 364,814, and 12,076 in primary OOT. +2. The 525-row V3-to-V4 difference must partition into 478 missing-date and 47 + post-cutoff rows. +3. Every reference column in coverage, prediction metrics, portfolio + evaluation, sharp contrasts, direction censuses, comparator envelopes, and + simulation artifacts must be exactly equal after Parquet loading. V4 may + append endpoint-reason and identification-width diagnostics only. +4. Scores, recipes, supports, rulers, coordinates, policies, allocations, + payoff definitions, LGD, and endpoint labels must not change. +5. Any scientific metric or direction difference stops the recovery. It cannot + be explained away as part of the taxonomy correction. + +## Interpretation Boundary + +V4 authorizes the statement that 525 archive-terminal candidates remain +unresolved under the conservative reconstruction, with 478 lacking a usable +availability date and 47 dated after the cutoff. It does not establish why a +date is missing, the true operational event date, missing-at-random behavior, +or a verified historical snapshot. diff --git a/docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md b/docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md new file mode 100644 index 0000000..e1941f7 --- /dev/null +++ b/docs/research/ijds_evaluation_endpoint_recovery_v3_protocol_2026-07-14.md @@ -0,0 +1,77 @@ +# IJDS V3 Evaluation and Endpoint Recovery Protocol + +## Status + +This is a locked, evaluation-only recovery of the V4 portfolio audit and the +five credit-risk coverage controls. It may import only the hash-verified V4-v1 +and credit-controls-v1b outcome-free freezes. It does not refit a learner, +change a residual recipe, alter a policy or comparator, select a window, or +inspect outcomes during construction. + +V2 remains immutable provenance. V3 writes to fresh run directories and is the +only candidate lineage for subsequent paper-facing evidence. + +## Defects Being Corrected + +The V2 candidate-coverage aggregation assigned every unresolved binary outcome +to failure for the lower endpoint and success for the upper endpoint. That is +conservative but not sharp when a prediction set is empty or contains both +binary outcomes. For observed covered count `O`, unresolved count `U`, +unresolved always-covered count `B`, unresolved never-covered count `E`, and +candidate count `N`, V3 uses + +`coverage_lower = (O + B) / N` + +and + +`coverage_upper = (O + U - E) / N`. + +The distributed archive is also not established as a September 30, 2020 +point-in-time status snapshot. V3 therefore does not use final terminal status +merely because it appears in the archive. It retains a terminal label only when +the declared conservative availability date is no later than September 30, +2020: last-payment month-end for Fully Paid and last-payment month-end plus six +months for Charged Off. All other rows remain unresolved. This is a +reconstruction rule, not a claim about the archive's actual publication date or +the exact operational charge-off date. + +## Locked Inputs + +- V4 outcome-free freeze: run + `ijds-binary-geometry-frontier-v4-2026-07-12-v1`, SHA-256 + `c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c`. +- Credit-control freeze: run `ijds-credit-risk-controls-2026-07-13-v1b`, + SHA-256 + `da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35`. +- Raw archive, design universe, scores, recipes, allocations, comparator + support, payoff, and solver settings remain unchanged. + +## Required Reconciliation + +1. Frozen artifact hashes and cardinalities must match their source freezes. +2. Scores, recipes, fit audits, solve records, allocations, and comparator + support must be byte-identical imports. +3. Coverage geometry and candidate counts must match V2; only endpoint + observability and the two all-candidate bound columns may change. +4. Every changed resolved label must have `label_available_at` after the cutoff. +5. Portfolio allocations must remain unchanged. Any metric movement must be + attributable solely to endpoint reconstruction. +6. All five learners and all eight windows remain co-primary; no model, window, + ruler, coordinate, or policy may be selected from V3 outcomes. + +## Stop and Interpretation Rules + +- If any learner-window sharp upper coverage endpoint reaches 0.90, the paper + must withdraw the universal all-model/all-window nontransport statement and + report the complete mixed result. +- If portfolio direction changes under the reconstructed endpoint, both V2 and + V3 directions must be reported and no direction may be promoted. +- The six-month charge-off lag remains a modeling assumption. Its sensitivity + must be reported, and the W7-W8 empirical crossing may not be described as a + causal mechanism. +- V3 cannot authorize a winner, causal effect, selected-set guarantee, + prospective guarantee, or deployment claim. + +Required tags are +`protocol/ijds-binary-geometry-frontier-v4-2026-07-14-v3` and +`protocol/ijds-credit-risk-controls-2026-07-14-v3`. diff --git a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md deleted file mode 100644 index 2f62787..0000000 --- a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md +++ /dev/null @@ -1,100 +0,0 @@ -# IJDS Exact-Alpha and Calibration-Selection Closeout - -Date: 2026-07-09 - -## Why the active claim changed - -The previous paper-facing alpha sweep did not recompute conformal quantiles at -each alpha. It scaled stored 90% row radii using average-width ratios from a -different conformal family. The resulting alpha-0.01 endpoints were useful as -an exploratory approximation, but they did not support the manuscript's -"exact alpha" language. The former `8/8`, `0.345084` threshold, and 50,010-grid -selection claims were therefore retired rather than cosmetically relabeled. - -## Exact replay - -`src/models/conformal_alpha_grid.py` reconstructs the frozen score-decile -Mondrian recipe from its result payload and recomputes quantiles for each alpha. -At the reference 90% level, replay matches the stored point, low, and high -vectors to at most `6.67e-16`. The exact alpha sweep also exposes why -`alpha = 0.01` is not decision-useful here: average interval width is `0.9882` -and `93.54%` of upper endpoints equal one. - -## Final policy protocol - -The final run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. -It uses the exactly replayed 90% interval recipe and a nine-policy round-number -grid: - -- `tau in {0.15, 0.17, 0.19}`; -- `gamma in {0.25, 0.50, 0.75}`; -- linear `q = p + gamma(u-p)` only; -- no cap, tail rule, or uncertainty-aversion penalty; -- point-PD expected net return in the objective; -- conformal `q` only in the portfolio-risk constraint. - -The loader keeps outcomes physically separate from the 12-column policy frame, -which has no default, realized-return, miscoverage, or assumption-conditional -quantity. It requires -at least 99.9% budget use, feasibility of the effective-PD cap, and -`B_u <= 0.28` on November 2017. Five candidates are eligible. Maximizing -expected point-PD objective selects `tau = 0.17, gamma = 0.50`, the -interpretable midpoint `q = (p+u)/2`. The selected row is invariant for caps in -`[0.259036, 0.290491)`. - -An outcome-free December replay selects the same policy. Outcomes are opened -only afterward: weighted default is `0.145650`, weighted miscoverage is -`0.124925`, and endpoint budget is `0.262082`. This independent audit is not a -coverage theorem; it is direct evidence that policy stability does not imply -selected-set conformal validity. - -## What was learned from the challengers - -- The exploratory 25-policy selector chose `gamma = 0.35` and earned - `$181,758.69` OOT, but its threshold was `0.646094` and its policy grid and - cap were harder to defend. -- The final 50/50 policy earns `$179,327.59`, only `$2,431.11` less, while - lowering weighted default from `0.04855` to `0.039375` and the threshold from - `0.646094` to `0.574279`. -- A 75% blend is safer (`0.035875` default; `0.516624` threshold) but earns - `$6,388.08` less than the selected midpoint. -- The matched point-PD policy earns more on the full OOT panel, but its default - rate and endpoint audit are substantially worse. It also beats CRPTO in some - temporal slices, so no universal dominance claim is supportable. - -## Endpoint-screen v7 promotion - -The v6 selector used `B_u + sqrt(0.10) <= 0.60`. Because alpha was fixed, that -screen was algebraically an endpoint cap but was described with the same -Markov expression whose interpretation requires weighted funded-set validity. -The v7 challenger removes that coupling: - -- selection uses the round deterministic cap `B_u<=0.28`; -- November selects and December replays the same nine-row grid; -- the winner is unchanged for caps in `[0.259036, 0.290491)`; -- the loader separates outcomes from a 12-column selector frame with no - statistical-bound columns; -- the OOT loader aligns candidates directly to exact-alpha rows by unique ID, - removing a private script import and redundant intermediate alignment; -- v6 and v7 produce identical 845 funded comparator rows, IDs, allocations, - exposures, interval values, and full/temporal metrics (`max abs diff = 0`); -- A39 now treats 31 origination months as the primary resampling units and - retains funded-loan resampling as a sensitivity. - -No protected DVC stage was rerun because the challenger changes only policy -selection semantics, panel alignment, and downstream evidence. The exact-alpha -artifact and every manifest-protected upstream byte remain the comparison -baseline. - -## Interpretation - -The scientific upgrade is not a larger search. It is a smaller and auditable -decision rule, an exact conformal replay at the level actually used, separation -of point-PD economics from conformal feasibility, and a policy selector whose -inputs can be inspected for outcome leakage. This is the active IJDS narrative. - -The historical OOT panel was inspected during earlier project development. -Accordingly, v7 is described as a retrospective lockbox replay with an -outcome-free, assumption-free final selector conditional on the frozen -conformal recipe, not as a pristine prospective holdout or preregistered trial. diff --git a/docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md b/docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md new file mode 100644 index 0000000..59ee8f0 --- /dev/null +++ b/docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md @@ -0,0 +1,63 @@ +# IJDS Fitting-Label Completion Sensitivity Protocol V2 - 2026-07-16 + +## Numerical Recovery + +The tagged V1 protocol stopped before writing scientific artifacts because an +`observed_only` refit under the current lock differed from the historical +outcome-free score by `7.389153786618863e-6`, above its unrealistic `1e-12` +score tolerance. V2 changes no scenario, sample, endpoint, model, or estimand. +Before evaluation it fixes a conservative `1e-4` score/recipe tolerance and, +independently, requires the observed-only coverage cells to reconcile with the +active artifact within `1e-5` and their geometry within `2e-5`. V1 is a stopped +numerical protocol, not evidence. + +## Question + +Does the active CatBoost/Platt coverage conclusion survive declared joint +completions of labels that have a terminal archive outcome but were unavailable +under the September 30, 2020 information contract when their fitting block was +constructed? + +## Locked Design + +- Retain the active status-independent 640,543-row universe, features, temporal + roles, CatBoost hyperparameters, Platt block, five-group taxonomy, eight + residual windows, endpoint contract, and primary OOT census. +- Refit the primary CatBoost model, Platt map, taxonomy, and eight canonical + residual recipes under exactly four complete scenarios: + `observed_only`, `all_unavailable_nondefault`, + `all_unavailable_default`, and `hindsight_terminal`. +- Change only the 41 PD-development, 24 probability-calibration, and 150 + conformal-fit labels unavailable at the information cutoff. Never change an + evaluation row or use an OOT outcome to choose a scenario. +- Freeze scores, recipes, fit audits, and scenario counts before the primary + OOT endpoint join. Evaluate every scenario over every window, overall and in + the declared phase stratum 2. +- Require the `observed_only` score and recipe replay to match the active + outcome-free freeze within the V2 numerical recovery tolerances. After the + endpoint join, separately reconcile all 16 overall/phase coverage cells with + the active artifact before interpreting any completion scenario. + +## Estimands + +For each scenario and residual window, report the sharp all-candidate primary +OOT coverage interval under the active six-month endpoint-availability rule, +plus phase-stratum geometry. The primary diagnostic is whether every overall +coverage upper bound remains below 0.90. + +## Interpretation Boundary + +CatBoost, Platt scaling, taxonomy construction, and conformal fitting are +nonlinear in the labels. The four scenarios are declared joint stress corners; +they are not sharp extrema over all `2^215` assignments. No result can select a +completion, model, window, encoding, or policy. `hindsight_terminal` is a +diagnostic and is not an information-feasible deployment rule. + +## Stop Rules + +- Stop on parent-artifact, ID-census, tag, commit, lock, or baseline-replay + mismatch. +- Stop if any fitting scenario changes a non-fitting row. +- Stop if any scenario/window/stratum cell is absent or nonfinite. +- Report all four scenarios regardless of direction. +- Do not execute or overwrite any protected historical stage or artifact. diff --git a/docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md b/docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md new file mode 100644 index 0000000..e7e0587 --- /dev/null +++ b/docs/research/ijds_fixed_taxonomy_c2_protocol_errata_2026-07-12.md @@ -0,0 +1,36 @@ +# Fixed-Taxonomy Protocol Semantic Errata - 2026-07-12 + +This note corrects terminology in the immutable V1/V2 protocol without +changing its design, code, artifacts, or numerical results. + +## Residual Interval + +The locked protocol called + +`[max(0, p - c_g), min(1, p + c_g)]` + +the convex-hull representation of a binary conformal prediction set. That +description is not correct. Intersecting the interval with `{0,1}` produces a +discrete set whose convex hull is generally `{0}`, `{1}`, `[0,1]`, or empty, +not the original continuous interval. + +The implemented object has always been a clipped split-conformal residual +interval in the ambient real line. Coverage means that the observed binary +outcome lies between its endpoints. Its upper endpoint is used as a decision +score. It is not a confidence limit for latent individual default probability. + +## Default Score And Objective + +The CatBoost margin is mapped through a 2011 Platt fit. Active prose calls the +result a Platt-scaled default score because OOT calibration drifts. The +coefficient `(1-p)r-p*LGD` is therefore a model-implied plug-in objective. It +equals a conditional expected payoff only if `p` equals the true conditional +default probability, which the retrospective audit does not establish. + +## Scientific Effect + +This is a semantic erratum only. The residual scores, finite-sample ranks, +interval endpoints, allocations, outcomes, bounds, tables, and figures are +unchanged. The original protocol file remains untouched so its tagged hash and +historical record stay intact. The 2026-07-12 active claim registry, manuscript, +supplement, and submission documents use the corrected terminology. diff --git a/docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md b/docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md new file mode 100644 index 0000000..c2592ae --- /dev/null +++ b/docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md @@ -0,0 +1,32 @@ +# IJDS Label-Lag Sensitivity Protocol + +## Question + +Does the empirical W7-W8 phase crossing survive plausible alternatives to the +six-month rule used to date a Charged Off label, and which alternatives still +satisfy the locked greater-than-99% monthly label-retention requirement? + +## Locked Design + +- Use the hash-verified V4-v1 CatBoost scores and 2011 taxonomy edges. +- Refit only the eight binary conformal residual recipes. +- Evaluate charge-off reporting lags of 0, 3, 6, 8, and 12 months. +- Report every lag and every window. Stratum 2 is the pre-existing phase + diagnostic; no lag or window may be selected from the results. +- Record retained rows, minimum monthly retention, phase-stratum prevalence, + residual quantile, mean interval width, and `{0,1}` share. +- A lag passes the inherited maturity rule only when every residual month has + retention strictly above 0.99. + +The archive and the six-month result were previously inspected. This is a +retrospective assumption sensitivity, not preregistration or independent +confirmation. No portfolio allocation or OOT outcome enters the run. + +## Paper Rule + +If the W7-W8 jump changes materially under any plausible lag, the finite-sample +observation must be described as lag-sensitive and illustrative. The +constant-score population proposition may remain as algebra, but the empirical +crossing cannot be described as a causal explanation or robust phase estimate. + +Required tag: `protocol/ijds-label-lag-sensitivity-2026-07-14-v1`. diff --git a/docs/research/ijds_literature_expansion_scan_2026-07-08.md b/docs/research/ijds_literature_expansion_scan_2026-07-08.md deleted file mode 100644 index bf11190..0000000 --- a/docs/research/ijds_literature_expansion_scan_2026-07-08.md +++ /dev/null @@ -1,514 +0,0 @@ -# IJDS literature expansion scan, 2026-07-08 - -## Scope - -This note records an external and local literature scan for the IJDS version of -CRPTO. It is designed as an editorial input, not as a request to reopen the -frozen champion. The active certificate remains the pool93 finite-grid decision -certificate documented in `docs/research/active_claims_2026-07-04.md`. - -Operational guardrails: - -- Do not modify `EXTRACTION_MANIFEST.json` or artifacts listed there. -- Do not rerun protected champion stages without explicit permission. -- Keep the IJDS body focused on the implemented certificate: finite alpha grid, - robust funding decision, exact violation audit, and economic return. -- Use the supplement for adjacent method families, diagnostics, and future work. - -Sources reviewed: - -- Local corpus: `Papers_tesis` inventory and benchmark snapshot - `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/snapshot.md`. -- Current bibliography: `book/references.bib`. -- Current manuscript anchors: `book/chapters/CRPTO_*.qmd`, `paper/submission`, - and the active claims register. -- External web scan on conformal decision-making, contextual optimization, - credit scoring uncertainty, IJDS-adjacent work, and 2025-2026 emerging papers. - -## Bottom line - -The paper already cites the core CRPTO neighborhood well: conformal risk control, -predict-then-calibrate, conformal contextual robust optimization, conformal -robustness control, conformal robust optimization/satisficing, end-to-end -conformal calibration, and credit/P2P decision papers. The most useful additions -are not dozens of new citations. The highest-value improvement is a small set of -strategic anchors that sharpen the IJDS story: - -1. Add one broad contextual optimization survey to show that CRPTO sits inside - the modern prediction-to-decision literature. -2. Add one credit-specific profit/uncertainty paper to show that economic credit - scoring is active, but CRPTO advances from score-level economics to a funded - portfolio decision certificate. -3. Add one non-exchangeability/source-shift conformal reference in the - supplement. -4. Add one post-selection or human-decision conformal limitation reference to - make the paper look honest and current. -5. Keep 2026 inverse/decision-calibrated and decision-aware conformal-set papers - as future-work comparators, not body-level foundations, unless reviewers ask. - -## Recommended additions - -### 1. Sadana et al. 2025, contextual optimization survey - -Candidate: - -- Rahul Sadana, Andrea Delage, Alexandre Forel, Emma Frejinger, Thibaut Vidal. - "A survey of contextual optimization methods for decision-making under - uncertainty." European Journal of Operational Research, 320(2), 2025. -- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724002200), - [arXiv](https://arxiv.org/abs/2306.10374). - -Why it matters: - -- It is an OR-facing survey of contextual optimization, prescriptive analytics, - predict-then-optimize, decision-focused learning, and smart predict/estimate- - then-optimize methods. -- It gives IJDS reviewers a clean map for why CRPTO is a decision paper rather - than a classifier leaderboard. - -Recommended placement: - -- Body, related work or positioning paragraph. -- One sentence is enough: CRPTO belongs to contextual/predictive-prescriptive - optimization, but differs by producing an auditable finite-grid robust funding - certificate rather than learning a new end-to-end policy. - -Priority: High. - -### 2. Xu, Kou, and Ergu 2025, profit-based uncertainty in credit scoring - -Candidate: - -- Zhuozhuo Xu, Gang Kou, Daji Ergu. "Profit-based uncertainty estimation with - application to credit scoring." European Journal of Operational Research, - 325(2), 2025. -- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221725002048), - [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v325y2025i2p303-316.html). - -Why it matters: - -- It is close to our application domain and explicitly links credit scoring, - uncertainty, rejection, and profitability. -- It supports the narrative that predictive uncertainty in lending should be - evaluated economically, while CRPTO moves the unit of decision from - application-level classification/rejection to portfolio funding under a - distribution-free certificate. - -Recommended placement: - -- Body if there is space in the credit/P2P paragraph. -- Otherwise supplement literature table. - -Priority: High. - -### 3. Xu et al. 2024, profit- and risk-driven credit scoring - -Candidate: - -- Zhuozhuo Xu, Yishun Dou, Gang Kou, Daji Ergu. "Profit- and risk-driven credit - scoring under parameter uncertainty: A multiobjective approach." Omega, 125, - 2024. -- Source: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0305048323001688). - -Why it matters: - -- It gives a credit-scoring reference for profit/risk tradeoffs under uncertain - cost-benefit parameters. -- It is useful if we want a short supplement note distinguishing CRPTO from - multiobjective credit-score design: CRPTO certifies a portfolio decision over - existing predictions and realized returns rather than optimizing an application - classifier under assumed parameter uncertainty. - -Recommended placement: - -- Supplement, not body, unless reviewers ask for more credit-scoring economics. - -Priority: Medium. - -### 4. Farinhas et al. 2024, non-exchangeable conformal risk control - -Candidate: - -- Antonio Farinhas, Alessandro Zecchin, Andre Martins, David Grangier. - "Non-Exchangeable Conformal Risk Control." ICLR 2024. -- Source: [arXiv](https://arxiv.org/abs/2310.01262). -- Local PDF already exists in `Papers_tesis/supplement`. - -Why it matters: - -- It directly addresses a limitation readers may raise: calibration/test - exchangeability, time dependence, and distribution shift. -- It can strengthen the supplement language around source-shift diagnostics - without making a new guarantee in the IJDS body. - -Recommended placement: - -- Supplement A23/A24 source-shift and calibration-drift discussion. -- Do not cite it as implemented protection unless we implement and validate the - method. - -Priority: High for supplement, low for body. - -### 5. Hegazy et al. 2025, valid selection among conformal sets - -Candidate: - -- Mahmoud Hegazy, Emma Frejinger, Pierre Pinson, Alexandre Forel. - "Valid Selection among Conformal Sets." 2025. -- Source: [arXiv](https://arxiv.org/abs/2506.20173). - -Why it matters: - -- It is conceptually important for CRPTO because our grid search selects among - candidate decisions and alpha values after evaluating constraints. -- The current paper is careful because it reports finite-grid denominators, - exact violation audit, and does not claim a new post-selection conformal - theorem. This paper helps articulate that boundary. - -Recommended placement: - -- Supplement limitations/future protocol. -- Possible sentence: future CRPTO variants could study formal post-selection - validity for choosing among multiple calibrated sets, whereas the present - paper reports a finite-grid certificate and exact audit for the selected - champion. - -Priority: High for limitations. - -### 6. Hullman et al. 2025, conformal prediction and human decision making - -Candidate: - -- Jessica Hullman, Christopher W. Zamecnik, Yuval Rabin, Fred Hohman. - "Conformal Prediction and Human Decision Making." 2025. -- Source: [arXiv](https://arxiv.org/abs/2503.11709). - -Why it matters: - -- It argues that valid conformal sets are not automatically useful for decisions - unless the downstream objective is explicit. -- This is useful for IJDS framing: CRPTO is not "uncertainty reporting"; it - operationalizes uncertainty through a robust funding decision and economic - audit. - -Recommended placement: - -- Optional short citation in introduction or limitations. -- Use sparingly; avoid turning the paper into a human-factors discussion. - -Priority: Medium. - -### 7. Djeundje, Crook, and Andreeva 2025, dynamic loan portfolio profitability - -Candidate: - -- Viani B. Djeundje, Jonathan Crook, Galina Andreeva. "The devil in the details: - Dynamic prediction of loan portfolio profitability with macroeconomic drivers - through multi-state modelling." European Journal of Operational Research, - 327(2), 2025. -- Sources: [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v327y2025i2p703-715.html), - [University of Edinburgh](https://www.research.ed.ac.uk/en/publications/the-devil-in-the-details-dynamic-prediction-of-loan-portfolio-pro/). - -Why it matters: - -- It is a recent credit-portfolio profitability paper rather than a pure - application-level classifier paper. -- It can help if a reviewer wants more credit portfolio literature around - dynamic profitability and macroeconomic drivers. - -Recommended placement: - -- Supplement only, unless the introduction is rewritten to emphasize - macro-sensitive portfolio profitability. -- Do not use it to imply that CRPTO currently models macro transitions; it does - not. - -Priority: Medium-low. - -## Emerging close comparators to monitor - -These papers are close to CRPTO but are very recent, mostly preprint-era, or in -different domains. They should strengthen future-work positioning rather than -drive the IJDS body. - -### Zhou and Zhu 2026, inverse conformal risk control for decision robustness - -- "Calibrating Decision Robustness via Inverse Conformal Risk Control." -- Sources: [arXiv](https://arxiv.org/abs/2510.07750), - [OpenReview](https://openreview.net/forum?id=lV4tqcVIyx&referrer=%5Bthe+profile+of+Shixiang+Zhu%5D%28%2Fprofile%3Fid%3D~Shixiang_Zhu1%29). -- Value: Very close to CRPTO because it treats the robustness level itself as - the calibrated object and reports finite-sample guarantees on miscoverage and - regret for robust predict-then-optimize policy families. -- Recommendation: strongest 2026 future-work comparator. It is too new to make - it a foundation of the current body, but it is the cleanest citation if we add - one sentence about future calibration of robustness levels. - -### Stratigakos et al. 2026, decision-calibrated prediction sets - -- "Decision-calibrated prediction sets for robust power system operations." -- Source: [arXiv](https://arxiv.org/abs/2606.02081). -- Value: Closest phrase-level comparator for calibrating prediction sets by - downstream decision reliability. -- Recommendation: future-work comparator only. - -### Chen, Zhou, and Zhu 2026, learning polyhedral conformal sets for RO - -- "Learning Polyhedral Conformal Sets for Robust Optimization." -- Source: [arXiv](https://arxiv.org/abs/2605.08506). -- Value: Decision-aware conformal uncertainty sets for robust optimization. -- Recommendation: cite in future work if adding a paragraph on learned - uncertainty-set geometry. - -### Wang and Dobriban 2026, optimal decisions from prediction sets - -- "Optimal Decision-Making Based on Prediction Sets." -- Source: [arXiv](https://arxiv.org/abs/2602.00989). -- Value: Decision-theoretic framework for downstream use of prediction sets. -- Recommendation: monitor; useful if reviewers ask for more theory around - prediction sets as decision objects. - -### Huang, Farzaneh, and Simeone 2026, OCE risk-controlling prediction sets - -- "Optimized Certainty Equivalent Risk-Controlling Prediction Sets." -- Source: [arXiv](https://arxiv.org/abs/2602.13660). -- Value: OCE/CVaR-style risk-control extension. -- Recommendation: supplement/future work near the existing OCE/CVaR diagnostics. - -### Baesens et al. 2026, foundation models for credit risk prediction - -- "Foundation Models for Credit Risk Prediction: A Game Changer?" -- Source: [arXiv](https://arxiv.org/abs/2605.18147). -- Value: Useful for monitoring PD-layer baselines in credit risk. -- Recommendation: do not add to IJDS body now. CRPTO does not claim a PD model - leaderboard. - -## Already-covered nearest neighbors - -The current corpus and official submission already include the most important -methodological neighbors. These should remain the core comparison set: - -- Patel et al. 2024, conformal contextual robust optimization. -- Sun et al. 2024, predict-then-calibrate. -- Hu et al. 2026, conformal robustness control. -- Zhao et al. 2025/2026, conformal robust optimization and satisficing. -- Yeh et al. 2025/2026, conformal risk training and end-to-end conformal - calibration. -- Bao et al. 2025, CROMS. -- Zhou et al. 2025/2026, CREDO and CREME. -- Yang and Bi 2025, cost-aware calibration. -- Liu et al. 2026, online conformal portfolio selection. -- Yang and Jin 2026, multidistribution conformal prediction. - -The key editorial move is to distinguish CRPTO from each in one sentence: - -- Not a new PD scorer. -- Not a generic conformal-set method. -- Not end-to-end training. -- Not online rebalancing. -- Not a multidistribution fairness theorem. -- A finite-grid robust portfolio decision certificate with realized-return and - exact-violation audit. - -## IJDS-specific venue scan - -Relevant IJDS-adjacent sources are useful mainly for framing, not for core -method positioning: - -- Wiberg et al. 2025, "Synergizing Artificial Intelligence and Operations - Research." IJDS. Source: - [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2025.0077). -- Morucci et al. 2022, "A Robust Approach to Quantifying Uncertainty in Matching - Problems of Causal Inference." IJDS. Source: - [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0020). -- "Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial - Data Augmentation." IJDS. Source: - [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0033). - -Recommendation: - -- Do not overload the paper with IJDS self-citations. -- If a cover letter or response-to-reviewers needs venue fit, Wiberg et al. - 2025 is a concise AI+OR anchor. -- Morucci et al. 2022 can be mentioned only if a reviewer asks about robust - uncertainty quantification precedent in IJDS. -- The cost-sensitive classification IJDS paper is less aligned than Yang and Bi - 2025 for our current framing. - -## Recent credit-risk papers to leave out unless needed - -The 2025-2026 EJOR credit-risk stream is active. Several papers are useful for -background but should not crowd the IJDS body: - -- Distaso, Roccazzella, and Vrins 2025, "Business cycle and realized losses in - the consumer credit industry." Source: - [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724009688). -- Ballegeer, Bogaert, and Benoit 2025, "Evaluating the stability of model - explanations in instance-dependent cost-sensitive credit scoring." Source: - [ScienceDirect](https://www.sciencedirect.com/science/article/abs/pii/S0377221725004230). -- Baesens et al. 2026, foundation models for credit risk prediction. Source: - [arXiv](https://arxiv.org/html/2605.18147v1). - -Recommendation: - -- Use these only if a reviewer asks for macro-credit loss, explanation - stability, or modern PD-model background. -- They do not change the CRPTO contribution because the paper is not proposing a - new PD learner. - -## Exact manuscript insertion plan - -Keep the body lean. The highest-value body additions are two citations: - -1. Add Sadana et al. 2025 to the contextual optimization / prescriptive - analytics positioning. -2. Add Xu, Kou, and Ergu 2025 to the credit scoring uncertainty / profit - paragraph. - -Then use the supplement for three limitation/future-work citations: - -3. Farinhas et al. 2024 in source-shift/non-exchangeability. -4. Hegazy et al. 2025 in post-selection validity. -5. Zhou and Zhu 2026 as the preferred emerging future-work comparator; if space - allows, add one of Stratigakos et al. 2026, Chen et al. 2026, or Wang and - Dobriban 2026 for decision-calibrated conformal-set geometry/use. - -Suggested body language: - -> CRPTO sits within contextual optimization and prescriptive analytics, where -> predictions and downstream decisions are optimized jointly or sequentially, -> but it differs by auditing a finite grid of robust funding decisions rather -> than training a new decision rule. - -> Recent credit-scoring work evaluates uncertainty through profit and rejection -> decisions; CRPTO instead evaluates uncertainty at the funded-portfolio level, -> where the reported object is an economically realized decision certificate. - -Suggested supplement language: - -> The present certificate assumes the calibration/audit design documented in -> the finite-grid protocol. Extensions to non-exchangeable conformal risk control -> and formal selection among multiple conformal sets are natural next steps, but -> are not claimed by the current champion. - -## Bibliography action list - -Add or verify BibTeX entries for: - -- `sadana2025contextual` -- `xu2025profit_uncertainty_credit` -- `xu2024profit_risk_credit` -- `farinhas2024nonexchangeable_crc` -- `hegazy2025valid_selection_conformal_sets` -- `hullman2025conformal_human_decision` -- `djeundje2025dynamic_loan_portfolio_profitability` -- `zhou2026inverse_crc_decision_robustness` -- `stratigakos2026decision_calibrated_sets` -- `chen2026polyhedral_conformal_ro` -- `wang2026optimal_decision_prediction_sets` -- `huang2026oce_rcps` -- `baesens2026foundation_credit_risk` - -Before adding all of them, apply a manuscript budget rule: - -- Body: at most two new citations unless a paragraph is rewritten. -- Supplement: three to six citations are acceptable. -- Future-work table: emerging 2026 papers are acceptable if clearly labeled as - future work and not as implemented guarantees. - -## Citation synchronization note - -The official submission `.tex` currently contains several compact citation -anchors that are not all mirrored in the source `.qmd` files. Examples observed -during this scan include: - -- `angelopoulos2024foundations` -- `bates2021rcps` -- `zhou2024` -- `sun2024ptc` -- `boosting2025default` -- `yeh2025training` - -If the paper is regenerated from Quarto, these anchors could be lost unless the -QMD sources are synchronized. The safest path is to update QMD first, regenerate -official submission artifacts, and then re-run the IJDS compile checks. - -## Do-not-add list for the current IJDS submission - -Do not add broad or weakly related references unless a reviewer asks: - -- Generic LLM/tabular-model credit-risk papers that do not affect the CRPTO - certificate. -- Generic conformal prediction surveys beyond the already cited foundations. -- Extra IJDS venue papers solely for journal signaling. -- Additional classifier benchmarking papers unless they directly affect the - LendingClub decision framing. - -## Decision recommendation - -For the current IJDS revision, implement the following compact literature -upgrade: - -1. Add Sadana et al. 2025 and Xu, Kou, and Ergu 2025 to the body. -2. Add Farinhas et al. 2024 and Hegazy et al. 2025 to the supplement. -3. Add one 2026 emerging decision-calibrated conformal reference to future work. -4. Keep all language explicit that these are positioning and future-work - references; they do not change the frozen champion or its claims. - -This gives reviewers the right signals: the paper is current, aware of adjacent -decision-calibrated conformal work, and still disciplined about what it actually -implements and certifies. - -## Post-read implementation note, 2026-07-08 - -After the missing PDFs were downloaded manually to `Downloads`, they were copied -into `Papers_tesis/supplement` with normalized names: - -- `Xu Kou Ergu 2025 - Profit-based uncertainty estimation with application to credit scoring.pdf` -- `Xu et al 2024 - Profit- and risk-driven credit scoring under parameter uncertainty.pdf` -- `Djeundje Crook Andreeva 2025 - Dynamic prediction of loan portfolio profitability.pdf` -- `Wiberg Dai Lam Kulkarni 2025 - Synergizing AI and OR.pdf` - -The expanded `academic-pdf-intake` inventory now sees 100 PDFs in scope: -97 under `Papers_tesis` and the three active CRPTO PDFs. The post-read -manuscript update keeps the body claim unchanged and implements only narrative -and boundary changes: - -- Body: CRPTO is framed as a contextual-optimization credit instance, anchored - by credit-scoring uncertainty/profit literature and the AI/OR IJDS perspective. -- Theory: post-selection conformal validity is named explicitly as future - protocol, not as an implicit property of the finite-grid frontier. -- Supplement: non-exchangeable CRC, valid selected conformal sets, - decision-calibrated/inverse conformal robustness, learned polyhedral sets, - OCE-RCPS, and recent credit-profitability/explanation-stability papers are - mapped to diagnostics or future-work boundaries. - -No frozen champion artifact, manifest entry, or protected DVC stage is changed -by this literature update. - -## Venue recheck, 2026-07-09 - -The current [IJDS submission guidelines](https://pubsonline.informs.org/page/ijds/submission-guidelines) -still require the Data + Models + Decisions + Implications synthesis and now -state an explicit abstract sequence: problem/data-science relevance, method and -results, then learned insight and implication. The CRPTO abstract already -covered the first two elements but ended by repeating pipeline architecture. -Its closing sentences now state the learned decision insight and committee use: -the return-bound frontier makes predictive uncertainty actionable, while the -certificate separates exact funded-set accounting from its weighted-validity -assumption. - -Two recent accepted IJDS papers provide a useful style check rather than new -citation requirements: - -- [Robust and Interpretable Policy Learning for Manufacturing Process Parameters](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0041) - leads from a concrete decision problem to a named policy method, robustness, - interpretability and practical deployment evidence. -- [Using Operational Data Analytics for Planning Decisions Under Uncertainty](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0051) - makes the estimate-then-optimize gap explicit, compares against several - decision baselines and closes with real-data effectiveness. - -CRPTO already follows those structural signals through the funded-set decision, -the exact certificate, the A19 regret comparator and the managerial implication. -No additional venue self-citation is warranted: the existing Das et al. IJDS -credit-risk anchor and Wiberg et al. AI/OR anchor are sufficient, and adding -more would crowd the body without changing the novelty boundary. diff --git a/docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md b/docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md new file mode 100644 index 0000000..ebefcf4 --- /dev/null +++ b/docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md @@ -0,0 +1,75 @@ +# IJDS Missingness Sensitivity Protocol + +## Status and Question + +This protocol closes the two sensitivity requirements declared by the full raw +data audit. The archive and active results have already been inspected, so the +exercise is retrospective discipline rather than preregistration or an +untouched confirmation. + +Question: does the primary CatBoost candidate-coverage conclusion depend on the +frozen conventions for structurally nullable delinquency recency and the +partially supported legacy bankruptcy count? + +The original required tag was +`protocol/ijds-missingness-sensitivity-2026-07-15-v1`. Before either phase was +run, the implementation-only V5 recovery erratum moved execution to V2. A +pre-execution red-team review then found that V2's native arm replaced a binary +bankruptcy indicator with a count, confounding feature semantics and missing- +value encoding. V1 and V2 were therefore stopped before freeze or evaluation. +The corrected execution tag is +`protocol/ijds-missingness-sensitivity-2026-07-15-v3`; its computational +erratum is part of the locked lineage. + +## Fixed Information Contract + +- Use the active 640,543-row status-independent design and every eligible row. +- Preserve the PD-development, 2011 Platt/taxonomy, eight residual-window, and + primary OOT blocks. +- Preserve seed 42, CatBoost hyperparameters, the five-stratum taxonomy, alpha + 0.10, finite-sample ranks, and the V4 endpoint reconstruction. +- Import the active CatBoost score and recipe as the baseline by hash. +- Fit alternatives before loading the primary OOT endpoint. Scores, taxonomies, + recipes, and fit audits must be frozen and hashed before evaluation. +- The alternatives are coverage controls only. They do not enter portfolio + optimization and cannot select a model, feature convention, residual window, + or endpoint. + +## Complete Specification Family + +All three specifications are co-reported: + +1. **Active sentinel convention.** Retain `delinq_recency=999` when + `mths_since_last_delinq` is missing and `has_bankruptcy=0` when + `pub_rec_bankruptcies` is missing. Import the frozen V4 score and recipes. +2. **Explicit missing indicators.** Retain the active mapped features and add + `delinq_recency_missing` and `bankruptcy_count_missing`. +3. **Native numeric missingness with preserved feature semantics.** Replace + mapped `delinq_recency` by the same numeric delinquency-recency field with + `NaN` instead of 999. Replace `has_bankruptcy` by the same binary transform + (strictly positive bankruptcy count is 1; observed zero is 0) while + preserving source missingness as `NaN`. CatBoost handles both nullable + values natively. The raw bankruptcy count is not a model feature. + +No hybrid or winning variant may be created after evaluation. The two +alternatives isolate sensitivity to the encoding contract; they are not a new +model contest or a claim that missingness is informative, ignorable, or causal. + +## Outputs and Stop Rules + +- Report all eight five-stratum primary OOT sharp coverage intervals and + descriptive AUC, Brier, calibration, KS, Gini, and average precision for all + three specifications. +- Report the raw missingness census by temporal role for both source fields. +- Stop before evaluation if the baseline score/recipe descriptors differ, a + taxonomy edge repeats, a canonical residual group has fewer than 1,000 rows, + or any OOT outcome-derived column reaches fitting. +- If any alternative upper coverage bound reaches 0.90, withdraw a + missingness-insensitive coverage statement and report the complete mixed + result. Do not select a convention that preserves failure. +- If every alternative remains below 0.90, report robustness only over these + three declared encodings. Do not claim missing-at-random identification, + general missing-data robustness, or model superiority. + +The run writes only fresh experiment directories and never modifies protected +historical artifacts or `EXTRACTION_MANIFEST.json`. diff --git a/docs/research/ijds_missingness_sensitivity_v3_erratum_2026-07-15.md b/docs/research/ijds_missingness_sensitivity_v3_erratum_2026-07-15.md new file mode 100644 index 0000000..69a5c58 --- /dev/null +++ b/docs/research/ijds_missingness_sensitivity_v3_erratum_2026-07-15.md @@ -0,0 +1,37 @@ +# IJDS Missingness Sensitivity V3 Erratum + +## Stop Decision + +V1 and V2 were stopped before freeze or evaluation. Their proposed native- +missingness arm removed the binary `has_bankruptcy` feature and inserted the +raw `pub_rec_bankruptcies` count. That arm changed both missing-value encoding +and feature semantics, so it could not isolate the declared sensitivity. + +No output directory exists for either stopped run. No outcome was inspected to +choose the repair, no protected stage was run, and no protected artifact was +modified. + +## Locked V3 Repair + +V3 retains all rows, splits, labels, CatBoost parameters, Platt calibration, +taxonomy groups, residual windows, endpoint reconstruction, and evaluation +rules. It changes only the third feature matrix: + +- `delinq_recency_native` is the same delinquency-recency source used by the + active feature, with source missingness retained as `NaN` instead of mapped + to 999; +- `has_bankruptcy_native` retains the active binary meaning: an observed + strictly positive count maps to 1 and an observed zero maps to 0, while a + missing source count remains `NaN`; and +- the raw bankruptcy count is not included. + +The complete family remains baseline sentinel, sentinel plus explicit missing +indicators, and native nullable values. The alternatives are coverage controls +for primary CatBoost only. They cannot select a model or encoding, enter +portfolio optimization, identify a missingness mechanism, or support MAR/MNAR +claims. + +The fresh tag is +`protocol/ijds-missingness-sensitivity-2026-07-15-v3`. Any implementation, +specification, row-universe, or pre-outcome contract mismatch stops execution +and requires a new tag. diff --git a/docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md b/docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md new file mode 100644 index 0000000..1da7984 --- /dev/null +++ b/docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md @@ -0,0 +1,212 @@ +# IJDS normalized-score and objective-matched frontier protocol + +## Status + +This protocol is locked before the complete outcome-free V1 run under tag +`protocol/ijds-normalized-objective-frontier-2026-07-12-v1`. It is a +specification-complete retrospective challenger over the already inspected V4 +archive. It is not a preregistration, a pristine lockbox, a submission freeze, +or a repair of conformal validity. + +V4 remains the active manuscript evidence until a separate V2 evaluator +verifies the V1 freeze by hash, joins outcomes once, and passes the promotion +rules below. The V1 runner cannot accept a status, outcome, default, payment, +realized-payoff, or miscoverage column. + +## Why another specification is necessary + +The outcome-free support audit established that the inherited fixed-cap family +was not semantically complete: + +- `gamma=0` was objective-slack in 624/624 cells; +- omitted `gamma=1` was feasible and decision-active in 624/624 cells; and +- at the same numeric cap, `gamma=1` sacrificed more plug-in objective than + `gamma=.75` in every cell. + +A fixed numeric `tau` therefore compares different locations on different +score scales. The challenger replaces that single ruler with two declared, +non-equivalent coordinates. Neither is called neutral. + +## Frozen research object + +The parent is the immutable V4-v1 outcome-free freeze at commit `2f8a760`. +Before use, the challenger verifies its freeze descriptor, protocol identity, +all referenced artifact hashes, empty outcome-column list, and source archive +hash. It imports only: + +- the status-independent score universe and design roles; +- the CatBoost five-group residual recipes for all eight windows; +- contractual amount, rate, and purpose from an explicit raw-column allowlist; +- the coherent plug-in objective, USD 1 million monthly budget, and 25% purpose + cap from the parent config. + +The complete family is +`gamma={0,.25,.50,.75,1}` with +`q_gamma=p+gamma(U-p)`. The 11 development and 15 primary monthly menus and all +eight windows are retained. No outcome selects a gamma, ruler, coordinate, +month, or window. + +## Common base polytope + +For month `t`, let `A_t` contain allocations satisfying exact budget, loan +bounds, and the purpose cap. Let `v` be the coherent plug-in payoff rate and +`s` one member of the complete gamma score path. The score-minimum and a common +score-independent plug-in optimum are + +\[ +m_s=B^{-1}\min_{a\in A_t}s^Ta, +\qquad +z^*=\max_{a\in A_t}v^Ta, +\qquad +o_s=B^{-1}s^Ta^*. +\] + +The optimizer `a*` is solved once per menu in stable ID order. For every score, +the minimum and maximum funded score subject to +`v^Ta >= z* - 1e-7 dollars` are also solved. A score range above `1e-8` at the +objective optimum is treated as unresolved objective-tie dependence and stops +the run. + +## Ruler 1: objective-matched efficient frontier + +This is the primary ruler. Let `z_min,s` be the plug-in objective of a +minimum-score portfolio and define the common lower endpoint + +\[ +z_L=\max_s z_{min,s}. +\] + +For `rho in {.25,.50,.75}`, set + +\[ +z(rho)=z_L+rho(z^*-z_L) +\] + +and solve + +\[ +\min_{a\in A_t}s^Ta\quad\text{subject to}\quad v^Ta\ge z(rho). +\] + +All scores therefore face the same absolute model-implied objective floor on +their common attainable support. This controls plug-in opportunity cost, not +true expected return. It does not make either score a true risk measure. + +## Ruler 2: normalized-score relaxation + +This is the secondary structural ruler. For +`lambda in {.25,.50,.75}`, define + +\[ +c_s(lambda)=m_s+lambda(o_s-m_s) +\] + +and maximize `v^Ta` over `A_t` subject to +`s^Ta <= B c_s(lambda)`. + +If `s_tilde=a s+b` for `a>0`, then +`m_tilde=a m_s+b`, `o_tilde=a o_s+b`, and +`c_tilde=a c_s+b`. The feasible allocation set is therefore unchanged at the +same lambda. This proves positive-affine score invariance and nestedness in +lambda. Equal lambda is not equal true risk, funded default, shadow price, +objective sacrifice, or operational tolerance. + +The denominator `o_s-m_s` must exceed `1e-4` in every cell. Every interior cap +must bind within `1e-8`; an endpoint failure is not dropped or repaired. + +## Frozen cells and primary contrast + +The V1 freeze contains exactly + +`8 windows x 26 months x 5 gammas x 3 coordinates x 2 rulers = 6,240` + +solve records, plus every positive funded exposure. The primary empirical +contrast, evaluated only in V2, is the full-upper endpoint minus the point-score +endpoint (`gamma=1 - gamma=0`) for each window, ruler, coordinate, and metric. +Interior gammas remain a complete diagnostic path and cannot be selected as a +winner. + +The endpoint contrast is a deterministic archive contrast. It is not a causal +effect, population expectation, investor return, or conformal robustness +certificate. Unresolved outcomes must use one common loan-wise assignment over +the funded union to obtain sharp fixed-allocation bounds. + +## Numerical falsification + +1. All solves use one-thread HiGHS simplex with exact budget. +2. Every primary endpoint allocation is rerun after reversing ID order. + Exposure distance is `L1/(2B)`. Distance above `1e-10` or objective drift + above USD `1e-5` stops the run. +3. For primary periods April 2016, November 2016, and June 2017, every endpoint, + window, ruler, and coordinate is independently resolved with OR-Tools GLOP. + Objective-rate or funded-score disagreement above `1e-7` stops the run. +4. Synthetic unit tests enumerate a small continuous LP's vertices and compare + both ruler formulations with the solver output. +5. Budget residual above USD `1e-6`, normalized-cap residual above `1e-8`, or + absolute objective-floor mismatch above USD `1e-5` stops the run. + +The second solver validates the declared finite grid, not every real-valued +frontier point. This challenger therefore makes no new "exact all-cap +frontier" claim. + +## Outcome-free stop rules + +Stop before any outcome join if: + +- a parent hash, source hash, ID alignment, month count, or window count differs; +- any forbidden column reaches the decision frame; +- one of the 6,240 cells is missing or infeasible; +- a normalized score or common objective range is numerically empty; +- the score-independent objective optimum has unresolved score tie range; +- ID reversal or GLOP validation exceeds tolerance; or +- all 360 primary endpoint comparisons for one ruler are allocation-identical + within `1e-6` of budget. + +No threshold or tolerance may be relaxed after seeing the run. A stopped or +degenerate run is reported as evidence rather than silently redesigned. + +## V2 evaluation and promotion rules + +V2 requires a new protocol commit and tag containing the exact V1 freeze +descriptor. Only then may it join archive outcomes. It must report every +predeclared endpoint contrast for standardized payoff, funded default, and +funded binary miscoverage under sharp common-outcome bounds. + +The challenger may strengthen the negative identification audit if rulers or +windows disagree. A positive descriptive direction is eligible for further +rolling-origin evaluation only if its sharp bound has the same nonzero sign in +all eight windows, all three coordinates, and both rulers. That condition still +does not restore conformal validity. Any positive CRPTO claim would additionally +have to survive every feasible rolling origin under a separately locked replay. + +No majority vote, selected coordinate, selected gamma, selected window, or +post-outcome comparator narrowing is allowed. + +## Pre-tag structural smoke + +Before locking the protocol tag, one outcome-free cell was used only to test +formulation and numerical scale: April 2016, W1, all five gammas, with no +status or outcome columns loaded. The attainable score ranges were +`0.078139`, `0.216985`, `0.355830`, `0.494675`, and `0.633521`. At an objective +floor USD `1e-6` below the optimum, the minimum-to-maximum score spans were +between `4.04e-10` and `6.54e-9`; this showed that the diagnostic was measuring +the deliberately allowed objective slack rather than an alternate exact +optimum. The locked floor tolerance was therefore tightened prospectively to +USD `1e-7`, while the `1e-8` score-range stop was retained. + +For gamma endpoints and coordinate `.50`, normalized and objective-matched +caps reconciled within `1.46e-11` dollars or machine precision. GLOP versus +HiGHS objective-rate and funded-score differences were at most `5.82e-17` in +that smoke. These values are implementation checks, not empirical findings and +did not determine any policy direction. + +## Neighbor-method boundary + +RAC, conformal robustness control, CROMS, CREME/inverse CRC, CREDO, +decision-calibrated sets, and end-to-end conformal optimization motivate the +decision-level question but are not valid plug-ins here. Their finite-sample +claims require exchangeable decision contexts or a specifically modeled +non-exchangeable sequence. CRPTO has 376,890 candidate loans but only 11 common +development menus, and the action is a coupled monthly allocation. The full +applicability contract is recorded in +`docs/research/ijds_decision_method_applicability_2026-07-12.md`. diff --git a/docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md b/docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md new file mode 100644 index 0000000..f56e474 --- /dev/null +++ b/docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md @@ -0,0 +1,56 @@ +# IJDS normalized-score and objective-matched frontier V1c protocol + +## Status and lineage + +V1c is the second outcome-free numerical erratum of the locked two-ruler +challenger. No archive outcome has been loaded in V1, V1b, or the diagnostics +that motivated V1c. The complete lineage is: + +1. V1 stopped because a slack-floor score span was not a valid exact-tie test. +2. V1b replaced that proxy with nonbasic reduced costs and reversed-ID order. +3. V1b then stopped because final validation required a budget residual below + `USD 1e-6`, while both solver wrappers accepted residuals through + `USD 1e-4`. + +The exact records are: + +- `docs/research/ijds_normalized_objective_frontier_v1_stop_2026-07-13.md`; +- `docs/research/ijds_normalized_objective_frontier_v1b_protocol_2026-07-13.md`; +- `docs/research/ijds_normalized_objective_frontier_v1b_stop_2026-07-13.md`. + +V1c is not a pristine preregistration, a conformal repair, or a submission +freeze. V4 remains active until a completed outcome-free freeze and a separate +outcome evaluator satisfy the promotion contract. + +## Single V1c correction + +The final census budget tolerance is `USD 1e-4`, identical to the hard +reconciliation check already used by `PointPortfolioSession` and +`ObjectiveFloorPortfolioSession`. On a USD 1 million monthly budget this is a +relative tolerance of `1e-10`. The stopped V1b maximum was `USD 6.366e-6`. + +V1c does not round, rescale, or repair allocations after optimization. It only +uses one internally consistent acceptance threshold for the same recomputed +sum of funded exposures. + +## Unchanged contract + +Everything else in V1b remains locked: + +- the hash-verified V4-v1 parent and four-column raw allowlist; +- eight residual windows, 11 development menus, and 15 OOT menus; +- `gamma={0,.25,.50,.75,1}` and coordinates `{.25,.50,.75}`; +- objective-matched primary and normalized-score secondary rulers; +- exactly 6,240 solves, 720 endpoint comparisons, 1,440 endpoint order reruns, + 288 GLOP validations, and 26 objective-optimum basis diagnostics; +- reduced-cost threshold `1e-7`, order exposure threshold `1e-10`, order + objective threshold `USD 1e-5`, cap residual `1e-8`, objective-floor mismatch + `USD 1e-5`, and HiGHS--GLOP tolerances `1e-7`; +- outcome-free stop on incomplete cells, empty ranges, near-zero reduced costs, + order sensitivity, solver mismatch, or a fully degenerate ruler; and +- no winner, selector, causal effect, conformal repair, selected-set validity, + equal-risk claim, or submission freeze. + +The future V2 contrast remains `gamma=1 - gamma=0` over every predeclared +window, coordinate, ruler, and metric. No majority vote or favorable subset may +be promoted. diff --git a/docs/research/ijds_normalized_objective_frontier_v1c_results_2026-07-13.md b/docs/research/ijds_normalized_objective_frontier_v1c_results_2026-07-13.md new file mode 100644 index 0000000..4fa382f --- /dev/null +++ b/docs/research/ijds_normalized_objective_frontier_v1c_results_2026-07-13.md @@ -0,0 +1,90 @@ +# IJDS normalized/objective frontier V1c outcome-free results + +## Status + +V1c completed from clean tagged commit `46f4df9` under +`protocol/ijds-normalized-objective-frontier-2026-07-13-v1c`. Its status is +`outcome_free_frontiers_frozen_before_archive_outcome_join`. No status, payment, +default, realized-payoff, coverage, or miscoverage column entered the frontier. +No protected stage or protected artifact was run or written. + +V1 and V1b remain immutable stopped runs. V1c did not overwrite either run tag +or output path. + +## Complete census + +The 3,332-second run produced: + +- 6,240 solve records; +- 622,455 positive funded rows; +- 720 primary endpoint comparisons; +- 26 score-independent objective-optimum diagnostics; +- 1,440 reversed-ID endpoint reruns; and +- 288 independent OR-Tools GLOP validations. + +All eight windows, both roles, 26 monthly menus, five gammas, three coordinates, +and two rulers are present in both solve and allocation artifacts. + +## Numerical falsification + +- score ranges: `0.0628853` to `0.8546764`; +- common plug-in objective ranges: USD `66,857.10` to `151,229.07`; +- minimum absolute nonbasic reduced cost: `0.0046533`; +- zero near-zero reduced costs and zero primal-degenerate objective bases; +- maximum budget residual: USD `6.3656e-6`; +- maximum absolute frontier-constraint mismatch: `1.1642e-10`; +- maximum endpoint order distance: `2.6833e-12`; +- maximum endpoint objective drift under reversed IDs: USD `2.1787e-7`; +- maximum GLOP--HiGHS objective-rate difference: `2.1787e-13`; and +- maximum GLOP--HiGHS funded-score difference: `3.5943e-13`. + +Every value is below its locked V1c threshold. All artifact, source, summary, +receipt, and implementation descriptors were recomputed and matched the freeze. +The Parquet schemas contain optimizer metadata such as `solver_status`, but no +loan-outcome field. + +## Endpoint geometry + +The normalized exposure distance is `L1/(2B)` between `gamma=1` and `gamma=0`. + +| Ruler | Coordinate | Cells | Nonidentical | Minimum | Median | Maximum | +|---|---:|---:|---:|---:|---:|---:| +| normalized score | .25 | 120 | 120 | .440800 | .583608 | .695504 | +| normalized score | .50 | 120 | 120 | .241637 | .328782 | .505132 | +| normalized score | .75 | 120 | 120 | .119339 | .168475 | .245161 | +| objective matched | .25 | 120 | 32 | .000000 | .000000 | .079757 | +| objective matched | .50 | 120 | 120 | .070744 | .148500 | .233846 | +| objective matched | .75 | 120 | 120 | .073525 | .309296 | .575863 | + +Neither ruler is globally degenerate. The normalized ruler changes every +endpoint allocation. The objective-matched ruler changes 272 of 360, but 88 of +120 low-objective-coordinate cells have exactly the same point and full-upper +allocation. + +## Scientific consequence before outcomes + +The 88 allocation-identical objective-matched `.25` month cells necessarily +have zero monthly payoff, default, and miscoverage contrasts under any outcome +vector. They do **not**, however, make a 15-month window contrast zero: each of +the eight windows has exactly four nonidentical `.25` months, with the same +outcome-free distance pattern. The locked promotion rule is evaluated at the +window aggregate, so V1c alone neither establishes nor rules out a common +nonzero aggregate sign. + +This is not a reason to delete `.25`, select `.50`/`.75`, or report only the +four active months. It shows that the low objective coordinate has a sparse +decision mechanism and that CRPTO's effect depends on where the common +efficient frontier is evaluated. V2 is necessary to compute each complete +15-month window contrast, its sharp common-outcome bounds, and disagreement +between rulers. Structural monthly zeros remain part of every aggregate. + +## Frozen sources + +- summary: + `models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/normalized_objective_frontier_summary.json`; +- freeze: + `models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/protocol_freeze.json`; +- data pointer: + `data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc`; +- model pointer: + `models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc`. diff --git a/docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md b/docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md new file mode 100644 index 0000000..47cd5ef --- /dev/null +++ b/docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md @@ -0,0 +1,89 @@ +# IJDS normalized/objective frontier V2 outcome-evaluation protocol + +## Status + +This protocol is locked after the completed V1c outcome-free freeze and before +the first V2 outcome join. It evaluates the allocations already frozen under +`protocol/ijds-normalized-objective-frontier-2026-07-13-v1c`; it cannot refit a +model, recalibrate a residual recipe, resolve a new LP, or select a favorable +window, ruler, coordinate, gamma, month, or metric. + +V2 is retrospective, not a pristine lockbox, a conformal repair, or a +submission freeze. The active manuscript remains V4 until V2 is verified and a +separate promotion decision reconciles all evidence. + +## Immutable source + +Before loading any outcome, V2 must verify the exact V1c freeze descriptor: + +- path: + `models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/protocol_freeze.json`; +- bytes: `15192`; +- SHA-256: + `7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185`; +- protocol commit: `46f4df915d38eb5a6cc144484c6e6fe56d8ed397`; +- status: `outcome_free_frontiers_frozen_before_archive_outcome_join`. + +Every V1c outcome-free artifact descriptor, schema, row count, and empty +outcome-column declaration must also match. V2 stops before reading the raw +archive if any check fails. + +## Single outcome join + +After freeze verification, V2 loads the raw archive with the parent V4 temporal +contract and materializes `snapshot_default` plus `snapshot_resolution` once. +It verifies unique IDs and exact role/period alignment for the 94,885 +policy-development and 376,890 primary-OOT candidates. Observed outcomes must +be binary; unresolved outcomes remain missing and are never imputed. + +The shared outcomes are joined many-to-one to all 622,455 positive funded rows. +Every one of the 6,240 fixed portfolios is evaluated with the existing coherent +payoff, funded-default, and binary-miscoverage identities. Development results +are diagnostics; primary contrasts use all fifteen OOT monthly menus. + +## Endpoint contrasts + +The only primary pair is `gamma=1 - gamma=0`. Interior gammas remain frozen +diagnostics and cannot be selected. + +V2 reports: + +- 720 monthly contrasts: + `8 windows x 15 months x 2 rulers x 3 coordinates`; +- 48 complete-window contrasts: + `8 windows x 2 rulers x 3 coordinates`; and +- 144 metric-direction rows: + `48 window contrasts x 3 metrics`. + +For each pair, V2 forms the union of loans funded by either endpoint. One +unresolved loan receives one common binary outcome when deriving the sharp +lower and upper bound. Separate worst cases for the two policies are forbidden. +Standardized payoff is reported in dollars and per-dollar rate; default and +miscoverage are exposure-weighted rates. + +## Direction and promotion + +A bound is positive only when its lower endpoint exceeds its locked metric +tolerance, negative only when its upper endpoint is below the negative +tolerance, exact zero when both endpoints lie within tolerance, and otherwise +crosses zero. + +For each metric, a universal descriptive direction exists only if all 48 +predeclared window/ruler/coordinate bounds have the same nonzero direction. +Any zero, crossing bound, or direction disagreement defeats that condition. No +majority vote or favorable subset is reported as a universal direction. + +Even a universal direction would not restore candidate coverage, funded-set +validity, exchangeability, or a conformal portfolio guarantee. A positive +policy-superiority narrative would additionally require higher realized +standardized payoff and non-higher funded default across the complete support, +then a separately locked rolling-origin challenger. V2 itself cannot promote a +winner. + +## Outputs and stops + +All evaluated portfolios, outcome-joined allocations, monthly contrasts, +window contrasts, metric directions, and outcome-join census are retained. V2 +stops on a source descriptor mismatch, nonbinary observed outcome, ID or +role/period misalignment, missing portfolio, incomplete contrast grid, or +attempted overwrite. A stopped run is reported rather than repaired in place. diff --git a/docs/research/ijds_policy_support_tie_audit_protocol_2026-07-12.md b/docs/research/ijds_policy_support_tie_audit_protocol_2026-07-12.md new file mode 100644 index 0000000..0860f33 --- /dev/null +++ b/docs/research/ijds_policy_support_tie_audit_protocol_2026-07-12.md @@ -0,0 +1,131 @@ +# IJDS Policy-Support and Solver-Tie Audit Protocol + +## Status and claim boundary + +This protocol is locked before executing the complete audit under tag +`protocol/ijds-policy-support-tie-audit-2026-07-12-v1`. It is a post hoc +structural audit of already inspected V4 decision objects, not confirmation and +not a new empirical policy search. It reads no realized outcome, default, +repayment, status, or miscoverage column. + +The audit may justify, narrow, or reject the interpretation of the current +policy family and comparator supports. It cannot select a winning policy, +change a realized direction, establish selected-set coverage, or promote an +endpoint because it looks favorable. + +## Questions + +1. Are `tau={0.15,0.17,0.19}` and `gamma={0.25,0.50,0.75}` feasible and + decision-active over all declared development and primary monthly menus? +2. What happens at the omitted semantic endpoints `gamma=0` and `gamma=1`? +3. Which cap-month pairs are actually covered by named C0/C1/C2 rules, + development-support endpoints, broad stress endpoints, and exact + period-specific basis breakpoints? +4. Does any evaluated point LP have a nonbasic near-zero reduced cost or a + degenerate primal basis, and can changing deterministic column order change + its allocation while preserving the primary objective? + +## Outcome-free source contract + +The parent is the immutable V4 V1 outcome-free freeze at commit `2f8a760`. Its +freeze descriptor, scores, residual recipes, solve records, comparator support, +and exact frontier artifacts are hash-verified before use. + +The raw CSV is read only for `id`, `loan_amnt`, `int_rate`, and `purpose`. +Membership, issue month, design role, and point PD come from the frozen score +artifact. Any column name containing `status`, `outcome`, `default`, `pymnt`, +`realized`, or `miscoverage` is forbidden in the decision panel. The runner +must report an empty outcome-column list. + +## Policy-family domain + +For each of eight residual windows, 11 policy-development months, 15 primary +months, and `gamma` in `{0,.25,.50,.75,1}`, define + +`q_gamma = p + gamma * (U-p)`. + +At a fixed budget and purpose cap, let `q_min` be the minimum achievable funded +score and `q_obj` the funded score of the unconstrained coherent-payoff +maximizer. Both are solved exactly without outcomes. Each fixed `tau` is then +classified as: + +- `infeasible` if below `q_min`; +- `minimum_boundary` if equal to `q_min` within `1e-8`; +- `decision_active` if strictly between `q_min` and `q_obj`; +- `objective_boundary` if equal to `q_obj` within `1e-8`; or +- `objective_slack` if above `q_obj`. + +Every feasible cap is solved and its score, budget, and coherent plug-in +objective are recorded. The inherited 1,872 interior cells are reconciled to +the frozen V4 solve records within USD `1e-4` in objective and `1e-8` in funded +effective score. + +`gamma=0` is the point-score endpoint already represented by C0 and is not +renamed a conformal guardrail. `gamma=1` is the full upper-endpoint diagnostic. +Neither endpoint is automatically added to the paper-facing family. The audit +must first expose feasibility, binding, informativeness, and duplication. + +## Comparator-support taxonomy + +The comparator is part of the estimand. This audit preserves three distinct +support roles: + +1. **Named rules:** C0 copies a numeric cap, C1 transports the mean development + funded point moment, and C2 matches the contemporaneous guardrail point + moment. They are not interchangeable votes. +2. **Development-transport support:** the minimal interval containing all 11 + development funded point moments for each window-policy pair. It is + outcome-free but conditional on that design period. +3. **Broad stress support `[.05,.12]`:** a sensitivity domain, not a normative + or universally admissible comparator set. + +The cap census is the tolerance-deduplicated union, by primary month, of all +named caps, all development-support endpoints, both broad-stress endpoints, +and every period-specific HiGHS basis breakpoint. Its cardinality is derived, +not chosen after diagnostics. + +## Basis and tie diagnostics + +At every cap in the union, the point-PD LP is solved with exact budget and one +thread. The audit records: + +- nonbasic lower/upper counts and minimum absolute and scaled reduced cost; +- number of nonbasic reduced costs within `1e-7` of zero; +- basic structural variables at a bound within `1e-9`; +- basic inequality-row slacks at zero within `1e-9`; +- dual-sign violation, objective reconciliation, cap slack, and basis range. + +A near-zero nonbasic reduced cost is a necessary warning for alternate optima; +primal degeneracy alone is not proof of one. Every warned or primal-degenerate +cap is rerun after sorting columns by descending loan ID. The exposure distance +is `L1/(2B)`. An objective difference above USD `1e-5` is a solver failure. A +nonzero exposure distance with reconciled objective is reported as tie-sensitive +and would require outcome envelopes or an explicit tie rule before manuscript +use. + +## Stop and interpretation rules + +1. Stop on source-hash mismatch, missing IDs, duplicate IDs, unexpected design + months, or any forbidden decision column. +2. Stop if an inherited interior solve cannot be reproduced within tolerance. +3. Report every endpoint, cap class, basis diagnostic, and order rerun. +4. Do not redesign `tau`, `gamma`, support, or tolerances after seeing this run. + Any normalized-cap challenger requires a new tagged protocol. +5. An outcome-free active range demonstrates computational relevance, not + empirical benefit. +6. Absence of a tie warning over the finite census supports deterministic + stability only at that census; it does not prove uniqueness over every real + cap. +7. If `gamma=1` is infeasible or uninformative, explain the endpoint omission. + If it is feasible and distinct, retain it as a declared sensitivity or give + a substantive reason for excluding it. +8. Broad-stress results must remain labeled sensitivity, while development + support remains conditional identification under a transported design range. + +## Required outputs + +- one row for every window-role-month-gamma-tau family cell; +- one row for every deduplicated primary cap-month basis diagnostic; +- one row for every triggered column-order sensitivity solve; +- a deterministic summary with source and implementation hashes; +- an execution receipt with empty protected-stage and protected-artifact lists. diff --git a/docs/research/ijds_policy_support_tie_results_2026-07-12.md b/docs/research/ijds_policy_support_tie_results_2026-07-12.md new file mode 100644 index 0000000..5a450c2 --- /dev/null +++ b/docs/research/ijds_policy_support_tie_results_2026-07-12.md @@ -0,0 +1,75 @@ +# IJDS Policy-Support and Solver-Tie Audit Results + +## Status + +The outcome-free audit completed under `protocol/ijds-policy-support-tie-audit-2026-07-12-v1` at commit +`115eaf1`. It read only ID, amount, contractual rate, purpose, +frozen design role, point score, and frozen conformal recipes. No outcome column +entered a solve. These results are pre-freeze structural evidence and do not by +themselves promote a policy or empirical direction. + +## Policy-family domain + +- The audit retained 3,120 cells: eight windows, 26 months, five gamma levels, + and three fixed risk tolerances. +- All 1,872 inherited interior cells were feasible; 1,846 were decision-active. + The 26 slack cells all occur at `gamma=.25` in W8. +- `gamma=0` is objective-slack in all 624 cells under + `tau={.15,.17,.19}`. It is correctly treated as a point-score nesting + control, not an uncertainty-aware policy. +- `gamma=1` is feasible and decision-active in all 624 cells. Relative to + `gamma=.75` on the same menu and cap, its plug-in objective is lower in all + 624 cells, by a mean of `-2519.44` + and a range from `-8337.85` to + `-579.98` plug-in objective dollars per + monthly USD 1 million budget. +- Parent V4 scores and objectives reconcile to + `2.220e-16` and + `8.731e-11`. + +The endpoint result means the current nine-policy family is computationally +active but semantically incomplete. The next specification must either include +`gamma=1` as a complete-family sensitivity or replace fixed caps with a tagged +normalized-stringency design that includes both endpoints. Silent omission is +no longer defensible. + +## Comparator support + +The tolerance-deduplicated union contains `7297` cap-month pairs in +15 primary months. The earlier exploratory statement of 2,249 solves was not a +complete census; the correct named unique count is +`2204` and the full union also includes support +endpoints and 2,952 period-specific basis breakpoints. + +- All 45 C0 cap-months are objective-slack for point PD. +- C1 has 1,079 active and one slack cap-month. +- C2 has 1,075 active and four objective-boundary cap-months. +- Every lower development endpoint is active; six upper endpoints are slack. +- Broad `.05` is active in every month, while broad `.12` is slack in every + month. `[.05,.12]` is therefore a stress interval spanning active and slack + regions, not a normative admissible support. + +## Solver ties + +There are `2941` primal-degenerate bases, mostly +because basis breakpoints are transition points. None has a nonbasic reduced +cost within `1e-7` of zero; the minimum absolute nonbasic reduced cost is +`0.000387573`. All +`2941` triggered caps were rerun after reversing loan-ID +order. Zero were tie-sensitive; maximum exposure distance was +`1.450e-14` and maximum absolute objective +difference `1.717e-09`. + +Thus primal degeneracy does not explain the portfolio directions in this finite +census. This supports deterministic stability at the evaluated caps, not a +universal uniqueness theorem over every real cap. + +## Required next decision + +Do not freeze the current family. The highest-value challenger is an +outcome-free normalized stringency parameter +`lambda=(q_cap-q_min)/(q_obj-q_min)` over both score endpoints. It directly +addresses the all-slack point endpoint and the arbitrary cross-score meaning of +one numeric tau. It must be separately tagged and reported whether it strengthens +or weakens the V4 conclusion. A simpler fallback is to add `gamma=1` to the +fixed-cap sensitivity and retain the exact support caveat. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v2_protocol_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v2_protocol_2026-07-15.md new file mode 100644 index 0000000..1d78f20 --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v2_protocol_2026-07-15.md @@ -0,0 +1,60 @@ +# IJDS Portfolio-Structure Sensitivity V2 Protocol + +## Question + +Does the active conclusion that portfolio direction is not universally +favorable depend on the stylized LGD, the purpose-concentration limit, or the +monthly budget? + +## Retrospective Boundary + +The archive, active endpoint results, baseline binding diagnostics, and one +failed V1 structural execution were already inspected. This is a complete +retrospective assumption sensitivity. It is not preregistration, prospective +confirmation, model selection, policy selection, or scenario selection. + +V1 stopped before writing an artifact in scenario `b0500k_p020_l025`. A +separate outcome-free diagnostic found 25 exact-minimum failures among 600 +window-month-gamma cells, all in October 2016. The menu had USD 1.2 million of +purpose-capped capacity for a USD 0.5 million budget, and every failed cell +resolved after adding `1e-14` to the score cap. The failure was therefore an LP +boundary reconciliation issue, not evidence that the economic constraints +were infeasible. + +## Locked Numerical Amendment + +- First solve the exact minimum cap without adjustment. +- Retry only `Infeasible` or budget-reconciliation failures at that exact + minimum endpoint. +- The sole retry is `minimum_score + 1e-12`. +- The retry is forbidden for every other exception and is 10,000 times smaller + than the locked `1e-8` cap-residual tolerance. +- Persist every retry indicator, applied slack, and achieved cap residual for + all five gamma states, including the three states used to construct a ruler + but not reported as endpoint policies. +- Stop if a retried endpoint exceeds the cap-residual tolerance. + +No outcome was used to choose or tune this numerical amendment. + +## Locked Outcome-Free Grid + +- Budgets: USD 0.5, 1, and 2 million. +- Maximum purpose shares: 0.20, 0.25, 0.30, and 1.00. +- LGD: 0.25, 0.45, and 0.65. +- Report the complete 36-cell Cartesian product. +- Preserve all five gamma states when constructing each common ruler range, + but materialize only the frozen `gamma=1` versus `gamma=0` endpoint contrast. +- Preserve both rulers, coordinates 0.25/0.50/0.75, all eight windows, and all + fifteen primary-OOT monthly menus. +- Rebuild allocations without any outcome column. Join the declared six-month + endpoint only after every scenario artifact is hash frozen. + +## Interpretation Rule + +No scenario may be promoted. A conclusion is structurally robust only if it is +supported by the complete grid. Standardized payoff remains a stylized +one-period payoff, not cash-flow return, IRR, NPV, welfare, or a lending +recommendation. Removing the purpose cap is a stress specification, not a +deployable policy recommendation. + +Required tag: `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v2`. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v3_execution_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v3_execution_2026-07-15.md new file mode 100644 index 0000000..6f7787e --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v3_execution_2026-07-15.md @@ -0,0 +1,23 @@ +# IJDS Portfolio-Structure Sensitivity V3 Execution Amendment + +V3 inherits every scientific choice, numerical tolerance, claim boundary, and +stop rule from the locked V2 protocol. No outcome was inspected to make this +amendment. The only change is execution architecture. + +- Spawn ten independent workers on the 12-core host. +- Keep each HiGHS solve at one thread. +- Read the 1.77 GB raw archive once in the parent, persist its explicit + outcome-free decision allowlist as a hash-recorded Parquet artifact, and + load that compact artifact once per worker. +- Assign each complete scenario to exactly one worker. +- Write each scenario only to its own directory. +- Consolidate artifacts in sorted scenario-ID order after all 36 workers' + tasks succeed. +- Preserve the hard no-overwrite rule and stop on any worker exception, + identity mismatch, missing scenario, or hash mismatch. + +Parallel scheduling cannot select a scenario or change a scenario's inputs, +solver settings, validation, output schema, or hash. The scientific protocol +remains the V2 protocol. + +Required tag: `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v3`. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v4_recovery_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v4_recovery_2026-07-15.md new file mode 100644 index 0000000..1b8e8a4 --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v4_recovery_2026-07-15.md @@ -0,0 +1,29 @@ +# IJDS Portfolio-Structure Sensitivity V4 Recovery Amendment + +V4 inherits the complete scientific and numerical protocol from V2 and the +parallel execution architecture from V3. The V3 parent process was externally +interrupted before consolidation, after 29 scenario directories had completed +and before any outcome join or `protocol_freeze.json` write. Seven scenario +directories were absent. No result from a structural outcome evaluation +existed or was inspected. + +V4 may recover only the 29 scenario identities locked in the V4 configuration. +For every recovered scenario it must verify: + +- the exact seven-file Parquet inventory; +- the locked row counts for all fixed-cardinality artifacts; +- one and only one matching scenario ID in every artifact; +- absence of endpoint/outcome columns; +- retry slack in `{0, 1e-12}` and cap residual at most `1e-8`; +- fresh SHA-256 descriptors for source and destination files. + +Only after those checks may V4 create same-volume NTFS hard links in a fresh +V4 run directory. Hard links preserve bytes but do not make an unvalidated V3 +directory evidence. V4 recomputes the seven absent scenarios with seven +single-thread workers, independently inspects them under the same contract, +and writes one consolidated freeze only after all 36 identities are present. + +This amendment changes neither a portfolio input nor a solver call. It cannot +select a scenario, model, ruler, coordinate, metric, or endpoint. + +Required tag: `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v4`. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v5_protocol_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v5_protocol_2026-07-15.md new file mode 100644 index 0000000..8b68925 --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v5_protocol_2026-07-15.md @@ -0,0 +1,32 @@ +# IJDS Portfolio-Structure Sensitivity V5 Numerical Amendment + +V5 inherits the scientific grid from V2, parallel execution from V3, and shard +validation/recovery from V4. V4 validated and recovered 29 V3 shards, completed +four additional shards, and stopped before consolidation when one USD 0.5 +million, 20%-purpose-cap scenario returned HiGHS model status `Unknown` at an +exact minimum-score endpoint. No structural outcome evaluation existed or was +inspected. + +An outcome-free diagnostic covered all 120 October 2016 +LGD-window-gamma endpoint cells for the three missing scenarios: + +- LGD 0.25: 25 of 40 exact endpoints initially failed as `Infeasible` or by + budget reconciliation; all 25 solved at `minimum_score + 1e-12`. +- LGD 0.45 and 0.65: all 80 exact endpoints returned `Unknown` at zero and + `1e-12`; all 80 solved at `minimum_score + 1e-10`. +- Every recovered solve filled the USD 500,000 budget to the locked tolerance. + Its score residual was no larger than the applied slack. + +V5 therefore replaces the single retry with the closed ladder +`[1e-12, 1e-10]`. It may advance only after `Infeasible`, `Unknown`, or the +already recognized budget-reconciliation failure at the exact minimum +endpoint. Any other error stops immediately. Failure at `1e-10` also stops. +The largest permitted retry remains 100 times smaller than the locked `1e-8` +cap-residual tolerance, and every applied step is persisted per +window-month-gamma cell. + +V5 may recover the 33 complete V4 shards only after the full V4 validation +contract. It recomputes exactly the three locked USD 0.5 million, 20%-cap +scenarios and consolidates only after all 36 scenario identities pass. + +Required tag: `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5`. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v5_stop_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v5_stop_2026-07-15.md new file mode 100644 index 0000000..6a69c2b --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v5_stop_2026-07-15.md @@ -0,0 +1,71 @@ +# IJDS Portfolio-Structure Sensitivity V5 Stop + +## Decision + +V5 is a stopped outcome-free run, not a structural-sensitivity result. It must +not be evaluated, cited as a completed grid, or registered as active paper +evidence. + +## What completed + +- The run started from tagged commit + `481528a6973eded94189435957f9b8064bc4bb06` under + `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5`. +- All 33 V4 shards named by the locked recovery complement passed inventory, + identity, outcome-column, row-count, retry-ladder, and cap-residual checks. +- The new `b0500k_p020_l045` and `b0500k_p020_l065` shards completed and pass + the same checks. Each used 40 declared minimum-endpoint retries at `1e-10`; + its largest absolute minimum-cap residual is approximately `1.000001e-10`, + below the locked `1e-8` tolerance. +- The only absent scenario directory is `b0500k_p020_l025`. + +Thus 35 scenario shards are physically complete, but no 36-scenario protocol +freeze exists. The two newly completed shards are outcome-free checkpoint +material only; their presence does not promote V5. + +## Stop trigger + +The missing scenario reached all eight residual windows and then failed the +locked order audit with: + +```text +RuntimeError: ID reversal changed a primary endpoint allocation. +``` + +The failure arose inside `_validate_complete_build` before the scenario could +be written. V5 therefore produced neither `protocol_freeze.json` nor any +structural outcome evaluation. No endpoint outcomes were joined and no +protected stage or protected artifact was touched. + +## Required diagnosis before any successor + +A successor may not merely increase the order-exposure tolerance. It must first +measure the failing cells, objective and score reconciliation, exposure +distance, and whether the difference reflects solver-scale noise or a genuinely +nonunique optimal allocation. If the allocation is nonunique, the successor +must define an outcome-free policy convention or a set-valued evaluation that +accounts for that ambiguity. Any amendment must use a fresh protocol, run tag, +and complete reporting; V5 remains immutable stopped provenance. + +## Completed outcome-free diagnosis + +The required rerun completed all eight windows for the missing scenario while +temporarily bypassing only the final order validator. It joined no outcome +columns and produced the following reconciliation over 1,440 order-audit rows: + +- maximum normalized exposure distance: `1.3504744038073114e-10`; +- rows above `1e-10`: `8`, all repetitions of the same October 2016, + normalized-score, coordinate-0.25, gamma-0 endpoint across the eight residual + windows; +- rows above `1e-8`: `0`; +- maximum absolute plug-in objective difference: `$0.000007320079021155834`; +- maximum absolute weighted-score difference: `3.62847252244336e-12`. + +The exposure discrepancy is therefore below the already locked `1e-8` +cap-residual tolerance, the objective discrepancy remains below the parent's +`1e-5` dollar order tolerance, and the score reconciliation is at floating-point +scale. This supports a numerical-tolerance diagnosis rather than materially +different optimal allocations. A successor may set the order-exposure +tolerance to `1e-8`, but must keep the objective tolerance unchanged, recover +only fully validated V5 shards, recompute the missing scenario, and retain the +complete 36-scenario grid. This diagnosis does not complete or promote V5. diff --git a/docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md b/docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md new file mode 100644 index 0000000..22a4ee7 --- /dev/null +++ b/docs/research/ijds_portfolio_structure_sensitivity_v6_protocol_2026-07-15.md @@ -0,0 +1,50 @@ +# IJDS Portfolio-Structure Sensitivity V6 Order-Tolerance Amendment + +V6 inherits the complete 36-scenario scientific grid from V2, the deterministic +worker architecture from V3, the validated shard-recovery contract from V4, +and the closed minimum-endpoint retry ladder from V5. It changes one numerical +validator only: normalized exposure distance under ID-order reversal increases +from `1e-10` to `1e-8`. The plug-in objective tolerance remains exactly +`USD 1e-5`. + +## Outcome-free basis for the amendment + +V5 physically completed 35 scenario shards but stopped before consolidation +and before any endpoint join. Its missing `b0500k_p020_l025` scenario reached +all eight residual windows and failed one order-invariance check. A diagnostic +reran that scenario while bypassing only the final validator and joined no +outcome columns. Across 1,440 order-audit rows it found: + +- maximum normalized exposure distance `1.3504744038073114e-10`; +- eight rows above `1e-10`, all repetitions of the same October 2016, + normalized-score, coordinate-.25, gamma-0 endpoint; +- zero rows above `1e-8`; +- maximum absolute plug-in objective difference + `USD 0.000007320079021155834`, below `USD 1e-5`; +- maximum absolute weighted-score difference `3.62847252244336e-12`. + +These discrepancies are solver-scale numerical variation, not evidence of a +materially different optimal allocation. The amended exposure tolerance equals +the already locked cap-residual tolerance and remains roughly 74 times larger +than the diagnosed maximum. The objective check is not relaxed. + +## Recovery and stop contract + +V6 may recover exactly the 35 physical V5 shards after checking their complete +seven-file inventories, fixed row counts, scenario identities, retry slacks, +cap residuals, hashes, and absence of outcome columns. It recomputes only +`b0500k_p020_l025` with one single-thread worker. It writes a consolidated +outcome-free freeze only after all 36 scenario identities pass. + +V6 stops if the recomputed scenario exceeds `1e-8` normalized exposure +distance, exceeds `USD 1e-5` objective difference, fails the closed endpoint +retry ladder, contains an outcome column, or leaves any grid cell incomplete. +Only a complete hash-frozen grid may proceed to the separately invoked +evaluation phase. Evaluation must report every declared scenario; it cannot +select a budget, purpose cap, LGD, ruler, coordinate, metric, or policy. + +This is a retrospective numerical amendment informed by an outcome-free +failure diagnosis. It is not preregistration, confirmation, or a new paper +claim, and it does not create a submission freeze. + +Required tag: `protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6`. diff --git a/docs/research/ijds_raw_data_contract_protocol_2026-07-13.md b/docs/research/ijds_raw_data_contract_protocol_2026-07-13.md new file mode 100644 index 0000000..5fd6b62 --- /dev/null +++ b/docs/research/ijds_raw_data_contract_protocol_2026-07-13.md @@ -0,0 +1,47 @@ +# IJDS full-archive data-contract audit protocol + +**Status:** declared descriptive audit; no model, learner, policy, window, or +comparator selection. + +## Research question + +The raw Lending Club file contains every accepted-loan record available in the +local 2007--2020Q3 archive, but rows and columns are not interchangeable across +scientific roles. This audit asks which information is legitimately usable at +origination and at the March 31, 2016 information cutoff of the active IJDS +estimand. + +## Locked rules + +1. Scan the complete DVC-tracked CSV, not a sample. +2. Preserve the active 36-month contract and status-independent membership. +3. Report every active fitting, policy, OOT, maturity-gap, extension, and + post-extension cohort. +4. A raw candidate is eligible for the current temporal model only when it is + not post-outcome, identifier/free text, or geography, and has at least 95% + coverage in PD development, probability calibration, and conformal fitting. +5. A feature is labeled late-schema when fitting coverage is below 50% while + primary-OOT coverage is at least 80%. +6. `loan_amnt` and `funded_amnt` are reconciled by cohort. Neither definition + may be chosen after inspecting portfolio outcomes. +7. Rows in the 2014--March 2016 maturity gap are not promoted as ordinary + binary training labels merely because their final 2020 status is known. + Label availability is evaluated at the declared 2016 cutoff. +8. No protected stage or manifest-protected artifact is executed or written. + +## Persisted evidence + +The audit stores complete archive row, ID, date, term, and loan-status +inventories; the exhaustive 36-month cohort partition; feature coverage by +cohort; the leakage and temporal-availability feature contract; requested +versus funded amount reconciliation; and label availability in the maturity +gap at the declared cutoff. + +## Interpretation boundary + +More raw rows or columns do not imply a more complete estimand. Late bureau +fields can support a later-vintage study, but imputing their total absence in +the early training block would not create a valid stronger version of the +active model. Likewise, resolved 2014--2016 loans form a duration-selected +subset at the information cutoff; using them as ordinary terminal labels would +reintroduce the maturity bias removed from the active paper. diff --git a/docs/research/ijds_rebaseline_2026-06-07.md b/docs/research/ijds_rebaseline_2026-06-07.md deleted file mode 100644 index 2a23c76..0000000 --- a/docs/research/ijds_rebaseline_2026-06-07.md +++ /dev/null @@ -1,85 +0,0 @@ -# IJDS formal rebaseline - 2026-06-07 - -This memo records the formal CRPTO rebaseline for the IJDS paper lane. It supersedes the exploratory dependency-upgrade memo from 2026-06-06 as the active paper-facing baseline. - -## Decision - -The official run tag is now `ijds-rebaseline-2026-06-07`. - -The promoted policy remains `bound_aware_276k_economic_champion`. The rebaseline did not reopen the forbidden 276k champion search; it replayed and revalidated the already selected policy family inside this standalone repository, then regenerated paper/book artifacts from local paths. - -## Champion metrics - -| Metric | Value | Reading | -| --- | ---: | --- | -| Robust realized return | `$170,464.54` | unchanged headline economic result | -| `V(alpha=0.01)` | `0.028875` | improved versus the old `0.03645` baseline | -| `Gamma_CP(alpha=0.01)` | `0.187987` | slightly wider conformal budget than the old `0.18591` value | -| Exact violation | `0.0` | exact funded-set pass | -| Robust region | `45/45` | full mini-grid remains alpha-safe | -| Loan-level export count | `335` | paper-facing funded-set rows in Table A7 | -| Solver funded count | `340` | HiGHS/re-solve count convention with fractional thresholding | - -The 335 vs 340 difference is a counting convention, not a portfolio-composition change. - -## Conformal gate semantics - -Kupiec and Christoffersen p-value checks are retired from the official promotion gate. They remain useful research utilities in `src.evaluation.coverage_tests`, but they are no longer serialized into `models/conformal_policy_status.json` or counted in `overall_pass`. - -| Check family | Current role | -| --- | --- | -| Coverage 90/95 | promotion gate | -| Minimum group coverage | promotion gate | -| Average width | promotion gate | -| Backtest alerts | promotion gate | -| Winkler 90/95 | promotion gate | -| Kupiec / Christoffersen | research diagnostics outside promotion | - -Current conformal status: - -| Field | Value | -| --- | ---: | -| `overall_pass` | `true` | -| `gate_overall_pass` | `true` | -| `strict_overall_pass` | `true` | -| Gate checks | `9/9` | -| Diagnostic checks | `0/0` | -| Coverage 90% | `0.929714` | -| Coverage 95% | `0.966388` | -| Average width 90% | `0.784230` | -| Minimum group coverage 90% | `0.918983` | -| Winkler 90% | `1.110742` | - -`strict_overall_pass` is now a compatibility alias for the material gate, not a stricter p-value lane. - -## PD layer drift - -The PD layer was replayed locally under the new run tag. The direction is benign and small: - -| Metric | Current value | Direction | -| --- | ---: | --- | -| AUC | `0.7126777846` | slightly better than the previous rounded `0.7124` | -| Brier | `0.1545907368` | essentially unchanged | -| ECE | `0.0061522936` | slightly better than the previous rounded `0.0064` | - -The paper still should not sell AUC as the contribution. The contribution remains the calibrated-PD-to-conformal-to-robust-decision chain. - -## Why formal rebaseline instead of code-only fixes - -Code-only fixes would remove stale WSL paths and noisy strict wording without updating the canonical hashes. That is cleaner in the diff, but weaker scientifically: the project would still contain old frozen artifacts that disagree with the standalone run. - -Formal rebaseline is better for IJDS because it: - -- proves the standalone repository can replay the paper-facing lane without depending on the parent WSL project; -- aligns `params.yaml`, DVC, tables, figures, book, paper text, and manifest under one run tag; -- removes the misleading 9/13 strict story from the official status artifact; -- keeps the champion search decision frozen while still validating reproducibility of the selected policy. - -## Boundaries - -The rebaseline is not a new HPO/champion-search campaign. The following remain separate, explicit future work: - -- opening `crpto.portfolio.bound_exact_eval` as a new search lane; -- changing the promoted policy family; -- replacing the conformal variant with CQR or another interval method; -- making the thesis PDF before the thesis section set and APA layout are curated. diff --git a/docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md b/docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md new file mode 100644 index 0000000..8c41668 --- /dev/null +++ b/docs/research/ijds_rolling_origin_endpoint_v3_protocol_2026-07-15.md @@ -0,0 +1,62 @@ +# IJDS Rolling-Origin Endpoint V3 Recovery Protocol + +## Status + +This protocol recovers the previously locked 2015--2017 rolling-origin audit +under the active reconstructed endpoint. The earlier rolling results and the +archive have already been inspected. V3 is retrospective error correction and +stability evidence, not preregistration, prospective validation, an independent +replication count, or submission freeze. + +Required tag: +`protocol/ijds-rolling-origin-stability-2026-07-15-v3`. Before execution, the +implementation-only V5 recovery erratum moved the fresh run to +`protocol/ijds-rolling-origin-stability-2026-07-15-v4`; the imported freeze, +endpoint contract, estimands, and stop rules are unchanged. + +## Immutable Inputs + +- The 2015 V2 failure receipt remains the complete result for that origin. Its + first five-stratum residual window has counts + `(1648, 1408, 1166, 927, 619)` against the locked minimum of 1,000. No 2015 + outcome join is permitted. +- The 2016 origin imports the active V4-v1 outcome-free freeze and is restricted + mechanically to April--June 2016. +- The 2017 origin imports the V2 outcome-free freeze from + `ijds-rolling-origin-2017-2026-07-12-v2`, SHA-256 + `e224e1ae534435d1b166a07c50fb1ce907b07d36257f37e826ee41a0cb086759`. +- Learners, hyperparameters, Platt blocks, taxonomies, residual windows, + policies, comparator supports, solver settings, budget, purpose cap, payoff, + and LGD remain unchanged at every origin. + +## Sole Evaluation Correction + +Both feasible origins use the V4 reconstructed endpoint at September 30, 2020 +and the exhaustive five-reason taxonomy. A terminal label is observed only +when its reconstructed availability date is nonmissing and no later than the +cutoff. Missing dates and dates after the cutoff remain distinct unresolved +reasons. + +The common horizon is exactly three issue months per feasible origin. Coverage +is recomputed over those candidate rows; portfolio bounds aggregate only the +same three monthly menus. No later month may rescue a primary result. + +## Complete Questions and Reporting Rules + +1. Report both learners and all eight windows for each feasible origin. +2. Report the 2015 infeasibility without relaxing the five-stratum minimum. +3. Report all nine guardrails, all declared comparator scopes, and all three + metrics for the feasible origins. +4. Call coverage failure recurrent only if every reported upper bound is below + 0.90 at both feasible origins. This is recurrence across two fitted origins, + not three-origin stability. +5. Call a portfolio direction stable only if one nonzero sign survives every + window-policy cell at both feasible origins. A crossing cell defeats that + statement. +6. Do not pool origins, calculate a vote, select an origin, search a fourth + origin, or claim selected-set validity, causality, or external validity. + +Any source-freeze mismatch, incomplete three-month census, endpoint alignment +failure, budget failure, C2 mismatch, or incomplete comparator envelope stops +the recovery. New outputs use fresh immutable run paths and do not overwrite +the V2 rolling artifacts. diff --git a/docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md b/docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md new file mode 100644 index 0000000..1fd02db --- /dev/null +++ b/docs/research/ijds_rolling_origin_stability_protocol_2026-07-12.md @@ -0,0 +1,111 @@ +# IJDS Rolling-Origin Stability Protocol + +## Status and purpose + +This document locks a retrospective stability audit before the 2015 and 2017 +outcome joins. The Lending Club archive has been inspected repeatedly, so this +is not a preregistration, a prospective validation, or an independent test set. +Its narrower purpose is to prevent the active V4 result from resting on one +calendar origin while preserving the exact maturity-safe information contract. + +Required protocol tag: +`protocol/ijds-rolling-origin-stability-2026-07-12-v1`. + +Executable configurations: + +- `configs/experiments/ijds_rolling_origin_2015_2026-07-12.yaml`; +- the existing V4 run, restricted mechanically to April--June 2016; and +- `configs/experiments/ijds_rolling_origin_2017_2026-07-12.yaml`. + +The audit cannot replace or mutate the V4 evidence manifest, the active claim +registry, `EXTRACTION_MANIFEST.json`, or any protected historical artifact. + +## Symmetric calendar design + +For calendar origin `Y` in `{2015, 2016, 2017}`, every block moves by exactly +`Y - 2016` years relative to V4: + +| Block | 2015 origin | 2016 origin | 2017 origin | +|---|---|---|---| +| PD development | through Dec 2009 | through Dec 2010 | through Dec 2011 | +| Platt/taxonomy block | Jan--Dec 2010 | Jan--Dec 2011 | Jan--Dec 2012 | +| Eight residual windows | Jan 2011--Jan 2012 | Jan 2012--Jan 2013 | Jan 2013--Jan 2014 | +| Outcome-free policy development | Feb--Dec 2012 | Feb--Dec 2013 | Feb--Dec 2014 | +| Information cutoff | Mar 31, 2015 | Mar 31, 2016 | Mar 31, 2017 | +| Common primary OOT | Apr--Jun 2015 | Apr--Jun 2016 | Apr--Jun 2017 | +| Secondary censored extension | Jul--Sep 2015 | Jul--Sep 2016 | Jul--Sep 2017 | + +The CatBoost specification, Platt method, numeric logistic control, five fixed +score strata, eight consecutive six-month residual windows, nine policies, +USD 1 million monthly budget, 25% purpose cap, LGD 0.45, payoff, comparator +definitions, exact frontier, solver tolerances, and unresolved-outcome bounds +remain unchanged. The 2016 origin is not refitted: its frozen V4 artifacts are +subselected by the three declared calendar months. + +Each origin is a different fitted model and taxonomy because only information +available under that origin enters fitting. Origins are not exchangeable +replicates and their nested training histories must not be counted as an +effective sample size of three. + +## Locked questions + +1. Does canonical five-stratum candidate coverage remain below 0.90 for every + learner and all eight windows at each calendar origin? +2. Does the binary-set phase behavior recur, or is its location origin- and + learner-dependent? +3. Does any payoff, default, or miscoverage direction remain identified over + the complete development-supported point-cap interval in every origin? +4. Do C2 moment reconciliation and plug-in objective dominance remain exact? +5. Are feasibility failures themselves origin-dependent under the unchanged + minimum group size, retention, budget, and solver rules? + +These questions are descriptive stability checks. They do not create +selected-set conformal validity, causal effects, a policy winner, investor +returns, or a superpopulation guarantee. + +## Reporting and decision rules + +- Report every feasible origin, learner, residual window, taxonomy diagnostic, + policy, metric, and declared comparator scope. Do not average away signs. +- Treat an origin that fails an unchanged protocol requirement as a reported + feasibility failure. Do not reduce the 1,000-row canonical-group floor, + retention threshold, number of groups, or number of windows after failure. +- Call below-target transport *stable across these origins* only if every + canonical aggregate upper coverage bound is below 0.90 for both learners in + all three origins. Otherwise report the exact origin/learner heterogeneity. +- Call a comparator direction stable only if its sharp envelope has the same + identified sign for every window-policy cell at every origin. A crossing or + opposite sign in one cell makes the direction unidentified over that scope. +- The April--June common horizon is primary. July--September is secondary and + must not rescue a primary result. +- No pooled p-value, vote count, preferred origin, preferred learner, preferred + residual window, or post-result policy may be introduced. +- The audit may narrow an active claim. It may expand the active manuscript + only after an explicit claim-registry update backed by a deterministic + cross-origin evidence artifact and independent reconciliation. + +## Stop rules + +1. Stop an origin before its outcome join if any label, terminal status, total + payment, or outcome-derived field reaches prediction, policy, comparator, + or frontier construction. +2. Stop and report if chronology is not exactly the table above, a residual + month retains at most 99% of labels, a taxonomy edge repeats, or a canonical + group has fewer than 1,000 observations. +3. Stop and report if any monthly budget is not filled, HiGHS is not optimal, + a C2 point-score moment misses by more than `1e-10`, or plug-in dominance + fails by more than `1e-5` dollars. +4. Do not alter any locked scientific choice after inspecting 2015 or 2017. + A coding defect may be fixed only with an erratum, a fresh run tag, and a + complete rerun of both new origins. +5. If results differ by origin, preserve the heterogeneity. Do not search for + a fourth origin or a narrower horizon to manufacture uniformity. + +## Reproducibility contract + +The protocol, configurations, generalized chronology validator, and tests must +be committed and tagged before either new outcome-free freeze. Each origin uses +an immutable run directory and a two-phase freeze/evaluate workflow. A single +aggregation script must verify protocol hashes, select the declared 2016 +months, reconcile cell keys, and emit both machine-readable evidence and a +human-readable report without editing source artifacts. diff --git a/docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md b/docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md new file mode 100644 index 0000000..71906a1 --- /dev/null +++ b/docs/research/ijds_rolling_origin_stability_v2_erratum_2026-07-12.md @@ -0,0 +1,67 @@ +# Rolling-Origin Stability V2 Execution Erratum + +## Why V2 is required + +The V1 protocol was committed and tagged at +`5aa241014134227b10746cc28507b599468b33b0` before either new outcome join. +Both V1 freeze attempts stopped before producing an outcome-free freeze: + +- the 2015 origin stopped while fitting the first canonical residual recipe + because at least one of its five fixed-taxonomy groups had fewer than the + locked 1,000 observations; and +- the 2017 origin completed model and residual fitting but stopped before any + allocation because `build_outcome_free_portfolios` contained an inherited + literal requirement for 15 primary OOT menus, while the rolling-origin + protocol declares exactly three. + +Neither attempt joined terminal outcomes, evaluated payoffs, inspected OOT +coverage, or wrote a protocol freeze. The V1 run directories are retained as +failed-run provenance and are not reused or overwritten. + +## Scientific decision + +The 2015 stop is a locked feasibility finding. V2 does not reduce the 1,000-row +minimum, pool score groups, remove a residual window, change the taxonomy, or +substitute another origin. The 2015 freeze is rerun only to persist a structured +failure receipt with the exact group counts. + +The 2017 stop is an implementation defect. V4 legitimately has 15 primary +months, but a reusable constructor must derive the expected count from the +declared design. V2 replaces only that literal with the inclusive month count +from `primary_oot_start_month` through `primary_oot_end_month`. It similarly +derives the already unchanged 11 policy-development months from their declared +dates. For the original V4 configuration the resulting expectation remains +exactly `(11, 15)`; for each rolling-origin configuration it is `(11, 3)`. + +No model, feature, seed, calibration rule, residual window, policy, payoff, +comparator, frontier, solver tolerance, outcome, reporting rule, hypothesis, or +promotion criterion changes in V2. + +## Failure receipts + +The phase runner now writes a deterministic JSON receipt when a locked phase +fails. The receipt records protocol/run tags, exception type and message, +structured protocol details when available, Git state, and an explicit empty +list of protected stages and artifacts. It is written inside the fresh model +run directory and cannot overwrite an existing receipt. + +For canonical-group infeasibility, the error now carries learner, window, +taxonomy size, all group counts, and the unchanged minimum. This improves audit +evidence but does not alter the condition that triggers the stop. + +## V2 execution contract + +Required tag: +`protocol/ijds-rolling-origin-stability-2026-07-12-v2`. + +Fresh run tags: + +- `ijds-rolling-origin-2015-2026-07-12-v2`; +- `ijds-rolling-origin-2017-2026-07-12-v2`. + +Both origins must be rerun from the freeze phase. The expected 2015 feasibility +failure must be retained and reported; it is not permission to adapt the +protocol. If 2017 passes its outcome-free freeze, only then may its evaluation +phase join outcomes under the same tagged implementation. The original V4 +origin remains the sole source for the mechanically restricted April--June +2016 comparison. diff --git a/docs/research/ijds_scientific_upgrade_audit_2026-07-07.md b/docs/research/ijds_scientific_upgrade_audit_2026-07-07.md deleted file mode 100644 index e75daa6..0000000 --- a/docs/research/ijds_scientific_upgrade_audit_2026-07-07.md +++ /dev/null @@ -1,59 +0,0 @@ -# IJDS scientific upgrade audit - 2026-07-07 - -Scope: decide which high-upside CRPTO extensions can strengthen the current -paper as framing, diagnostics, or limitations, and which would require a new -research result before promotion. - -## Decision - -The current IJDS paper should not promote a new selector, learner, conformal -protocol, or external certificate before submission. It should, however, make -the upgrade boundary explicit. This turns likely reviewer questions into a -strength of the paper: CRPTO is an auditable post-hoc decision certificate over -frozen evidence, and the supplement shows where stronger future certificates -would enter. - -## Changes applied - -- Added a body-level scientific upgrade map in `paper/CRPTO_ijds.qmd`. -- Ported the same map to the official submission `.tex`. -- Added a longer supplement section that separates paper improvements available - now from evidence required for promotion. -- Updated the active claim registry so the recommended body wording says - "selected policy" rather than "selected pool93 body point". - -## What can improve the current paper without a new run - -| Upgrade | Current-paper use | -|---|---| -| Tail-aware selection | Present A20--A22 and A37 as tail-profile and challenger diagnostics, not selectors. | -| Prospective/nested selection | Use A3, A9, and A35 denominators to answer post-selection concerns. | -| Multi-distribution validity | Use A23 to show which grade/vintage cells are stable or thin. | -| Online validity | Use A24 as a static OOT replay, not as live control. | -| Decision-focused learning | Use A19 to show the regret-auditability frontier. | -| Causal decision layer | Keep as a limitation and future protocol, not a claim. | - -## What would become a new research result - -A new result is required if the paper wants to claim any of the following: - -1. CVaR/OCE is the promoted selector rather than a diagnostic. -2. The selected policy comes from a fully prospective search/evaluation design. -3. The conformal layer targets multi-distribution, group-weighted, or online - validity as its promoted guarantee. -4. The PD model is trained end-to-end through the optimizer. -5. A causal policy effect is identified rather than a predictive/prescriptive - certificate. - -Those may be good CRPTO v2 directions. They are not safe as current-body claims -without new protocols, new outputs, and new claim-sync guards. - -## Code architecture implication - -The current codebase is strongest when treated as a frozen research pipeline: -data -> features -> PD -> conformal intervals -> robust portfolio -> exact audit --> paper outputs. Refactoring within that architecture is useful only when it -improves maintainability without changing the evidence. If Carlos wants a -simpler scientific claim, the clean path is not a broad refactor of old scripts; -it is a new tagged run with a smaller declared protocol and a side-by-side -comparison against the current selected policy. diff --git a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md b/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md deleted file mode 100644 index 209815d..0000000 --- a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md +++ /dev/null @@ -1,133 +0,0 @@ -# IJDS simplification and cleanup audit - 2026-07-06 - -> Certificate metrics in this historical cleanup memo were superseded on -> 2026-07-09 by `pool93_certificate_semantics_v2_2026-07-09.md`. The cleanup -> decisions remain valid; the active paper uses the policy-aware A35 frontier -> and matched point-PD baseline A40. - -Scope: body manuscript, IJDS submission `.tex`, online supplement, code -refactor posture, and local repository weight. This memo follows the -2026-07-05 full audit but focuses on reader-facing parsimony: remove technical -runbook language from the paper body unless it protects a claim the reviewer -must understand. - -## 1. Reader-facing prose decision - -The body paper should speak in paper language: - -- selected policy, selected decision, declared finite-grid frontier; -- frozen PD model, conformal intervals, robust portfolio decision; -- traceability, validation harness, governance files when needed. - -The body paper should avoid internal run labels unless a number cannot be -understood otherwise: - -- `pool93` as a prose label; -- A35--A39 as the main way to describe a body-level concept; -- `sidecars`, `artifact paths`, script names, test filenames, exact DVC - implementation details; -- "bit-exact" as a rhetorical claim. The scientific point is narrower: - retraining is a new research run, not the routine reproduction target. - -Where those details belong: - -- online supplement: appendix labels, table provenance, guardrails, DVC/manifest - boundary, reproduction commands; -- submission package docs: raw-data permanence, DVC sanitization, code/data - timing, ScholarOne checklist; -- code/tests/docs: exact file names and hash gates. - -## 2. Changes applied in this pass - -- Rewrote the body and `.tex` abstract/introduction language from - "selected pool93 body point" to "selected policy" where the run label did not - help the reader. -- Replaced the small Table 10 with a reviewer-question table: - "Reviewer question / Body answer / Boundary". -- Removed `\resizebox{\textwidth}{!}` from the official `.tex` Table 10 and - replaced it with wrapped `p{}` columns at `\small`, so the table no longer - shrinks to unreadable text. -- Simplified the reproducibility body paragraph: the body now says that frozen - inputs regenerate the paper surfaces, claim-sync tests protect body numbers, - and retraining is a new research run. The detailed DVC/hash/drift language - remains for the supplement and package docs. -- Softened supplement framing where it introduced A35--A39 with too much - `pool93` terminology. The detailed appendix rows now use reader-facing - `selected-policy` language while retaining source filenames where traceability - matters. -- Removed rhetorical uses of `artifact` from the body, submission `.tex`, and - supplement. The paper now uses "model", "evidence", "files", "outputs", or - "reproducibility bundle" unless a package/debug context genuinely needs - implementation language. -- Recompiled the official INFORMS-style `.tex` and visually checked page 20: - Table 10 is readable, wrapped, and ragged-right rather than compressed by - `resizebox`. - -## 3. Code/refactor assessment - -Current evidence does not support a broad code refactor before submission: - -- The 2026-07-05 audit already closed the safe refactor lanes and reported no - AI-slop/dead-code pattern in live code. -- The largest files are mostly search or paper-generation entrypoints: - `scripts/train_pd_model.py`, `scripts/generate_conformal_intervals.py`, - `scripts/search/run_portfolio_bound_aware_search.py`, - `scripts/search/run_pool93_ijds_local_refinement.py`, and - `scripts/search/run_regret_auditability_sandbox.py`. -- Splitting those files further would improve aesthetics, but it does not - simplify the IJDS claim unless the claim itself changes. - -The important distinction: - -- If the goal is "same paper claim, cleaner code", drift-gate should remain - strict. Otherwise a "refactor" can silently become a different empirical - result. -- If the goal is "simpler claim and simpler code, small metric changes allowed", - that is no longer a refactor. It is a new research run and should get a new - run tag, a simpler claim target, and a new evidence bundle. - -Recommended new-run shape if Carlos chooses that route: - -1. Declare a simpler target before running: for example, "one selected policy - family, one alpha level plus sensitivity grid, no pool93 local-refinement - narrative in the body." -2. Write outputs under a new experiment path; do not overwrite the current - frozen A35--A39 or model/conformal artifacts. -3. Compare against the current selected policy on: return, `V(0.01)`, - `Gamma_CP`, `Gamma_res`, exact loss threshold, realized risk-tolerance - excess, table/page simplicity, and code-path length. -4. Promote only if the paper becomes materially simpler or the result is - materially easier to defend. A few basis points of metric loss may be fine, - but only if the new claim is easier to explain. - -## 4. Local repository weight - -Before cleanup the workspace was roughly 24.8 GB. Safe cleanup performed: - -- removed `.venv-champion-search` (~6.0 GB); -- removed `.venv-tabpfn` (~4.9 GB); -- removed local caches: `.mypy_cache`, `.pytest_cache`, `.ruff_cache`, - `.hypothesis`, `.codex_tmp`; -- ran `git gc`, reducing `.git` from ~1.66 GB of loose objects to ~30 MB. - -After cleanup the workspace is roughly 12 GB. Remaining expected weight: - -| Path | Approx. size | Recommendation | -|---|---:|---| -| `data/` | 5.0 GB | Keep for local reproduction unless DVC remote availability is verified. | -| `.dvc/` | 2.8 GB | Keep for offline DVC cache; run `dvc gc` only after deciding what historical outputs can be dropped. | -| `models/` | 1.8 GB | Keep; contains frozen model/governance/search evidence. | -| `.venv/` | 1.6 GB | Keep; active project environment. | -| `reports/` | 0.3 GB | Keep; paper tables/figures/MRM outputs. | -| `Papers_tesis/` | 0.18 GB | Optional local literature cache; ignored by Git. | - -Do not use broad `git clean -fdX` in this repo: it would delete DVC-tracked -data/model material, local templates, and ignored PDFs. - -## 5. Current recommendation - -For IJDS submission, continue with the current frozen claim but make the body -more reader-facing. The immediate acceptance risk is not code complexity; it is -overloading the body with internal labels. A new tolerant run is worth doing -only if Carlos wants to simplify the actual claim, not merely the code around -the existing claim. diff --git a/docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md b/docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md new file mode 100644 index 0000000..5372831 --- /dev/null +++ b/docs/research/ijds_two_ruler_endpoint_recovery_v3_protocol_2026-07-14.md @@ -0,0 +1,36 @@ +# IJDS Two-Ruler V3 Endpoint Recovery Protocol + +## Status + +This is an evaluation-only recovery of the normalized-score and +objective-matched two-ruler audit. The archive and earlier V2 outcomes have +already been inspected. V3 is therefore retrospective error correction, not a +new confirmation. + +## Immutable Source + +V3 imports the V1c outcome-free frontier freeze from run +`ijds-normalized-objective-frontier-2026-07-13-v1c`, SHA-256 +`7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185`. +It must reproduce 6,240 solve records and 622,455 funded rows. No learner, +recipe, ruler, coordinate, gamma, allocation, or endpoint contrast may change. + +## Sole Change + +The outcome join uses the V3 reconstructed endpoint: a terminal archive status +is observed only when its conservative label-availability date is no later than +September 30, 2020. The archive is not represented as a verified point-in-time +snapshot. V2 remains immutable provenance. + +## Stop Rules + +- Report all 48 window-level endpoint contrasts, 720 monthly contrasts, and 144 + direction cells. +- If direction counts differ from V2, retain the V3 result and disclose the + endpoint sensitivity. Do not choose a ruler or coordinate that preserves a + preferred sign. +- No winner, causal effect, selected-set validity, prospective guarantee, or + continuous-frontier claim is authorized. + +Required tag: +`protocol/ijds-normalized-objective-frontier-2026-07-14-v3`. diff --git a/docs/research/literature/fernandez_loria_provost_2022_causal_decision_making_note.md b/docs/research/literature/fernandez_loria_provost_2022_causal_decision_making_note.md deleted file mode 100644 index 7a5346d..0000000 --- a/docs/research/literature/fernandez_loria_provost_2022_causal_decision_making_note.md +++ /dev/null @@ -1,54 +0,0 @@ -# Fernandez-Loria and Provost (2022) - reading note - -Source PDF: - -- Local ignored archive: `Papers_tesis/paper/Fernandez-Loria Provost 2022 - Causal Decision Making and Causal Effect Estimation Are Not the Same and Why It Matters.pdf` -- User download checked: `C:\Users\carlos\Downloads\papers_nuevos\2104.04103v3.pdf` -- SHA-256: `72551FD3AD3FEA779BC114578680750ABBB2A619C02ECF56CB7705FB2900DF69` -- Bib key: `fernandezloria2022causaldecision` -- DOI: `10.1287/ijds.2021.0006` -- Pages inspected with PyMuPDF: 24 - -## What the paper contributes - -The paper argues that causal decision making (CDM) and causal effect estimation -(CEE) are not the same task. Its key claim is that accurate effect-size -estimation is not necessary for accurate decision making when the operational -goal is treatment assignment. In the authors' framing, the estimand changes: -the model should be judged by whether it assigns the right action, not only by -whether it estimates individual effects precisely. - -The abstract and introduction emphasize three implications: - -1. The modeling objective should, where possible, optimize treatment assignment - rather than effect-size accuracy. -2. Confounding affects decision quality differently from effect estimation - quality; confounded data can sometimes be decision-useful. -3. A non-causal proxy target can sometimes support a useful decision rule, - which explains why firms often use predictive models for intervention - targeting even when those models are not causal models. - -The paper does not mention credit, loans, Lending Club, portfolio selection, or -conformal prediction. Its value for CRPTO is conceptual rather than empirical. - -## How CRPTO should use it - -Use this source to support a narrow point in related work: prediction, effect -estimation, and downstream action are different objects. CRPTO uses the same -separation in a non-causal credit setting. The PD model is an input artifact; -the manuscript's claim is about the auditable portfolio decision and the -funded-set risk certificate. - -Do not use this source as evidence that CRPTO estimates causal treatment -effects, that Lending Club decisions are causal interventions, or that the -portfolio policy has experimental policy value. Those would require a separate -design. - -Action taken on 2026-06-14: - -- Kept `fernandezloria2022causaldecision` in the manuscript. -- Rewrote the related-work sentence so the citation supports the estimand / - assignment-rule distinction rather than a generic "prediction vs action" - slogan. -- Kept the PDF in the ignored local literature archive instead of committing a - copyrighted PDF under `docs/`. diff --git a/docs/research/literature_reference_audit_2026-06-14.md b/docs/research/literature_reference_audit_2026-06-14.md deleted file mode 100644 index 25c4fc3..0000000 --- a/docs/research/literature_reference_audit_2026-06-14.md +++ /dev/null @@ -1,98 +0,0 @@ -# Literature reference audit - 2026-06-14 - -Scope: cited references in the IJDS manuscript body, using -`paper/submission/CRPTO_ijds_submission.aux` as the current body-citation -surface. This is not a full supplement audit. - -The audit question is deliberately strict: does the repository contain evidence -that a source was actually read or curated, beyond being present in BibTeX? - -## Status labels - -- `strong`: local PDF/read-note/deep-audit evidence exists, or the source was - inspected directly in this session. -- `partial`: the source is discussed in the book or manuscript, but no standalone - read-note or deep-audit row was found. -- `citation-only`: no local reading evidence found beyond BibTeX and citation - use. These are not automatically wrong, but they should be spot-checked before - freeze/submission if they carry a specific claim. - -## Strong local reading evidence - -These body references have local reading evidence from -`docs/research/papers_tesis_deep_audit_2026-06-06.md`, the new -Fernandez-Loria/Provost note, or direct PDF inspection in this session: - -`albanesi2024credit`, `angelopoulos2023`, `angelopoulos2024foundations`, -`angelopoulos2024risk`, `bao2025croms`, `bates2021rcps`, `bental2007oce`, -`bertsimas2004`, `cresswell2024`, `donti2017`, `elmachtoub2022`, -`fernandezloria2022causaldecision`, `gibbs2024`, `hu2026crc`, -`jagtiani2019altdata`, `johnstone2021`, `liu2026portfolio`, `mandi2024`, -`patel2024`, `rockafellar2000cvar`, `schutte2024robust`, `sun2024ptc`, -`aior2025lendingclub`, `yang2026multidistribution`, `yeh2025training`, -`zhao2025robust`, `zhou2025credo`, `zhou2026creme`. - -Supplement-only check: `fernandezloria2025observational` was also inspected in -this session. It supports the supplement's causal/experimental-design boundary, -but it is not credit or loan evidence. - -## Partial evidence - -These sources are used in plausible places and are discussed in the book or -manuscript, but I did not find a standalone local read note: - -`ayari2026`, `bostrom2021`, `chen2024creditrisk`, `lessmann2015`, `vovk2005`, -`xia2017`, `yang2025costaware`, `zhou2024`. - -Recommended action before freeze/submission: spot-check the exact sentence each -one supports, especially if it claims recency, a benchmark frontier, or a credit -domain fact. - -## Citation-only / spot-check before freeze - -These references appear in the manuscript body, but I did not find local -evidence that they were read in detail: - -`boucheron2013concentration`, `das2023creditgraph`, `delage2010dro`, -`ghosh2002`, `goldfarb2003robustportfolio`, `hoeffding1963`, -`boosting2025default`, `serrano2016profitscoring`, `zhao2016p2pportfolio`, -`zheng2026twostage`. - -The most important items to verify are `hoeffding1963`, -`boucheron2013concentration`, and `ghosh2002` because they support theory or -inequality language; `goldfarb2003robustportfolio` and `delage2010dro` because -they anchor robust-optimization positioning; and `das2023creditgraph`, -`yang2025costaware`, `zheng2026twostage`, and `boosting2025default` because they -are recent IJDS/credit references that make the manuscript look current. - -## Bottom line - -The core CRPTO spine is well supported by read/curated sources: conformal -prediction/risk control, conformal robust optimization, SPO+/DFL comparison, -Lending Club fintech context, and the new decision-vs-estimation framing. - -The weaker area is not the main claim; it is citation hygiene. Several classical -or recent references are being used as positioning anchors without a local note -proving close reading. They can stay for now, but they deserve a targeted -pre-freeze source check rather than another broad literature expansion. - -## 2026-07-07 targeted pre-freeze spot-check - -The pre-freeze spot-check resolved the highest-risk citation-only items that -support the IJDS body and supplement theory/positioning language: - -| Key | Source checked | Result | -|---|---|---| -| `ghosh2002` | Taylor & Francis DOI landing page, `10.1198/000313002119`. | Metadata matches BibTeX; appropriate for Markov-related probability-inequality language. | -| `hoeffding1963` | Taylor & Francis DOI landing page, `10.1080/01621459.1963.10500830`. | Metadata matches BibTeX; appropriate for bounded-sum tightening language when independence is stated. | -| `boucheron2013concentration` | Oxford Academic book page / DOI `10.1093/acprof:oso/9780199535255.001.0001`. | Metadata matches BibTeX; appropriate as a modern concentration reference. | -| `goldfarb2003robustportfolio` | INFORMS PubsOnLine DOI landing page, `10.1287/moor.28.1.1.14260`. | Metadata matches BibTeX; appropriate for robust portfolio selection under parameter uncertainty. | -| `delage2010dro` | INFORMS PubsOnLine DOI landing page, `10.1287/opre.1090.0741`. | Metadata matches BibTeX; appropriate for DRO/moment-uncertainty positioning. | -| `zhao2016p2pportfolio` | ACM/KDD DOI landing page, `10.1145/2939672.2939861`. | Title updated in BibTeX to include the subtitle "A Multi-Objective Perspective"; appropriate for P2P portfolio-selection positioning. | -| `serrano2016profitscoring` | ScienceDirect DOI landing page, `10.1016/j.dss.2016.06.014`. | Author accents corrected in BibTeX; appropriate for P2P profit-scoring positioning. | -| `falconer2026replication` | INFORMS/IJDS DOI landing page, `10.1287/ijds.2025.0075`. | Metadata matches BibTeX; appropriate for the replication-robust IJDS precedent. | - -No broad literature expansion is recommended before submission. Remaining -recent-credit references (`das2023creditgraph`, `yang2025costaware`, -`boosting2025default`, `zheng2026twostage`) are lower-risk positioning anchors; -verify only if their exact body sentences become more specific. diff --git a/docs/research/papers_tesis_deep_audit_2026-06-06.md b/docs/research/papers_tesis_deep_audit_2026-06-06.md deleted file mode 100644 index 888ba59..0000000 --- a/docs/research/papers_tesis_deep_audit_2026-06-06.md +++ /dev/null @@ -1,317 +0,0 @@ -# Papers_tesis Deep Audit - 2026-06-06 - -## Resumen ejecutivo - -Esta auditoría cubre `61` PDFs locales en `Papers_tesis` y fue generada para el corte `2026-06-06` con `scripts/build_papers_tesis_deep_audit.py`. - -La decisión central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoría, related work, appendices y límites de claim. La agenda extendida CRPTO/tesis absorbe el material que sí pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance. - -Artefactos generados: - -- Matriz fuente: `reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv` -- Índice compacto de captions: `reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv` -- Curaduría de visual sinks: `reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv` - -## Inventario - -| folder | pdfs | -| --- | --- | -| paper | 21 | -| supplement | 28 | -| tesis | 12 | - -## Decisiones por destino editorial - -| decision | n | -| --- | --- | -| append_claim_boundary | 2 | -| append_comparator | 3 | -| append_conformal_variant | 1 | -| append_crpto_related_work | 1 | -| append_crpto_selector | 1 | -| append_extended_context | 1 | -| append_extended_governance | 4 | -| append_extended_ifrs9 | 1 | -| append_extended_source_shift | 3 | -| append_future_work | 3 | -| append_governance | 1 | -| append_mixed_diagnostic_source_not_better | 1 | -| append_mixed_diagnostic_wider_than_mondrian | 1 | -| append_tail_risk | 2 | -| context_credit_domain | 3 | -| context_crpto_related_work | 2 | -| context_decision_conformal | 1 | -| context_decision_support | 1 | -| context_finance_cp | 2 | -| context_governance | 1 | -| context_source_shift | 1 | -| park_future_dfl | 1 | -| park_future_work | 11 | -| park_width_fail | 1 | -| promote_crpto_body | 11 | -| thesis_foundation | 1 | - -## Acciones requeridas - -| action_required | n | -| --- | --- | -| completed_bib_quarto_patch | 7 | -| completed_quarto_patch | 1 | -| experiment_completed_appendix_diagnostic | 2 | -| experiment_completed_parked | 1 | -| none_now | 50 | - -## Familias conceptuales - -| primary_domain | n | -| --- | --- | -| AI/OR digital lending | 1 | -| CQR | 1 | -| CROMS / model selection | 1 | -| CVaR optimization | 1 | -| ECL / IFRS9 governance | 1 | -| ML credit governance | 1 | -| ML credit markets / equity | 1 | -| OCE / convex risk measures | 1 | -| P2P credit decision support | 1 | -| P2P credit portfolio optimization | 1 | -| P2P investment recommendation | 1 | -| SPO+ / predict-then-optimize | 1 | -| adaptive conformal inference | 1 | -| beyond exchangeability / weighted conformal | 1 | -| conditional conformal guarantees | 1 | -| conditional coverage limits | 1 | -| conformal contextual robust optimization | 1 | -| conformal decision theory | 1 | -| conformal foundations | 2 | -| conformal inverse optimization | 1 | -| conformal portfolio optimization | 1 | -| conformal portfolio selection | 1 | -| conformal risk control | 1 | -| conformal robust optimization | 1 | -| conformal robust optimization / satisficing | 1 | -| conformal robustness / decision risk | 1 | -| consumer credit ML | 1 | -| covariate shift conformal systems | 1 | -| credit data noise / disparities | 1 | -| credit invisibles / unscored consumers | 1 | -| credit scoring / equity | 1 | -| data-driven robust optimization | 1 | -| decision risk certificates | 1 | -| decision theory for conformal prediction | 1 | -| decision-aware conformal prediction | 1 | -| decision-focused learning | 1 | -| decision-focused learning survey | 1 | -| end-to-end conditional robust optimization | 1 | -| end-to-end conformal calibration for optimization | 1 | -| end-to-end conformal risk training | 1 | -| fair-lending proxy methodology | 1 | -| fintech lending / LendingClub | 1 | -| group-weighted conformal prediction | 1 | -| human decision support with CP | 1 | -| inverse conformal risk control | 1 | -| label noise / conformal robustness | 1 | -| localized conformal prediction | 1 | -| multi-distribution robust conformal prediction | 1 | -| multi-source conformal inference | 1 | -| non-monotonic conformal risk control | 1 | -| online conformal prediction | 1 | -| ordinal credit scoring / conformal prediction | 1 | -| predict-then-calibrate | 1 | -| prescriptive analytics | 1 | -| risk control / post-hoc testing | 1 | -| risk-controlling prediction sets | 1 | -| robust DFL | 1 | -| robust optimization | 1 | -| robust optimization foundations | 1 | -| weighted conformal / covariate shift | 1 | - -## Lectura integrada para Paper CRPTO - -El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son: - -| relative_path | bib_key | core_concepts | crpto_value | limitations | -| --- | --- | --- | --- | --- | -| paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf | angelopoulos2023 | Exchangeability, split conformal, prediction sets, finite-sample marginal coverage. | Core theory support for the uncertainty layer. | Introductory; not a portfolio optimizer or credit-risk paper. | -| paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf | angelopoulos2024risk | Risk control, monotone bounded losses, post-hoc calibration, user-defined loss. | Core theory support for mapping uncertainty to a decision-relevant loss. | Does not itself define a portfolio-funded-set bound. | -| paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf | angelopoulos2024foundations | Exchangeability, exact finite-sample coverage, randomized quantiles, modern CP theory. | Canonical theory reference for the proof and notation. | Too broad for the paper body; cite selectively. | -| paper/Angelopoulos et al 2025 - Learn Then Test.pdf | angelopoulos2025ltt | Learn-then-test, finite-sample risk guarantees, post-hoc risk calibration. | Supports exact risk-gate language and post-selection caveats. | Does not solve downstream optimization by itself. | -| paper/Barber et al 2021 - Limits of Distribution-Free Conditional Predictive Inference.pdf | barber2021limits | Conditional coverage impossibility, approximate conditional validity, subgroup limits. | Keeps Mondrian coverage language honest. | Boundary paper, not an implementation. | -| paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf | bates2021rcps | RCPS, set-valued prediction, expected loss control, black-box models. | Core CRPTO theory lineage. | General prediction sets; no credit portfolio objective. | -| paper/Bertsimas Sim 2004 - The Price of Robustness.pdf | bertsimas2004 | Budgeted uncertainty, protection level, tractable robust LP/IP, price of robustness. | Core RO foundation. | Budget is chosen, not conformally calibrated. | -| paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf | jagtiani2019altdata | LendingClub, alternative data, credit grades, FICO relationship, fintech underwriting. | Core empirical-setting support. | Does not validate CRPTO or fairness claims. | -| paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf | johnstone2021 | Conformal regions as uncertainty sets, robust optimization, finite-sample validity. | Core method foundation. | No credit-specific funded-set weighted bound. | -| paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf | patel2024 | Contextual robust optimization, conformal regions, generative uncertainty. | Direct neighbor in related work. | Not credit-specific and no funded-set PD constraint. | -| paper/Sun et al 2024 - Predict-then-Calibrate.pdf | sun2024ptc | Predict-then-calibrate, robust contextual LP, box/ellipsoid uncertainty sets. | Core closest-neighbor contrast. | No Lending Club and no Mondrian funded-set bound. | - -Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y límites de claim: - -| relative_path | bib_key | decision | crpto_value | figures_tables_useful | -| --- | --- | --- | --- | --- | -| paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf | chi2019p2p | append_crpto_related_work | Important applied-credit related work. | Use experimental design/tables as comparator template for Lending Club portfolio papers. | -| paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf | donti2017 | append_comparator | Comparator lineage only. | Use as historical DFL lineage, not as CRPTO method figure. | -| paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf | elmachtoub2022 | append_comparator | Core comparator, not replacement. | Use benchmark/regret tables as comparator template. | -| supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf | bao2025croms | append_crpto_selector | Appendix selector support. | Use selection diagrams/tables to structure A5/A10 appendix. | -| supplement/Ben-Tal Teboulle 2007 - Optimized Certainty Equivalent.pdf | bental2007oce | append_tail_risk | Appendix tail-risk foundation. | Use definitions, not figures. | -| supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf | mandi2024 | append_comparator | Comparator framing. | Use taxonomy table for agenda extendida CRPTO/tesis DFL suite framing. | -| supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf | rockafellar2000cvar | append_tail_risk | Appendix tail-risk diagnostic. | Use formulation, not figures. | - -## Lectura integrada para agenda extendida CRPTO/tesis - -La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO: - -| relative_path | decision | extended_lab_value | evidence_gate | stop_rule | -| --- | --- | --- | --- | --- | -| paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf | append_extended_governance | Strong appendix support for score-proxy vs champion ML and equity boundary. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf | append_extended_context | agenda extendida CRPTO/tesis digital-lending architecture context. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf | append_extended_source_shift | Strong source/shift conformal lane reference. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf | append_mixed_diagnostic_source_not_better | Potential source/shift experiment over grade/period/source families. | Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; absolute gate passed, but worst-source coverage fell 0.0111 below Mondrian. | Append as source/shift diagnostic only; do not claim group-weighted source improvement over Mondrian. | -| supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf | append_extended_governance | Strong claim-boundary source for proxy governance. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Guan 2023 - Localized Conformal Prediction.pdf | append_mixed_diagnostic_wider_than_mondrian | Potential source/shift experiment. | Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; coverage and worst-source coverage improved, but width increased 0.0867 versus Mondrian. | Append only as reviewer-facing localized diagnostic; no localized guarantee or champion replacement claim. | -| supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf | append_extended_source_shift | Source-family holdout lane. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf | append_extended_source_shift | Source/shift conformal lane. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| tesis/Basel Committee 2015 - Guidance on Credit Risk and Expected Credit Losses.pdf | append_extended_ifrs9 | Strong IFRS9/SICR proxy boundary source. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf | append_extended_governance | Strong metric/data governance context. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | -| tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf | append_extended_governance | Strong governance/fairness boundary source. | No run: literature integration only. | Do not reopen a lane unless it changes a manuscript claim. | - -## Experimentos evidence-gated - -Esta sección lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado. - -| relative_path | action_required | implementation_or_experiment | evidence_gate | artifact_sink | stop_rule | -| --- | --- | --- | --- | --- | --- | -| supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf | experiment_completed_appendix_diagnostic | Group-weighted source-max replay: coverage 0.9387, avg width 0.9394, worst defended source 0.8602. | Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; absolute gate passed, but worst-source coverage fell 0.0111 below Mondrian. | docs/research/papers_tesis_deep_audit_2026-06-06.md | Append as source/shift diagnostic only; do not claim group-weighted source improvement over Mondrian. | -| supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf | experiment_completed_parked | Utility-directed loss replay: coverage 0.9954, avg width +0.2205 versus Mondrian, worst defended source 0.9499. | Executed with fixed width/violation/tail-miss loss on frozen v4 replay; coverage rose to 0.9954 but avg width 0.9981 failed the 0.98 width gate. | docs/research/papers_tesis_deep_audit_2026-06-06.md | Park as negative width result; no utility-directed selector claim. | -| supplement/Guan 2023 - Localized Conformal Prediction.pdf | experiment_completed_appendix_diagnostic | Localized score-bin replay: coverage 0.9497, avg width 0.8643, worst defended source 0.9252. | Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; coverage and worst-source coverage improved, but width increased 0.0867 versus Mondrian. | docs/research/papers_tesis_deep_audit_2026-06-06.md | Append only as reviewer-facing localized diagnostic; no localized guarantee or champion replacement claim. | - -## Curaduría de figuras/tablas - -El índice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduría siguiente solo define qué visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers. - -| relative_path | caption_type | caption_index | editorial_sink | why_useful | claim_boundary | -| --- | --- | --- | --- | --- | --- | -| paper/Hu et al 2026 - Conformal Robustness Control.pdf | figure | 1 | agenda extendida CRPTO/tesis robustness-certificate appendix | Shows the conceptual contrast between conventional CRO and CRC-style robustness control. | Future gate over V/violation, not current CRPTO evidence. | -| supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf | figure | 2 | agenda extendida CRPTO/tesis source/shift conformal appendix | Coverage-width caption is useful for explaining why non-exchangeability gates must report both validity and efficiency. | Requires declared weighting or shift structure. | -| supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf | figure | 1 | agenda extendida CRPTO/tesis source/shift conformal appendix | Canonical weighted-conformal covariate-shift coverage caption for density-ratio caveats. | No source-shift deployment claim without credible weights. | -| supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf | figure | 3 | agenda extendida CRPTO/tesis group/source governance appendix | Compares older weighted CP guarantees to the new group-weighted guarantee. | Needs target/source group weights before any run. | -| supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf | figure | 1 | agenda extendida CRPTO/tesis multi-source conformal appendix | Illustrates multi-source calibration structure and source-combination logic. | LC source families are retrospective, not validated external sources. | -| supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf | figure | 1 | agenda extendida CRPTO/tesis utility-directed CP future gate | Contrasts standard CP with utility-directed CP in a decision-aware frame. | No selector change without fixed loss and coverage gate. | -| supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf | figure | 4 | agenda extendida CRPTO/tesis utility-directed CP future gate | Connects base model accuracy to downstream optimization value. | Not evidence that CRPTO currently optimizes utility-directed sets. | -| supplement/Guan 2023 - Localized Conformal Prediction.pdf | figure | 1 | agenda extendida CRPTO/tesis localized conformal candidate appendix | Visualizes global versus localized conformal bands. | No current localized calibration design. | -| paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf | figure | 2 | CRPTO/agenda extendida CRPTO/tesis applied robust-credit related work | Performance-comparison caption anchors the P2P robust portfolio context. | Different data, objective and uncertainty construction. | -| paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf | figure | 4 | agenda extendida CRPTO/tesis score/equity governance appendix | Gini-over-time caption is a useful template for score-vs-model governance reporting. | Not a legal fair-lending claim. | -| tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf | figure | 1 | agenda extendida CRPTO/tesis equity/noise governance appendix | Clarifies how better prediction technology can create group-specific effects. | No protected-attribute causal claim in LC. | -| tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf | table | 4 | agenda extendida CRPTO/tesis data-quality/equity governance appendix | Links predictive-performance gaps to disadvantaged consumers and data quality. | Project lacks the paper's protected-group and lender data. | -| supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf | table | 5 | agenda extendida CRPTO/tesis fairness-proxy boundary appendix | Shows proxy-probability validation needs richer race/ethnicity methodology. | LC lacks surname/fine geography and protected labels. | -| tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf | figure | 2 | agenda extendida CRPTO/tesis governance/committee communication appendix | Human-facing uncertainty display can inspire committee explanation design. | Different task, subjects and utility function. | - -## Future work y stop rules - -| relative_path | decision | crpto_value | extended_lab_value | stop_rule | -| --- | --- | --- | --- | --- | -| paper/Hu et al 2026 - Conformal Robustness Control.pdf | park_future_work | Future-work frontier; supports not overclaiming current method. | Candidate future gate for robustness certificates. | Park if it cannot be evaluated on frozen CRPTO artifacts without retraining a new method. | -| paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf | append_future_work | Related-work frontier. | Satisficing/governance candidate. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf | park_future_work | Future-work only. | Strong future gate for composite risk. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf | park_future_work | Future-work contrast. | End-to-end lane context. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf | park_future_work | Future-work caveat. | Online/source conformal lane context. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf | append_future_work | Future/positioning support. | Decision-risk lab framing. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf | park_future_work | Future-work context. | Decision audit/source governance context. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf | park_future_work | Future-work only. | Online lane context and stop-rule support. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf | park_future_dfl | Comparator/future work. | Potential DFL suite extension. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf | append_future_work | Future-work context. | Source-governance lane and caveat. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Yeh et al 2025 - Conformal Risk Training.pdf | park_future_work | Future-work contrast. | Candidate composite decision-risk training lane. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf | park_future_work | Future-work frontier. | Lane 3/end-to-end context. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf | park_future_work | Future auditability contrast. | Strong future lane for decision certificate rather than policy promotion. | Do not reopen a lane unless it changes a manuscript claim. | -| supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf | park_future_work | Future-work context. | Decision robustness gate candidate. | Do not reopen a lane unless it changes a manuscript claim. | -| tesis/Kawasumi Kato Duan 2026 - Conformal Prediction for Ordinal Credit Scoring.pdf | park_future_work | Future credit conformal context. | Ordinal score/rating future lane. | Do not reopen a lane unless it changes a manuscript claim. | - -## Matriz paper-by-paper - -La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, método/evidencia, conclusión, figuras/tablas útiles, limitación y destino editorial. Para auditoría operativa usar el CSV completo. - -| relative_path | title | status | core_concepts | key_claims | conclusions | figures_tables_useful | limitations | decision | action_required | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf | Credit Scores: Performance and Equity | NBER working paper | Score performance, equity, misclassification, rank disagreement, observable vulnerable groups. | Better scores can improve accuracy while also changing error allocation across groups. | Useful for metric governance: report discrimination, calibration, utility and equity together. | Use tables on score performance/equity as template for FICO proxy vs champion diagnostics. | Not Lending Club CRPTO evidence and not a legal fair-lending protocol. | append_extended_governance | completed_quarto_patch | -| paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf | Conformal Prediction: A Gentle Introduction | Foundations and Trends in Machine Learning | Exchangeability, split conformal, prediction sets, finite-sample marginal coverage. | Distribution-free uncertainty can be wrapped around black-box predictors. | Best pedagogical citation for explaining why CRPTO's intervals are valid before optimization. | Use tutorial figures only conceptually; the paper needs its own CRPTO diagram. | Introductory; not a portfolio optimizer or credit-risk paper. | promote_crpto_body | none_now | -| paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf | Conformal Risk Control | ICLR 2024 | Risk control, monotone bounded losses, post-hoc calibration, user-defined loss. | Conformal calibration can control expected loss beyond ordinary set coverage. | Direct ancestor of CRPTO's funded-set weighted miscoverage framing. | Use conceptual loss-control diagrams as related-work inspiration. | Does not itself define a portfolio-funded-set bound. | promote_crpto_body | none_now | -| paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf | Theoretical Foundations of Conformal Prediction | Cambridge University Press pre-publication / monograph | Exchangeability, exact finite-sample coverage, randomized quantiles, modern CP theory. | Conformal validity is a theorem about a procedure and assumptions, not a model-quality claim. | Use as the canonical proof reference for split conformal language and limitations. | Use definitions/proof structure, not figures, in CRPTO. | Too broad for the paper body; cite selectively. | promote_crpto_body | none_now | -| paper/Angelopoulos et al 2025 - Learn Then Test.pdf | Learn then Test: Calibrating Predictive Algorithms to Achieve Risk Control | Annals of Applied Statistics | Learn-then-test, finite-sample risk guarantees, post-hoc risk calibration. | Predictive algorithms can be calibrated after learning to satisfy risk constraints. | Supports predeclared gates and reviewer-safe validation language. | Use algorithmic workflow conceptually for gate diagrams. | Does not solve downstream optimization by itself. | promote_crpto_body | none_now | -| paper/Barber et al 2021 - Limits of Distribution-Free Conditional Predictive Inference.pdf | The Limits of Distribution-Free Conditional Predictive Inference | Information and Inference | Conditional coverage impossibility, approximate conditional validity, subgroup limits. | Exact distribution-free conditional coverage is impossible without assumptions or restrictions. | This is a claim-boundary source: it prevents overclaiming subgroup guarantees. | Use theorem statements, not figures. | Boundary paper, not an implementation. | promote_crpto_body | none_now | -| paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf | Distribution-Free, Risk-Controlling Prediction Sets | arXiv / risk-control foundation | RCPS, set-valued prediction, expected loss control, black-box models. | Prediction sets can be calibrated to control a user-defined risk distribution-free. | Provides the bridge from coverage to risk-control vocabulary. | Use algorithmic schematic as conceptual support only. | General prediction sets; no credit portfolio objective. | promote_crpto_body | none_now | -| paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf | Data-Driven Robust Optimization | Mathematical Programming | Uncertainty sets learned from data, robustness guarantees, finite-sample feasibility. | Robust sets can be constructed data-driven rather than manually specified. | Supports the historical progression from ad-hoc RO to calibrated/data-driven uncertainty sets. | Useful as related-work table contrasting source of uncertainty sets. | Not conformal and not Lending Club-specific. | context_crpto_related_work | none_now | -| paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf | From Predictive to Prescriptive Analytics | Management Science | Prescriptive analytics, decision quality, predictive-to-prescriptive bridge. | Prediction alone is insufficient; decision policies should be evaluated by downstream outcomes. | Useful framing for why CRPTO evaluates funded-set outcomes, not AUC only. | Related-work schematic only. | Not conformal and not a risk-control guarantee. | context_crpto_related_work | none_now | -| paper/Bertsimas Sim 2004 - The Price of Robustness.pdf | The Price of Robustness | Operations Research | Budgeted uncertainty, protection level, tractable robust LP/IP, price of robustness. | Robustness trades objective value for protection in a tunable, tractable way. | Use as the language ancestor for alpha -> Gamma_CP -> return trade-off. | Use price-of-robustness framing in CRPTO figures. | Budget is chosen, not conformally calibrated. | promote_crpto_body | none_now | -| paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf | Data-Driven Robust Credit Portfolio Optimization for Investment Decisions in P2P Lending | Applied Soft Computing / Elsevier article | P2P lending, robust credit portfolio, data-driven scoring, risk-return allocation. | Robust optimization can improve P2P investment decisions under model uncertainty. | Closest credit-domain precedent for robust P2P allocation. | Use experimental design/tables as comparator template for Lending Club portfolio papers. | No conformal guarantees and no funded-set miscoverage bound. | append_crpto_related_work | completed_bib_quarto_patch | -| paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf | Task-based End-to-end Model Learning in Stochastic Optimization | NeurIPS 2017 | End-to-end learning, differentiable optimization, downstream task loss. | Training against task loss can outperform prediction-error training for decisions. | DFL lineage for SPO+/PyEPO comparator. | Use as historical DFL lineage, not as CRPTO method figure. | No conformal coverage or credit-specific auditability. | append_comparator | none_now | -| paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf | Smart Predict, then Optimize | Management Science | SPO loss, SPO+ surrogate, regret-oriented training, downstream decision quality. | Models can be trained to reduce decision regret rather than prediction error. | Primary DFL comparator: SPO+ may reduce regret but does not provide conformal auditability. | Use benchmark/regret tables as comparator template. | No coverage guarantee and no uncertainty-set audit trail. | append_comparator | none_now | -| paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf | Instance-based Credit Risk Assessment for Investment Decisions in P2P Lending | European Journal of Operational Research | Instance-based learning, P2P loan investment, risk-return decision support. | Loan-level decision support can improve P2P investment performance. | Useful historical Lending Club/P2P context. | Portfolio decision tables can inspire thesis background. | No conformal or robust bound. | context_credit_domain | none_now | -| paper/Hu et al 2026 - Conformal Robustness Control.pdf | Conformal Robustness Control | ICLR 2026 | Robustness control, conformal calibration, robust decision criteria. | Conformal machinery can certify robustness-oriented decision properties. | Confirms CRPTO is timely, but should remain future-work unless reproduced. | Use as frontier map figure/table in related work, not main evidence. | Not yet implemented in the project; no Lending Club funded-set result. | park_future_work | completed_bib_quarto_patch | -| paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf | The Roles of Alternative Data and Machine Learning in Fintech Lending | Financial Management | LendingClub, alternative data, credit grades, FICO relationship, fintech underwriting. | Fintech grades and alternative data provide empirical credit information beyond traditional scores. | Best setting citation for why Lending Club is a meaningful credit-risk lab. | Use tables about grade/FICO/performance as context, not CRPTO evidence. | Does not validate CRPTO or fairness claims. | promote_crpto_body | none_now | -| paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf | Conformal Uncertainty Sets for Robust Optimization | COPA / PMLR | Conformal regions as uncertainty sets, robust optimization, finite-sample validity. | Conformal prediction regions can feed robust optimization with validity. | Closest foundational CP -> RO bridge. | Use as conceptual predecessor for CRPTO pipeline figure. | No credit-specific funded-set weighted bound. | promote_crpto_body | none_now | -| paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf | Conformal Contextual Robust Optimization | AISTATS 2024 | Contextual robust optimization, conformal regions, generative uncertainty. | Informative conformal uncertainty regions can improve robust contextual decisions. | Strong neighboring method but with different uncertainty geometry and domains. | Use related-work contrast table. | Not credit-specific and no funded-set PD constraint. | promote_crpto_body | none_now | -| paper/Sun et al 2024 - Predict-then-Calibrate.pdf | Predict-then-Calibrate: A New Perspective of Robust Contextual LP | arXiv / robust contextual LP | Predict-then-calibrate, robust contextual LP, box/ellipsoid uncertainty sets. | Post-hoc calibration can build valid uncertainty sets for robust contextual optimization. | Closest conceptual comparator to post-hoc CRPTO. | Use contrast table: PtC set membership vs CRPTO funded-set weighted risk. | No Lending Club and no Mondrian funded-set bound. | promote_crpto_body | none_now | -| paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf | Integrating AI and OR for investment decision-making in emerging digital lending businesses | Journal of the Operational Research Society; online 2025 | Digital lending, multi-objective risk-return optimization, AI + OR pipeline. | Combining ML and OR improves investment recommendations in digital lending. | Useful applied competitor/context; not a conformal method. | Use risk-return frontier and experimental setup as agenda extendida CRPTO/tesis comparator inspiration. | No conformal coverage, no CRPTO bound, no Lending Club champion evidence. | append_extended_context | completed_bib_quarto_patch | -| paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf | Conformal Robust Optimization and Satisficing for Prescriptive Analytics with Black-Box Predictors | AISTATS workshop / SSRN working paper | Conformal robust optimization, satisficing, black-box predictors, parameter mapping. | CRO and conformal robust satisficing can certify robust prescriptive decisions. | Valuable frontier for satisficing margins and policy acceptance thresholds. | Use satisficing margin idea for appendix, not body. | Preprint/workshop and not credit-specific. | append_future_work | none_now | -| supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf | Conformal Risk Control for Non-Monotonic Losses | arXiv preprint | Non-monotonic loss, risk control, multi-objective loss families. | CRC ideas can extend beyond monotone losses under new machinery. | Useful for future compound loss: return + V + CVaR + source risk. | Use loss examples for future gate design. | Not implemented; would change current CRPTO claim. | park_future_work | none_now | -| supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf | Optimal Model Selection for Conformalized Robust Optimization | arXiv preprint | Conformalized robust optimization, decision-aware model selection, downstream risk. | Selecting conformal models should account for robust decision performance. | Justifies CRPTO's CROMS-lite selector but not a full implementation claim. | Use selection diagrams/tables to structure A5/A10 appendix. | Project only has a selector screen over artifacts, not CROMS training. | append_crpto_selector | none_now | -| supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf | Conformal Prediction Beyond Exchangeability | Annals of Statistics | Weighted exchangeability, distribution shift, non-exchangeable conformal validity. | Conformal validity can be extended with weights under structured departures from exchangeability. | Important boundary/future source for source/shift lanes. | Use assumptions table for source governance appendix. | Requires weight/shift structure not currently guaranteed. | append_extended_source_shift | completed_bib_quarto_patch | -| supplement/Ben-Tal Teboulle 2007 - Optimized Certainty Equivalent.pdf | An Old-New Concept of Convex Risk Measures: The Optimized Certainty Equivalent | Mathematical Finance | Optimized certainty equivalent, convex risk measures, CVaR relation. | OCE provides a principled convex risk-measure family. | Grounds OCE/CVaR tail-risk appendix. | Use definitions, not figures. | Does not choose a credit champion by itself. | append_tail_risk | none_now | -| supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf | Group-Weighted Conformal Prediction | Electronic Journal of Statistics | Group-weighted CP, group-based shift, weighted conformal prediction. | When groups drive covariate shift, group-weighted calibration can improve guarantees. | Natural candidate for agenda extendida CRPTO/tesis source/grade reweighting, but not current CRPTO. | Use group-weighting assumptions for source-governance table. | Needs target/source group weights and prospective design. | append_mixed_diagnostic_source_not_better | experiment_completed_appendix_diagnostic | -| supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf | Using Publicly Available Information to Proxy for Unidentified Race and Ethnicity | official methodology report | BISG-style proxy, surname/geography, proxy validation, protected-attribute limitation. | Race/ethnicity proxying requires richer inputs and careful validation. | Supports why Lending Club zip3/state is insufficient for legal fair-lending claims. | Use proxy-method workflow as governance appendix reference. | Project lacks surname and fine geography. | append_extended_governance | none_now | -| supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf | End-to-End Conditional Robust Optimization | UAI 2024 / PMLR | Conditional robust optimization, differentiable optimization, end-to-end training. | End-to-end conditional robust training can improve conditional coverage/objective tradeoffs. | Frontier contrast: CRPTO is post-hoc and auditable, not end-to-end. | Use as frontier comparison table. | Different stack and conditional guarantees; not current project method. | park_future_work | none_now | -| supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf | Utility-Directed Conformal Prediction | ICLR 2025 | Utility-directed CP, decision loss, actionable uncertainty, coverage preservation. | Prediction sets can incorporate downstream utility while retaining standard coverage. | Highly relevant future step: optimize CP usefulness without abandoning coverage. | Use framework diagram for future decision-loss conformal selector design. | Current CRPTO does not implement this training/calibration objective. | park_width_fail | experiment_completed_parked | -| supplement/FinRegLab 2023 - Explainability and Fairness in ML Credit Underwriting.pdf | Explainability and Fairness in Machine Learning for Credit Underwriting: Policy Analysis | policy report | Explainability, fairness, adverse action, model governance, underwriting controls. | ML credit systems require governance controls beyond predictive performance. | Supports MRM/fairness appendix and explains why claims must be bounded. | Use control-taxonomy tables for governance mapping. | Policy report, not CRPTO empirical evidence. | append_governance | none_now | -| supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf | Adaptive Conformal Inference Under Distribution Shift | NeurIPS 2021 | ACI, online updating, distribution shift, adaptive quantiles. | Conformal thresholds can be adapted under shift to improve long-run coverage. | Future online/drift candidate, not current static CRPTO. | Use online update schematic for agenda extendida CRPTO/tesis future work. | Needs streaming/prospective feedback to claim deployment validity. | park_future_work | none_now | -| supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf | Conformal Prediction with Conditional Guarantees | JRSS-B | Conditional guarantees, coverage spectrum, restricted conditional targets. | Relaxed conditional goals can be achieved where exact conditional coverage cannot. | Supports future conditional-tightening language and source-group caveats. | Use conceptual spectrum for appendix if conditional validity is discussed. | Not implemented in project; current Mondrian is finite-group conditional-ish, not exact individual conditional. | append_claim_boundary | none_now | -| supplement/Guan 2023 - Localized Conformal Prediction.pdf | Localized Conformal Prediction | Biometrika | Localized conformity scores, local calibration, approximate conditional validity. | Local weighting can improve adaptivity while retaining conformal inference structure. | Candidate comparator to Mondrian if future coverage sharpness matters. | Use local-vs-global schematic for appendix only. | Would require a new localized calibration design. | append_mixed_diagnostic_wider_than_mondrian | experiment_completed_appendix_diagnostic | -| supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf | Conformal Predictive Systems Under Covariate Shift | arXiv / preprint | Conformal predictive systems, covariate shift, calibration under shift. | Conformal predictive systems can be adapted for shifted covariate distributions. | Useful for source/shift caveat, not current champion. | Use shift taxonomy only. | Requires shift assumptions and is not credit-specific. | context_source_shift | none_now | -| supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf | Decision Theoretic Foundations for Conformal Prediction | AISTATS / ICML-era preprint | Decision-theoretic CP, utility, actionability, uncertainty quantification. | Conformal sets can be understood through decisions and utilities, not only coverage. | Good conceptual support for CRPTO's decision-aware framing. | Use as theory bridge in future-work paragraph. | Does not implement CRPTO portfolio optimization. | append_future_work | none_now | -| supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf | Conformal Decision Theory | NeurIPS workshop / preprint | Conformal prediction for decisions, calibrated decision policies, set-valued actions. | Conformal uncertainty can be integrated with decision-making objectives. | Supports agenda extendida CRPTO/tesis decision-risk framing but is not a CRPTO baseline. | Use only as conceptual context. | No credit portfolio evidence. | context_decision_conformal | none_now | -| supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf | Conformal Inverse Optimization | NeurIPS 2024 | Inverse optimization, conformal uncertainty, decision ambiguity. | Conformal methods can quantify uncertainty in inverse optimization. | Valuable frontier for auditing human/legacy decisions, not current CRPTO. | Use CREDO/inverse-optimization family as future audit lane. | Different problem: inferring preferences/parameters from decisions. | park_future_work | none_now | -| supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf | Multi-Source Conformal Inference Under Distribution Shift | arXiv preprint | Multi-source inference, heterogeneous source distributions, shifted test populations. | Combining calibration sources carefully can maintain valid intervals under source shift. | Directly relevant to agenda extendida CRPTO/tesis source-family holdout but needs stronger data design. | Use source-combination taxonomy for source governance. | Current LC source groups are retrospective, not validated external sources. | append_extended_source_shift | none_now | -| supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf | Online Conformal Prediction via Universal Portfolio Algorithms | arXiv preprint | Online conformal prediction, universal portfolios, regret-to-coverage, parameter-free adaptation. | Online CP can achieve long-run coverage via universal portfolio algorithms. | Excellent future direction for deployment, but current project is retrospective. | Use online update process as future-work schematic. | No production feedback stream in current data. | park_future_work | none_now | -| supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf | Decision-Focused Learning: Foundations, State of the Art, Benchmark and Future Opportunities | Journal of Artificial Intelligence Research | DFL taxonomy, SPO+, gradient-based/gradient-free methods, benchmarks. | DFL is mature but no single method dominates across tasks. | Use to defend treating SPO+/PyEPO as comparator, not CRPTO replacement. | Use taxonomy table for agenda extendida CRPTO/tesis DFL suite framing. | Survey does not provide conformal guarantees. | append_comparator | none_now | -| supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf | Optimization of Conditional Value-at-Risk | The Journal of Risk | CVaR, tail risk, tractable convex optimization. | CVaR is optimizable and more useful than VaR for tail-risk control. | Grounds A12 and agenda extendida CRPTO/tesis tail challenger. | Use formulation, not figures. | Tail-risk improvement does not imply wealth champion. | append_tail_risk | none_now | -| supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf | Conformalized Quantile Regression | NeurIPS 2019 | Conformalized quantile regression, adaptive intervals, split conformal. | Quantile regression plus conformalization yields valid adaptive intervals. | Baseline/variant reference for interval adaptivity. | Use interval-width/adaptivity examples in conformal chapter. | Current champion is Mondrian score-decile, not CQR. | append_conformal_variant | none_now | -| supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf | Robust Losses for Decision-Focused Learning | NeurIPS / arXiv-era DFL paper | Robust decision-focused losses, decision regret, misspecification. | Robust losses can improve DFL under uncertainty or misspecification. | agenda extendida CRPTO/tesis DFL challenger context; not a CRPTO body claim. | Use loss comparison table for DFL appendix only. | No conformal auditability or credit-specific bound. | park_future_dfl | none_now | -| supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf | Conformal Prediction Under Covariate Shift | NeurIPS 2019 | Covariate shift, weighted conformal prediction, likelihood-ratio weights. | Conformal coverage can adapt under covariate shift if weights are known/estimated. | Core source for future source/shift gates. | Use assumption table for source-shift appendix. | Requires credible density-ratio/weight estimation. | append_extended_source_shift | none_now | -| supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf | Multi-Distribution Robust Conformal Prediction | arXiv preprint | Multiple source distributions, robust coverage, max-p aggregation. | Finite-sample coverage can be made robust over multiple distributions/mixtures. | Strong source-robustness future-work citation. | Use source distribution diagram if agenda extendida CRPTO/tesis source appendix expands. | Current project has retrospective source proxies, not true multi-source deployment. | append_future_work | none_now | -| supplement/Yeh et al 2025 - Conformal Risk Training.pdf | Conformal Risk Training | NeurIPS 2025 | Conformal risk training, OCE/CVaR, differentiable conformal risk. | Conformal risk can be optimized end-to-end, including OCE-style risks. | Future direction for replacing diagnostic tail risk with optimized conformal risk. | Use risk-training diagram as future-work context only. | Current CRPTO is post-hoc; implementing CRT would change method. | park_future_work | none_now | -| supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf | End-to-End Conformal Calibration for Optimization Under Uncertainty | TMLR; arXiv v2 in 2026 | End-to-end conformal calibration, downstream optimization, learned uncertainty sets. | Calibration can be optimized for decision usefulness while retaining validity. | Confirms future direction but not current post-hoc CRPTO. | Use related-work contrast table only. | Different training stack and not credit-specific. | park_future_work | none_now | -| supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf | Conformalized Decision Risk Assessment | ICLR 2026 / arXiv 2025 | CREDO, decision risk certificates, inverse optimization, conformalized risk estimation. | A candidate decision can receive a distribution-free upper bound on probability of suboptimality. | Very relevant for agenda extendida CRPTO/tesis auditability; not part of current CRPTO champion. | Use risk-certificate diagram as agenda extendida CRPTO/tesis future audit inspiration. | Different problem geometry; no Lending Club implementation. | park_future_work | completed_bib_quarto_patch | -| supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf | Calibrating Decision Robustness via Inverse Conformal Risk Control | arXiv preprint | Inverse CRC, decision robustness, robustness calibration. | Decision robustness can be calibrated through inverse conformal risk-control ideas. | Relevant future lane for robustness calibration. | Use only as future-work contrast. | Not implemented and not credit-specific. | park_future_work | none_now | -| tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf | A multi-objective instance-based decision support system for investment recommendation in peer-to-peer lending | Journal article | P2P lending, multi-objective recommendation, risk-return tradeoff, NPV. | Decision-support systems can optimize investment recommendations over risk and return. | Useful thesis context for digital lending optimization. | Use risk-return recommendation setup as background only. | No conformal guarantees. | context_credit_domain | none_now | -| tesis/Basel Committee 2015 - Guidance on Credit Risk and Expected Credit Losses.pdf | Guidance on credit risk and accounting for expected credit losses | official supervisory guidance | Expected credit losses, credit risk governance, forward-looking information, controls. | ECL estimation needs governance, data quality, forward-looking information and controls. | Supports IFRS9-inspired boundary and why contractual IFRS9 is not claimed. | Use guidance checklist for agenda extendida CRPTO/tesis IFRS9 proxy appendix. | Open Lending Club data lacks contractual monthly DPD/EAD/recovery infrastructure. | append_extended_ifrs9 | none_now | -| tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf | Robust Optimization | Princeton University Press book | Uncertainty sets, robust counterparts, tractability, convex robust optimization. | Robust optimization provides a systematic language for feasible decisions under uncertainty. | Use as deep thesis foundation; CRPTO body can rely on shorter RO citations. | Use formulations/definitions, not figures. | Very broad; not conformal or credit-specific. | thesis_foundation | none_now | -| tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf | How Costly is Noise? Data and Disparities in Consumer Credit | working paper / credit economics | Data noise, credit scores, disparities, unequal information quality. | Noisy credit data can create unequal access and distort credit allocations. | Important governance caveat: model performance can be constrained by source data quality. | Use disparity/noise tables as governance appendix inspiration. | Not Lending Club CRPTO evidence. | append_extended_governance | none_now | -| tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf | Credit Invisibles and the Unscored | Cityscape / CFPB research | Credit invisibles, unscored consumers, data availability, inclusion. | Large consumer segments lack conventional scores or have weak scoring coverage. | Supports why open accepted-loan data cannot make broad credit-access claims. | Use population breakdowns as motivation only. | Not a CRPTO method source. | context_governance | none_now | -| tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf | Conformal Prediction Sets Improve Human Decision Making | preprint / empirical decision support | Human-AI decision making, prediction sets, uncertainty communication. | Conformal sets can improve human decisions when uncertainty is communicated well. | Useful for discussing auditability and committee-facing uncertainty. | Use decision-support figures as inspiration for MRM/committee communication. | Not credit portfolio optimization. | context_decision_support | none_now | -| tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf | Label Noise Robustness of Conformal Prediction | Journal of Machine Learning Research | Label noise, conformal robustness, noisy outcomes, validity under imperfect labels. | Conformal prediction has robustness properties under forms of label noise. | Useful threat-to-validity note for default labels and hardened shift tests. | Use noise taxonomy for appendix caveat. | Does not validate all Lending Club label issues. | append_claim_boundary | none_now | -| tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf | Predictably Unequal? The Effects of Machine Learning on Credit Markets | Journal of Finance | ML credit screening, distributional impacts, mortgage markets, disparities. | More flexible ML can change rate disparities and who benefits from credit-market technology. | Critical fairness/equity context: better predictive technology does not automatically imply equitable outcomes. | Use disparity mechanism figures/tables in governance discussion. | Mortgage market, not Lending Club; no protected attributes in current project. | append_extended_governance | completed_bib_quarto_patch | -| tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf | Conformal Predictive Portfolio Selection | arXiv preprint | Portfolio selection, conformal predictive sets, financial decisions. | Conformal prediction can support portfolio selection under uncertainty. | Relevant neighbor in finance; CRPTO differs by credit PD/funded-set risk. | Use as finance-related-work table. | Not credit lending and not Lending Club. | context_finance_cp | none_now | -| tesis/Kawasumi Kato Duan 2026 - Conformal Prediction for Ordinal Credit Scoring.pdf | Conformal Prediction for Ordinal Credit Scoring | arXiv preprint | Ordinal credit scoring, conformal prediction, rating categories. | Conformal methods can be adapted to ordinal credit-score outputs. | Useful for future grade/rating-set extension, not current PD interval CRPTO. | Use only as future-work note. | No portfolio optimization and very short current evidence. | park_future_work | none_now | -| tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf | Consumer Credit-Risk Models via Machine-Learning Algorithms | Journal of Banking & Finance | Consumer credit risk, nonlinear ML, transaction data, economic value of risk forecasts. | ML forecasts can materially improve consumer credit-risk management and economic decisions. | Classic support for economic evaluation of credit ML, not just classification metrics. | Use economic-benefit tables as background for decision-value framing. | Credit cards/bank data, not LC/CRPTO. | context_credit_domain | completed_bib_quarto_patch | -| tesis/Noguer i Alonso 2024 - Conformal Portfolio Optimization.pdf | Conformal Portfolio Optimization | SSRN preprint | Conformal prediction, portfolio optimization, finance uncertainty. | Conformal intervals can support portfolio decisions in financial assets. | Finance-adjacent context, but CRPTO's credit/funded-set bound is different. | Use only as related-work mention. | Short, non-credit, not a robust credit allocation benchmark. | context_finance_cp | none_now | - -## Control bibliográfico - -Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explícitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografía. - -| bib_status | n | -| --- | --- | -| added_2026_06_06 | 10 | -| existing | 36 | -| needs_bib_if_cited | 15 | - -## Fronteras que permanecen falsas - -- CRPTO no reclama legal fair lending con atributos protegidos directos. -- CRPTO no implementa IFRS9 contractual. -- agenda extendida CRPTO/tesis no reclama CATE policy value. -- agenda extendida CRPTO/tesis no reclama online deployment. -- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto. -- SPO+/DFL puede ganar regret, pero no reemplaza la garantía/auditabilidad CRPTO. - -## Cierre - -La auditoría agrega valor como integración bibliográfica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empírica del paper actual. diff --git a/docs/research/pool93_certificate_semantics_v2_2026-07-09.md b/docs/research/pool93_certificate_semantics_v2_2026-07-09.md deleted file mode 100644 index 02f71f2..0000000 --- a/docs/research/pool93_certificate_semantics_v2_2026-07-09.md +++ /dev/null @@ -1,132 +0,0 @@ -# Auditoria consolidada del certificado pool93 v2 - 2026-07-09 - -## Decision - -Se promueve la semantica `certificate-semantics-v2` como fuente activa para -A35, la gobernanza pool93 y el baseline A40. No cambia el modelo PD, calibrador, -intervalos conformales, asignacion seleccionada, retorno del body, grilla alpha -ni denominadores. Corrige la lectura del endpoint para policies no lineales y -reemplaza el comparador Lending Club mal rotulado por una baseline point-PD -emparejada. - -Tag activo: - -`champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` - -## Hallazgo 1: descomposicion policy-aware - -La formula historica - -`B_u = tau + (1 - gamma) * Gamma_CP` - -es exacta solo para un blend lineal cuyo cap efectivo liga. La frontera tambien -contiene policies `capped`, `tail` y `segment_relative_tail`; en ellas `gamma` -no basta para reconstruir el endpoint. - -Para cualquier score efectivo declarado `q_i` con -`p_hat_i <= q_i <= u_i`, la identidad general es: - -```text -Gamma_CP = sum_i w_i (u_i - p_hat_i) -Gamma_int = sum_i w_i (q_i - p_hat_i) -Gamma_res = sum_i w_i (u_i - q_i) -Gamma_CP = Gamma_int + Gamma_res -B_u = sum_i w_i q_i + Gamma_res -B_u <= tau + solver_slack + Gamma_res -T_Markov = B_u + sqrt(alpha) -``` - -`T_Markov` es un umbral de evento probabilistico bajo weighted funded-set -validity; no es un cap determinista. El campo historico `violation` mide exceso -realizado de default sobre `tau`, no violacion de la identidad -`sum(wY) <= B_u + V`. - -## Punto seleccionado - -En `alpha=0.01`, la asignacion seleccionada conserva: - -| Cantidad | Valor | -|---|---:| -| Retorno realizado | `$184,832.48` | -| Filas financiadas | `314` | -| `V` / default ponderado | `0.035350` | -| PD puntual ponderada | `0.082468` | -| Score efectivo ponderado | `0.171500` | -| `Gamma_CP` | `0.162616` | -| `Gamma_int` | `0.089032` | -| `Gamma_res` | `0.073584` | -| Endpoint exacto `B_u` | `0.245084` | -| Umbral Markov exacto | `0.345084` | -| Exceso realizado sobre `tau` | `0.000000` | -| Pass grilla alpha | `8/8` | - -El cap row-level de la policy `capped_blended_uncertainty` esta inactivo en las -314 filas financiadas. Por eso la formula lineal coincide numericamente en este -punto, aunque no sea valida como formula universal de la frontera. - -## Auditoria de A35 sin nueva busqueda - -La reconstruccion lee las estadisticas suficientes de seis evaluaciones exactas -ya existentes. No ejecuta HPO, busqueda de policy, generacion conformal ni solve -de portafolio. - -- filas crudas: `51,678`; -- policies semanticas deduplicadas: `50,010`; -- policies elegibles all-alpha y sobre floor: `27,508`; -- seleccion del body: sin cambio; -- thresholds con cambio material: `10,423`; -- policies tail con understatement: `2,866`; -- understatement maximo: `0.241324`; -- antiguas filas `<=0.50` que exceden `0.50` exacto: `716`. - -Los modos afectados son `tail_blended_uncertainty` y -`segment_relative_tail_blended_uncertainty`. El endpoint de maximo retorno sigue -ganando `$223,458.14`, pero su endpoint es `0.597056` y su umbral Markov exacto -es `0.697056`. El body y todos los denominadores permanecen iguales. - -## Hallazgo 2: baseline point-PD A40 - -El campo historico `price_of_robustness=-10.56%` comparaba contra un solve -rotulado `nonrobust` que aun heredaba una restriccion `pd_high`. Una -recomputacion preliminar a `tau=0.175` sirvio para detectar el problema, pero no -era el contraste final del body y se retira como superficie activa. - -A40 resuelve el contraste correcto con: - -- los mismos `276,869` candidatos; -- presupuesto `$1M`; -- misma concentracion, `tau=0.1715`, LGD, solver y controles operativos; -- point PD en objetivo y restriccion como unica diferencia semantica; -- outcomes OOT usados solo despues del solve. - -| Policy | Retorno | Funded | Default / `V` | `Gamma_CP` | `B_u` | Threshold | -|---|---:|---:|---:|---:|---:|---:| -| Point-PD two-stage LP | `$196,369.14` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | -| CRPTO seleccionado | `$184,832.48` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | - -CRPTO cede `$11,536.66` (`5.875%`) y reduce default/miscoverage en `0.08305` -(8.305 puntos porcentuales) y el threshold en `0.435495` (43.55 puntos). Ambos -funded sets quedan debajo de `tau`; solo CRPTO pasa el screen tight -`V <= sqrt(0.01)`. - -## Limite del claim - -A40 es una auditoria emparejada sobre un OOT historico congelado. No demuestra -causalidad, significancia prospectiva ni dominancia universal. A35 es una -frontera de grilla finita, no un optimo continuo. La probabilidad del teorema -requiere weighted funded-set validity; el draw observado audita `V` pero no -prueba por si solo ese supuesto. - -## Artefactos promovidos - -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex` -- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` -- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex` -- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` -- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - -Las filas financiadas de A40 viven bajo `data/processed/experiments/`, no en el -directorio de modelos. Los artefactos antiguos se conservan como provenance, -pero no son fuentes de claims activos. diff --git a/docs/research/pool93_tail_risk_closeout_2026-07-02.md b/docs/research/pool93_tail_risk_closeout_2026-07-02.md deleted file mode 100644 index b081f6c..0000000 --- a/docs/research/pool93_tail_risk_closeout_2026-07-02.md +++ /dev/null @@ -1,84 +0,0 @@ -# Pool93 Tail-Risk Closeout - -Date: 2026-07-02 - -Certificate terminology was synchronized on 2026-07-09 with the policy-aware -A35 audit. A37--A39 values remain valid for the fixed selected allocation; -`0.345084` is now called the exact loss threshold, not a generic cap. - -This memo closes the post-promotion caveat that tail-risk, cluster-bound, and -bootstrap diagnostics should not be cited as pool93-specific unless regenerated -from the selected pool93 funded allocation. - -## Generated Artifacts - -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.tex` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.tex` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.tex` -- Generator: `scripts/search/build_pool93_tail_risk_audit.py` - -The generator reads the selected allocation from: - -`data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_body_allocation_alpha01.parquet` - -## A37 Tail-Risk Repricing - -At baseline `LGD = 0.45`, the selected body allocation has: - -- realized return: `$184,832.48` -- weighted default rate / `V`: `0.035350` -- realized CVaR95 loss rate: `0.276211` -- decision-time CVaR95 loss rate: `0.218140` -- exact loss threshold at `alpha = 0.01`: `0.345084` - -Across the LGD grid, repriced return ranges from `$188,367.48` at `LGD = 0.35` -to `$179,529.98` at `LGD = 0.60`. - -## A38 Cluster-Bound Repricing - -At `alpha = 0.01` and `delta = 0.10`, the distribution-free Markov increment -is `sqrt(alpha) = 0.100000`. -The regenerated cluster-aware Hoeffding thresholds are: - -- period: `0.395502` -- grade bucket: `0.728588` -- period-grade: `0.281247` -- score-vintage: `0.348546` - -None is tighter than the distribution-free Markov step. This supports the -manuscript's theory boundary: the policy-aware certificate remains the -body-level distribution-free statement, while cluster-aware tightening is -shown as an assumption-priced sensitivity. - -## A39 Fixed-Allocation Bootstrap - -The final pool93 bootstrap diagnostic resamples funded-loan contributions under -the fixed selected body allocation with `5,000` draws and seed `20260702`. - -Key baseline results: - -- observed baseline return at `LGD = 0.45`: `$184,832.48` -- bootstrap return mean: `$184,623.11` -- bootstrap return interval, 2.5%--97.5%: `$167,963.20` to `$198,650.47` -- observed weighted default / `V`: `0.035350` -- bootstrap `V` interval, 2.5%--97.5%: `0.018157` to `0.057193` -- observed `Gamma_CP`: `0.162616` -- bootstrap `Gamma_CP` interval, 2.5%--97.5%: `0.137160` to `0.193092` -- observed realized CVaR95: `0.276211` -- observed decision-time CVaR95: `0.218140` - -This is a fixed-allocation empirical contribution interval. It does not resample -solver inputs, the PD model, calibration data, conformal intervals, or the policy -search. - -## Claim Boundary - -A37--A39 are selected-allocation risk-profile audits. They do not change the -pool93 body selector, do not make CVaR/OCE the optimized objective, and do not -turn bootstrap intervals into a conformal guarantee. The paper-facing claim -remains the finite-grid policy-aware decision certificate in A35 plus the exact -funded-set audit; A37--A39 close reviewer questions about the selected point's tail, -concentration, and empirical contribution profile. diff --git a/docs/security/DEPENDABOT_TRIAGE_2026-07-02.md b/docs/security/DEPENDABOT_TRIAGE_2026-07-02.md deleted file mode 100644 index 70e7241..0000000 --- a/docs/security/DEPENDABOT_TRIAGE_2026-07-02.md +++ /dev/null @@ -1,51 +0,0 @@ -# Dependabot Triage - -Date: 2026-07-02 - -GitHub Dependabot reported open alerts against `uv.lock`. These are -security/dependency-maintenance items, not scientific-result blockers. The -paper artifacts and champion metrics do not depend on notebook server exposure -or authenticated web services. - -## Local Lockfile Remediation - -The lockfile was updated with: - -```bash -uv lock \ - --upgrade-package pydantic-settings \ - --upgrade-package msgpack \ - --upgrade-package jupyterlab \ - --upgrade-package jupyter-server \ - --upgrade-package starlette \ - --upgrade-package cryptography \ - --upgrade-package tornado \ - --upgrade-package aiohttp \ - --upgrade-package torch -``` - -Resolved local versions: - -| Package | Updated version | -|---|---:| -| `aiohttp` | `3.14.1` | -| `cryptography` | `48.0.1` | -| `jupyter-server` | `2.20.0` | -| `jupyterlab` | `4.6.1` | -| `msgpack` | `1.2.1` | -| `pydantic-settings` | `2.14.2` | -| `starlette` | `1.3.1` | -| `torch` | `2.12.1` | -| `tornado` | `6.5.7` | - -Dependabot may continue showing the old alerts until the branch is pushed and -GitHub refreshes the dependency graph. - -## Residual Notes - -- `diskcache` alert #1 was already dismissed upstream; no local change made. -- This triage intentionally avoids changing paper claims, model artifacts, or - canonical scientific outputs. -- If GitHub still reports an alert after push, re-run the API query and compare - the manifest path and resolved version in `uv.lock` before making a new - dependency change. diff --git a/docs/security/SECRETS_AND_REMOTES.md b/docs/security/SECRETS_AND_REMOTES.md index 29229ca..4f0b7ae 100644 --- a/docs/security/SECRETS_AND_REMOTES.md +++ b/docs/security/SECRETS_AND_REMOTES.md @@ -1,92 +1,50 @@ -# Secrets, remotes and local credentials +# Secrets And Remotes -This public repository should be safe to push to GitHub without real -credentials. Only templates belong in Git. +CRPTO uses GitHub for source and DagsHub S3-compatible storage for DVC +artifacts. Credentials are local or GitHub Actions secrets; they never belong +in tracked files. -## Files +## Local Variables -- Commit: `.env.example`, `.env.local.example`, `.github/workflows/*`. -- Do not commit: `.env`, `.env.*` except examples, `.dvc/config.local`, - `.claude/settings.local.json`, cloud keys, private keys, local tokens. +Keep credentials in an ignored `.env` or in the current PowerShell process: -## Required variables - -Use `.env.example` as the canonical list. The most common values are: - -- `DAGSHUB_OWNER=EigenCharlie94`, `DAGSHUB_USER=EigenCharlie94`, - `DAGSHUB_REPO=Paper_CRPTO` -- `DAGSHUB_TOKEN` or `DAGSHUB_USER_TOKEN` -- `MLFLOW_TRACKING_URI`, `MLFLOW_TRACKING_USERNAME`, - `MLFLOW_TRACKING_PASSWORD` -- `CRPTO_DATA_DIR`, `CRPTO_MODELS_DIR`, `CRPTO_DUCKDB_PATH` -- `OPTUNA_STORAGE` when using persistent Optuna studies - -## Standalone remotes - -CRPTO no longer points to the parent DagsHub project. The committed DVC remote -is: - -```text -https://dagshub.com/EigenCharlie94/Paper_CRPTO.s3 -``` - -The MLflow tracking URI is: - -```text -https://dagshub.com/EigenCharlie94/Paper_CRPTO.mlflow +```powershell +$env:DAGSHUB_USER = "..." +$env:DAGSHUB_TOKEN = "..." +$env:DAGSHUB_REPO = "Paper_CRPTO" +$env:AWS_ACCESS_KEY_ID = $env:DAGSHUB_TOKEN +$env:AWS_SECRET_ACCESS_KEY = $env:DAGSHUB_TOKEN +$env:AWS_ENDPOINT_URL = "https://dagshub.com/EigenCharlie94/Paper_CRPTO.s3" ``` -The old upstream GitHub remote (`EigenCharlie94/Lending-Club-End-to-End`) may still be -mentioned in learning/provenance notes, but it must not appear in active -runtime config, `.env.example`, DVC config or GitHub Actions secrets. +Do not print tokens into logs or place them in `.dvc/config`, YAML, Python, +paper sources, or submission forms. Machine-specific DVC configuration belongs +in ignored `.dvc/config.local`. -Historical WSL-era paths should not appear in active runtime configuration or -current JSON status artifacts. After the Windows revalidation, committed paths -that are consumed by scripts should be repository-relative. Active scripts must -construct paths from the repository root, config files, or CLI arguments. +## Active DVC Capsule -## GitHub Actions secrets +The source registry declares exactly 31 pointer files. Use the capsule manager +instead of an unrestricted historical replay: -For CI with DVC/MLflow enabled, configure these in GitHub repository settings, -not in files: - -- `DAGSHUB_TOKEN` -- `DAGSHUB_USER_TOKEN` -- `MLFLOW_TRACKING_USERNAME` -- `MLFLOW_TRACKING_PASSWORD` -- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_ENDPOINT_URL` if using an - S3-compatible DVC remote - -The default workflows avoid pulling large DVC artifacts. They validate the -book, metadata, and artifact-independent tests. Full artifact tests should run -locally or in CI only after a DVC remote is configured. - -## GitHub security settings - -For `EigenCharlie/Paper_CRPTO`, keep these enabled in repository settings: - -- Dependency graph and Dependabot security updates. -- Secret scanning. -- Optional branch protection on `main` if the project becomes multi-author. - In the current single-author academic mode, `lint` and `book-publish` run on - push, while `tests-full` is manually triggered before journal milestones. - -`dependency-review` requires the Dependency graph. If GitHub reports the -repository as unsupported, enable it in Settings -> Security and analysis -before merging dependency PRs. +```powershell +just ijds-pull +just ijds-dvc-status +just ijds-dvc-verify-remote +``` -## Windows-native environment +The historical `dvc.yaml` graph is sealed compatibility metadata. Never run its +protected PD, conformal, validation, portfolio, or exact-evaluation stages +without explicit permission. -The official local environment for this standalone repository is Windows -PowerShell with `.venv/Scripts`. +## GitHub Actions -Use: +The manual full workflow requires `DAGSHUB_USER`, `DAGSHUB_TOKEN`, and +`DAGSHUB_REPO` repository secrets. It installs the locked development +environment with: ```powershell -uv venv -uv sync --extra dev --extra search +uv sync --frozen --extra dev ``` -Do not set custom Python/Quarto environment overrides for normal work. `uv run` -selects the project venv and `uv run -- quarto ...` provides the right Python -context for Quarto renders. +Secret scanning and dependency alerts should remain enabled. A leaked token +must be revoked first, then removed from Git history if it was committed. diff --git a/dvc.yaml b/dvc.yaml index 5806a30..ec3bc63 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -1,6 +1,6 @@ stages: crpto.data.dataset: - cmd: python src/data/make_dataset.py --input data/raw/Loan_status_2007-2020Q3.csv --output data/processed/loan_master.parquet + cmd: python src/data/make_dataset.py --input data/raw/Loan_status_2007-2020Q3.csv --output data/processed/loan_master.parquet --legacy-resolved-only deps: - data/raw/Loan_status_2007-2020Q3.csv - src/data/make_dataset.py @@ -204,13 +204,12 @@ stages: outs: - data/processed/crpto_vs_spo_stability.json crpto.book.render: - cmd: uv run python scripts/write_book_build_info.py && uv run -- quarto render book --to html + cmd: uv run -- quarto render book --to html --no-execute deps: - book/_quarto.yml - book/index.qmd - book/chapters - book/styles.scss - - scripts/write_book_build_info.py params: - params.yaml: - book.freeze diff --git a/eda_summary_2.xlsx b/eda_summary_2.xlsx deleted file mode 100644 index 02bab98..0000000 Binary files a/eda_summary_2.xlsx and /dev/null differ diff --git a/justfile b/justfile index 87b41d8..1842498 100644 --- a/justfile +++ b/justfile @@ -1,25 +1,14 @@ -# CRPTO task runner — cross-platform via `just`. -# Install: https://github.com/casey/just (or `winget install Casey.Just` on Windows) +# Current CRPTO/IJDS task surface. All recipes are Windows-compatible via `just`. set windows-shell := ["powershell.exe", "-NoLogo", "-NoProfile", "-Command"] -set dotenv-load := true +set dotenv-load -# --- Setup --------------------------------------------------------------- +default: help -# Full setup including SPO (pyepo + torch) extras -default: setup +# --- Environment and quality --------------------------------------------- setup: - uv sync --extra dev --extra search --extra spo - -# Lighter setup without SPO/torch -setup-base: - uv sync --extra dev --extra search - -setup-spo: - uv sync --extra dev --extra search --extra spo - -# --- Quality gates ------------------------------------------------------- + uv sync --extra dev lint: uv run ruff check . @@ -30,199 +19,138 @@ fmt: uv run ruff format . type-check: - uv run mypy src scripts + uv run mypy src scripts tests -# Fast type checker from Astral. Daily active-scope use remains advisory while -# ty matures; the clean full scope is blocking in the final submission gate. -type-advisory: - @uv run python scripts/run_ty_advisory.py --scope active - -type-advisory-full: - @uv run python scripts/run_ty_advisory.py --scope full --fail-on-diagnostics --output reports/ci/ty-advisory-full.txt - -complexity-report: - uvx radon cc src scripts -s -n D - -api-docs-core: - uv run --with pdoc pdoc src.optimization.portfolio_model src.optimization.policy_evaluation src.optimization.policy_selection src.models.conformal_alpha_grid src.models.calibration src.evaluation.backtesting src.evaluation.fairness --docformat google --output-directory reports/api-docs --no-browser +type-check-fast: + @uv run python scripts/run_ty_advisory.py --scope active --fail-on-diagnostics --no-report hooks-check: uv run pre-commit validate-config uvx prek validate-config .pre-commit-config.yaml -# Fast smoke: paper-final guardrails + Quarto book guardrails -smoke: - uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py tests/test_publication_integrity.py -q - -publication-integrity: - uv run python scripts/check_publication_integrity.py +complexity-report: + uv run python scripts/run_complexity_report.py test: uv run pytest -q -test-fast: - uv run pytest -q -m "not slow" - -# --- Paper outputs (safe — do NOT touch the frozen champion) ------------- - -tables: - uv run python scripts/export_crpto_tables.py +smoke: + uv run pytest tests/test_publication_integrity.py tests/test_ijds_active_claim_sync.py tests/test_publication_targets.py -q -figures: - uv run python scripts/generate_crpto_figures.py --paper crpto +# Read-only regression gate for the current PD/conformal implementation and +# every paper-facing numerical contract. It does not execute a scientific run. +drift-gate: publication-integrity + uv run pytest -q tests/test_models/test_binary_conformal_guardrail.py tests/test_ijds_audit_core.py tests/test_ijds_active_claim_sync.py tests/test_ijds_v4_claim_sync.py tests/test_ijds_audit/test_credit_controls.py tests/test_ijds_audit/test_endpoint_recovery.py tests/test_ijds_audit/test_evaluation_outcome_contracts.py -evidence: - uv run python scripts/analyze_crpto_evidence.py +# --- Active evidence and protocol entrypoints ----------------------------- -journal-package: - uv run python scripts/build_crpto_journal_package.py +publication-integrity: + uv run python scripts/check_publication_integrity.py ijds-evidence: - uv run python scripts/build_ijds_calibration_selected_evidence.py - -# Explicit methodology replays. These write only to versioned experiment paths. -ijds-exact-alpha: - uv run python scripts/experiments/run_ijds_exact_alpha_grid_challenger.py --config configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml + uv run python scripts/build_ijds_binary_geometry_frontier_v4_evidence.py -ijds-policy-challenger: - uv run python scripts/experiments/run_ijds_calibration_selected_policy_challenger.py --config configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml +ijds-tie-evidence: + uv run python scripts/build_ijds_policy_support_tie_evidence.py -ijds-active-replay: ijds-exact-alpha ijds-policy-challenger ijds-evidence +ijds-v4 PHASE CONFIG: + uv run python scripts/experiments/run_ijds_binary_geometry_frontier_v4.py "{{ PHASE }}" --config "{{ CONFIG }}" -paper-export: tables figures evidence journal-package ijds-evidence book +ijds-credit-controls PHASE CONFIG: + uv run python scripts/experiments/run_ijds_credit_risk_controls.py "{{ PHASE }}" --config "{{ CONFIG }}" -# IJDS-oriented manuscript body (HTML writing preview). -paper-ijds: - uv run -- quarto render paper/CRPTO_ijds.qmd --to html --no-execute - -# IJDS-oriented online supplement (HTML writing preview). -paper-ijds-supplement: - uv run -- quarto render paper/supplement_ijds.qmd --to html --no-execute - -# Render the current submission-shaped manuscript surfaces. -paper-submission: paper-ijds paper-ijds-supplement - -# Compile and scan the official INFORMS/IJDS LaTeX handoff draft. -paper-submission-official: - @uv run python scripts/compile_ijds_submission.py - -# Final local IJDS gate before freezing or uploading. -submission-check: ijds-evidence publication-integrity lint type-check type-advisory-full test validate-champion paper-submission paper-submission-official - -# IJDS-oriented manuscript body (local HTML-print PDF verification draft). -paper-ijds-pdf: - uv run python scripts/render_submission_pdf_previews.py --body-only - -# IJDS-oriented online supplement (local HTML-print PDF verification draft). -paper-ijds-supplement-pdf: - uv run python scripts/render_submission_pdf_previews.py --supplement-only - -# Render local PDF verification drafts for the submission surfaces. -paper-submission-pdf: paper-submission - uv run python scripts/render_submission_pdf_previews.py +ijds-two-ruler-freeze CONFIG="configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml": + uv run python scripts/experiments/run_ijds_normalized_objective_frontier.py --config "{{ CONFIG }}" -# --- Quarto book --------------------------------------------------------- +ijds-two-ruler-evaluate CONFIG: + uv run python scripts/experiments/run_ijds_normalized_objective_frontier_v2.py --config "{{ CONFIG }}" -book: - uv run python scripts/write_book_build_info.py - uv run -- quarto render book --to html +ijds-raw-data-audit CONFIG="configs/experiments/ijds_raw_data_contract_2026-07-14_v2.yaml": + uv run python scripts/experiments/run_ijds_raw_data_audit.py --config "{{ CONFIG }}" -book-pdf: - @echo "CRPTO.pdf is intentionally not maintained as a routine artifact. Use paper-submission-pdf for IJDS PDFs; create a curated thesis PDF later from selected sections." +ijds-label-lag CONFIG="configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml": + uv run python scripts/experiments/run_ijds_label_lag_sensitivity.py --config "{{ CONFIG }}" -book-all: book - uv run python scripts/write_book_build_info.py - @echo "book-all currently means HTML book only; full thesis PDF is deferred until the thesis section set and APA layout are fixed." +ijds-fit-label-completion PHASE CONFIG="configs/experiments/ijds_fit_label_completion_sensitivity_2026-07-16.yaml": + uv run python scripts/experiments/run_ijds_fit_label_completion_sensitivity.py "{{ PHASE }}" --config "{{ CONFIG }}" -book-preview: - uv run -- quarto preview book +ijds-endpoint-sensitivity CONFIG="configs/experiments/ijds_endpoint_availability_sensitivity_2026-07-14.yaml": + uv run python scripts/experiments/run_ijds_endpoint_availability_sensitivity.py --config "{{ CONFIG }}" -book-clean: - uv run python -c "import shutil; [shutil.rmtree(p, ignore_errors=True) for p in ('book/_book', 'book/_freeze', 'book/.quarto')]; print('Quarto cache cleaned.')" +ijds-missingness PHASE CONFIG="configs/experiments/ijds_missingness_sensitivity_2026-07-15_v3.yaml": + uv run python scripts/experiments/run_ijds_missingness_sensitivity.py "{{ PHASE }}" --config "{{ CONFIG }}" -# --- DVC ----------------------------------------------------------------- +ijds-structure PHASE CONFIG="configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml": + uv run python scripts/experiments/run_ijds_portfolio_structure_sensitivity.py --phase "{{ PHASE }}" --config "{{ CONFIG }}" -dvc-status: - uv run dvc status +ijds-allocation-granularity PHASE CONFIG="configs/experiments/ijds_allocation_granularity_sensitivity_2026-07-16.yaml": + uv run python scripts/experiments/run_ijds_allocation_granularity_sensitivity.py "{{ PHASE }}" --config "{{ CONFIG }}" -dvc-dag: - uv run dvc dag --md +ijds-tie-audit CONFIG="configs/experiments/ijds_policy_support_tie_audit_2026-07-12.yaml": + uv run python scripts/experiments/run_ijds_policy_support_tie_audit.py --config "{{ CONFIG }}" -# Regenerates downstream paper artefacts only (does NOT touch champion stages). -dvc-paper: - uv run dvc repro --single-item crpto.paper.export_tables - uv run dvc repro --single-item crpto.paper.evidence - uv run dvc repro --single-item crpto.paper.journal_package - uv run dvc repro --single-item crpto.paper.tail_satisficing_audit - uv run dvc repro --single-item crpto.paper.figures - uv run dvc repro --single-item crpto.book.render +# Read-only gate over all registered lineages and current paper surfaces. +ijds-active-check: publication-integrity + uv run pytest -q tests/test_ijds_anonymity.py tests/test_ijds_active_claim_sync.py tests/test_ijds_v4_claim_sync.py tests/test_ijds_rolling_origin_protocol.py tests/test_publication_targets.py tests/test_submission_preview_layout.py tests/test_supplement_table_sync.py + uv run pytest -q tests/test_ijds_audit tests/test_ijds_audit_core.py tests/test_ijds_normalized_objective_frontier.py tests/test_ijds_normalized_objective_frontier_v2.py tests/test_ijds_policy_support_tie_audit.py tests/test_ijds_policy_support_tie_evidence.py -# --- dbt ----------------------------------------------------------------- +# --- DVC capsule ---------------------------------------------------------- -dbt-parse: - uv run dbt parse --project-dir dbt_project --profiles-dir dbt_project +ijds-pull: + uv run python scripts/manage_ijds_dvc_capsule.py pull -dbt-test: - uv run dbt test --project-dir dbt_project --profiles-dir dbt_project +ijds-push: + uv run python scripts/manage_ijds_dvc_capsule.py push -dbt-build: - uv run dbt build --project-dir dbt_project --profiles-dir dbt_project +ijds-dvc-status: + uv run python scripts/manage_ijds_dvc_capsule.py status -# --- Governance --------------------------------------------------------- +ijds-dvc-remote-status: + uv run python scripts/manage_ijds_dvc_capsule.py status --cloud -validate-champion: - uv run pytest tests/test_manifest_regression.py -q +ijds-dvc-verify-remote: + uv run python scripts/manage_ijds_dvc_capsule.py verify-remote -drift-gate: - $env:CRPTO_RUN_CHAMPION_DRIFT = "1"; uv run pytest tests/test_models/test_conformal_mapie_drift.py -q -s +# --- Manuscript ----------------------------------------------------------- -bound-audit: - uv run pytest tests/test_scripts/test_build_bound_tightening_audit.py tests/test_scripts/test_run_portfolio_bound_aware_search.py tests/test_scripts/test_run_portfolio_bound_exact_eval.py -q - -mrm-card: - uv run python -c "print('use /crpto-mrm-card via Claude Code or write the script')" +paper-body: + uv run -- quarto render paper/CRPTO_ijds.qmd --to html --no-execute -pipeline-state: - uv run python -c "from src.utils.pipeline_state import load_pipeline_state; import json; s = load_pipeline_state(); print(json.dumps({'missing': s.missing, 'namespaces': list(s.state.keys())}, indent=2))" +paper-supplement: + uv run -- quarto render paper/supplement_ijds.qmd --to html --no-execute -params-check: - uv run python scripts/build_params_view.py --check +paper-tex: + uv run python scripts/build_ijds_submission_tex.py -# --- Optuna Dashboard --------------------------------------------------- +paper-tex-check: + uv run python scripts/build_ijds_submission_tex.py --check -# Local HPO dashboard. Defaults to the journal file used by make_study(); pass -# OPTUNA_DASH_FILE to point at a different study. -optuna-dashboard FILE="data/processed/optuna/pd_catboost_hpo.log": - uv run optuna-dashboard "journal:{{FILE}}" +paper-official: paper-tex + @uv run python scripts/compile_ijds_submission.py --skip-render -# --- Dbt extras --------------------------------------------------------- +paper-official-scan: + @uv run python scripts/compile_ijds_submission.py --scan-only -dbt-deps: - uv run dbt deps --project-dir dbt_project --profiles-dir dbt_project +paper-pdf-audit: + @uv run python scripts/inspect_ijds_pdfs.py -dbt-docs: - uv run dbt docs generate --project-dir dbt_project --profiles-dir dbt_project - uv run dbt docs serve --project-dir dbt_project --profiles-dir dbt_project --port 8088 +paper-previews: paper-body paper-supplement + uv run python scripts/render_submission_pdf_previews.py -# --- DuckDB CLI --------------------------------------------------------- +submission-build: ijds-tie-evidence ijds-evidence paper-body paper-supplement paper-official paper-previews -# Interactive DuckDB session over the CRPTO warehouse. Useful for MRM -# reviewers who want to inspect the marts without booting a Quarto chunk. -duckdb FILE="data/processed/crpto.duckdb": - uv run duckdb "{{FILE}}" +validate-champion: + uv run pytest tests/test_manifest_regression.py -q -# Optional Datasette UI. Requires the duckdb-datasette plugin; if it is not -# installed this recipe fails fast with a helpful pointer. -datasette FILE="data/processed/crpto.duckdb": - @uv run python -c "import datasette" 2>&1 || echo "Run: uv pip install datasette datasette-duckdb" - uv run datasette serve --plugins-dir=. -i "{{FILE}}" +validate-champion-strict: + $env:CRPTO_REQUIRE_DVC_ARTIFACTS = "1"; uv run pytest tests/test_manifest_regression.py -q -# --- One-shot orchestrator ---------------------------------------------- +submission-check: ijds-active-check drift-gate paper-tex-check paper-official-scan paper-pdf-audit lint type-check type-check-fast validate-champion-strict -all: - uv run python scripts/run_crpto_pipeline.py +submission-closeout: submission-build submission-check ijds-dvc-verify-remote -# --- Help --------------------------------------------------------------- +all: submission-check + @echo "Read-only checks complete: no evidence-generating or protected stage was executed." help: @just --list diff --git a/models/ab_attribution_status.json b/models/ab_attribution_status.json deleted file mode 100644 index 40815b5..0000000 --- a/models/ab_attribution_status.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "schema_version": "2026-03-17.1", - "generated_at_utc": "2026-03-18T01:44:20.745591+00:00", - "run_tag": "paper-grade-2026-03-13-final-heavy-2026-03-13-230650", - "sharpe_metrics": { - "n_bootstrap": 15000, - "bootstrap_exceeds_10k": true, - "sharpe_like_diff": -0.1276, - "roic_a": 0.2095, - "roic_b": 0.2041, - "calmar_like_a": 0.5179, - "calmar_like_b": 0.5046, - "std_diff_approx": 0.1925, - "ci_95": [ - -0.4045, - 0.35 - ], - "p_value": 0.44753333333333334, - "note": "n_bootstrap=15,000 (\u226510K \u2713). Sharpe-like = E[diff] / std(diff) from 95% CI of bootstrap distribution. ROIC = total_return / total_allocated. Calmar-like = ROIC / |downside_CI|." - }, - "breakdown_dimensions": [ - "grade", - "cohort", - "amount" - ], - "artifact_paths": { - "grade": "data/processed/ab_attribution_by_grade.parquet", - "cohort": "data/processed/ab_attribution_by_cohort.parquet", - "amount": "data/processed/ab_attribution_by_amount.parquet" - }, - "note": "Attribution uses canonical portfolio_allocations + conformal_intervals_mondrian (OOT test set)." -} diff --git a/models/ab_simulation_status_explicit_champion_only_baseline.json b/models/ab_simulation_status_explicit_champion_only_baseline.json deleted file mode 100644 index 76bfe91..0000000 --- a/models/ab_simulation_status_explicit_champion_only_baseline.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "strategy_a": "non_robust", - "strategy_b": "robust_selected_for_champion", - "comparison": { - "mean_a": 2.1491526782721784, - "mean_b": 2.101386499204445, - "diff": -0.04776617906773328, - "ci_low": -1.1397842657614892, - "ci_high": 1.0047372634360316, - "p_value": 0.4688, - "significant": false - }, - "metrics_a": { - "total_return": 171932.21426177426, - "n_funded": 231, - "total_allocated": 999999.9999999999, - "avg_return_per_funded": 744.2952998345206 - }, - "metrics_b": { - "total_return": 168110.9199363556, - "n_funded": 254, - "total_allocated": 1000000.0000000001, - "avg_return_per_funded": 661.8540154974629 - }, - "n_candidates_available": 80000, - "n_candidates_used": 80000, - "max_candidates_requested": 150000, - "dataset_scope": "sampled_candidates", - "solver_backend": "highs", - "policy_selector": "explicit_champion_only", - "decision_scenario": "baseline", - "max_portfolio_pd_requested": 0.18, - "max_portfolio_pd_effective": 0.18, - "robust_policy": { - "source": "champion_policy_artifact::explicit_champion_only", - "risk_tolerance": 0.18, - "uncertainty_aversion": 0.25, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0 - }, - "champion_policy_path": "models/champion_portfolio_policy_candidate_canonical_monotonic_economic_only_2026_03_29.json", - "candidate_universe_path": "data/processed/champion_candidate_universe.parquet", - "gate_contract": { - "gate": "no_regression", - "significance_role": "diagnostic" - }, - "diagnostics": { - "p_value": 0.4688, - "significant": false, - "n_boot": 5000, - "seed": 42 - }, - "policy_search": [], - "frontier_path": "data/processed/portfolio_robustness_frontier.parquet", - "no_regression": { - "diff_total_return": -3821.294325418654, - "tolerance_total_return": 8596.610713088714, - "tolerance_pct_of_control": 0.05, - "passed": true - }, - "decision_scenario_meta": { - "decision_scenario": "baseline", - "rows_removed": 0, - "rows_remaining": 80000, - "ambiguity_rate_removed": 0.0 - }, - "baseline_comparison_context": { - "artifact_truth_role": "current_run_status", - "official_truth_may_live_in_comparison_json": true - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-03-30T15:36:55.064108+00:00", - "run_tag": "canonical-monotonic-promotion-2026-03-29-0929" -} diff --git a/models/alpha_sweep_status.json b/models/alpha_sweep_status.json deleted file mode 100644 index 84d7339..0000000 --- a/models/alpha_sweep_status.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-17T18:59:55.356271+00:00", - "run_tag": "untracked", - "alphas": [ - 0.01, - 0.03, - 0.05, - 0.07, - 0.1, - 0.12, - 0.15, - 0.2 - ], - "n_alphas": 8, - "n_cal": 237584, - "n_test": 276869, - "low_risk_threshold": 0.2, - "avg_interest_margin": 0.05, - "mondrian": true, - "global": true, - "pareto_path": "data/processed/alpha_sweep_pareto_both.parquet", - "summaries": { - "global": { - "best_coverage_alpha": 0.01, - "best_coverage": 0.9868313173378024, - "tightest_avg_width_alpha": 0.2, - "tightest_avg_width": 0.7418537400997983, - "most_eligible_alpha": 0.01, - "most_eligible_n": 0 - }, - "mondrian": { - "best_coverage_alpha": 0.01, - "best_coverage": 0.9898905258443523, - "tightest_avg_width_alpha": 0.2, - "tightest_avg_width": 0.5922833916523539, - "most_eligible_alpha": 0.2, - "most_eligible_n": 50399 - } - } -} diff --git a/models/champion_search_bundle.json b/models/champion_search_bundle.json deleted file mode 100644 index cd1ccd0..0000000 --- a/models/champion_search_bundle.json +++ /dev/null @@ -1,4116 +0,0 @@ -{ - "schema_version": "2026-03-13.2", - "generated_at_utc": "2026-04-07T17:44:00.138595+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "pipeline_family": "crpto_e2e", - "pipeline_profile": "crpto_e2e_default", - "artifact_scope": "search", - "promotion_state": "paper_grade", - "writes_canonical_artifacts": false, - "upstream_canonical_run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "artifact_run_tags": { - "governance": "crpto-e2e-all-champions-2026-04-07", - "portfolio": "crpto-e2e-all-champions-2026-04-07", - "threshold_semantics": "crpto-e2e-all-champions-2026-04-07", - "causal_effect_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "causal_policy_rule": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "causal_policy_oot_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "cate_portfolio_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "conformal_policy_status": "crpto-e2e-all-champions-2026-04-07", - "time_series": "canonical-audit-rebuild-2026-04-06-r2" - }, - "mixed_run_tags_present": true, - "threshold_semantics": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-07T17:42:58.069075+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "pd_internal_selected_threshold": 0.05, - "pd_internal_fallback_threshold": 0.35, - "fairness_primary_threshold": 0.35, - "decision_policy_global_threshold": 0.35, - "source_artifacts": { - "fairness_status": "models/fairness_audit_status.json", - "fairness_decision_policy": "models/fairness_decision_policy.json", - "fairness_frontier": "data/processed/fairness_threshold_frontier.parquet", - "decision_threshold": "models/decision_threshold.json", - "decision_threshold_v2": "models/decision_threshold_v2.json", - "test_predictions": "data/processed/test_predictions.parquet" - }, - "business_meaning": { - "pd_internal_selected_threshold": "Threshold interno de screening/seleccion PD usado en busqueda y analisis tecnico.", - "pd_internal_fallback_threshold": "Fallback interno tomado de la policy de fairness durante la busqueda de threshold PD.", - "fairness_primary_threshold": "Threshold operativo principal para auditoria de fairness y narrativa de aprobacion.", - "decision_policy_global_threshold": "Threshold operativo global aplicado por la policy de decision/aprobacion." - }, - "fairness_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval" - }, - "pd": { - "best_model": "CatBoost (tuned + calibrated)", - "best_calibration": "Venn-Abers", - "training_regime": { - "mode": "recent_8q", - "recent_window_quarters": 8, - "rows_after_recent_window": 798550 - }, - "stable_core": { - "enabled": true, - "excluded_features": [ - "rev_utilization", - "high_util_pct" - ], - "feature_count_after_filter": 42 - }, - "decision_threshold": { - "enabled": true, - "selected_threshold": 0.05, - "fallback_threshold": 0.35, - "selection_metrics": { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.12031281223211832, - "max_eo_gap": 0.13650858900821128, - "min_dir": 0.8764002995361211, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01141866959887805, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9885801768059442, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.12031281223211832, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8764002995361211, - "distance_from_fallback": 0.3 - }, - "search_summary": [ - { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.12031281223211832, - "max_eo_gap": 0.13650858900821128, - "min_dir": 0.8764002995361211, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01141866959887805, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9885801768059442, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.12031281223211832, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8764002995361211, - "distance_from_fallback": 0.3 - }, - { - "threshold": 0.060000000000000005, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.12923893180319368, - "max_eo_gap": 0.14659464365331498, - "min_dir": 0.866671635114004, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.020389620892614202, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9795969822845203, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.12923893180319368, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.866671635114004, - "distance_from_fallback": 0.29 - }, - { - "threshold": 0.07, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.13571798910399735, - "max_eo_gap": 0.15376483184759016, - "min_dir": 0.8594380032351373, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.029446728811672207, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9705021182180342, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.13571798910399735, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8594380032351373, - "distance_from_fallback": 0.27999999999999997 - }, - { - "threshold": 0.9500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0, - "max_eo_gap_secondary": 0.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0, - "robust_max_eo_gap": 0.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.6000000000000002 - }, - { - "threshold": 0.9400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 1.6729961688387733e-05, - "max_eo_gap_secondary": 7.053180984624065e-05, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 1.6729961688387733e-05, - "robust_max_eo_gap": 7.053180984624065e-05, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5900000000000002 - }, - { - "threshold": 0.9300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0001003797701303264, - "max_eo_gap_secondary": 0.00029645440531246293, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0001003797701303264, - "robust_max_eo_gap": 0.00029645440531246293, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5800000000000002 - }, - { - "threshold": 0.9200000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0004015190805213056, - "max_eo_gap_secondary": 0.0011265267401873592, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0004015190805213056, - "robust_max_eo_gap": 0.0011265267401873592, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5700000000000003 - }, - { - "threshold": 0.9100000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0011291934481546262, - "max_eo_gap_secondary": 0.0029211167372760563, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0011291934481546262, - "robust_max_eo_gap": 0.0029211167372760563, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5600000000000003 - }, - { - "threshold": 0.9000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 3.211716341212744e-05, - "max_eo_gap": 0.00016020506247997436, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.002673195187207413, - "max_eo_gap_secondary": 0.006536353120962595, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.002673195187207413, - "robust_max_eo_gap": 0.006536353120962595, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5500000000000003 - }, - { - "threshold": 0.8900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 3.211716341212744e-05, - "max_eo_gap": 0.00016020506247997436, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.00482604118641904, - "max_eo_gap_secondary": 0.011403876109005682, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.00482604118641904, - "robust_max_eo_gap": 0.011403876109005682, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5400000000000003 - }, - { - "threshold": 0.8800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 3.211716341212744e-05, - "max_eo_gap": 0.00016020506247997436, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.00788198925355742, - "max_eo_gap_secondary": 0.01774347829931586, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.00788198925355742, - "robust_max_eo_gap": 0.01774347829931586, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5300000000000002 - }, - { - "threshold": 0.8700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 3.211716341212744e-05, - "max_eo_gap": 0.00016020506247997436, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.011702354431394339, - "max_eo_gap_secondary": 0.025910904978268083, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.011702354431394339, - "robust_max_eo_gap": 0.025910904978268083, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5200000000000002 - }, - { - "threshold": 0.8600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 3.211716341212744e-05, - "max_eo_gap": 0.00016020506247997436, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.015934087148817885, - "max_eo_gap_secondary": 0.03438045680345147, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.015934087148817885, - "robust_max_eo_gap": 0.03438045680345147, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5100000000000002 - }, - { - "threshold": 0.8500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0001441794746099945, - "max_eo_gap": 0.0006215813028344107, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.02109081302282387, - "max_eo_gap_secondary": 0.0440358262498847, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.02109081302282387, - "robust_max_eo_gap": 0.0440358262498847, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5000000000000002 - }, - { - "threshold": 0.8400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00017301536953199343, - "max_eo_gap": 0.0007458975634012929, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.02644422595928661, - "max_eo_gap_secondary": 0.05611024011882169, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.02644422595928661, - "robust_max_eo_gap": 0.05611024011882169, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4900000000000002 - }, - { - "threshold": 0.8300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00017301536953199343, - "max_eo_gap": 0.0007458975634012929, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.03278282832808537, - "max_eo_gap_secondary": 0.07009654261902798, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.03278282832808537, - "robust_max_eo_gap": 0.07009654261902798, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4800000000000002 - }, - { - "threshold": 0.8200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.04006143343949919, - "max_eo_gap_secondary": 0.085856918887697, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.04006143343949919, - "robust_max_eo_gap": 0.085856918887697, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4700000000000002 - }, - { - "threshold": 0.8100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.04625852495405232, - "max_eo_gap_secondary": 0.10141100750887037, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.04625852495405232, - "robust_max_eo_gap": 0.10141100750887037, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4600000000000002 - }, - { - "threshold": 0.8000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.054996692337961534, - "max_eo_gap_secondary": 0.11791401930852381, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.054996692337961534, - "robust_max_eo_gap": 0.11791401930852381, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4500000000000002 - }, - { - "threshold": 0.7900000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.06189877747337684, - "max_eo_gap_secondary": 0.1349946365211651, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.06189877747337684, - "robust_max_eo_gap": 0.1349946365211651, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.44000000000000017 - }, - { - "threshold": 0.7800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.07035534453340261, - "max_eo_gap_secondary": 0.15195148114530901, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.07035534453340261, - "robust_max_eo_gap": 0.15195148114530901, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.43000000000000016 - }, - { - "threshold": 0.7700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0006636435066027812, - "max_eo_gap": 0.002287191726332538, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0784011589081359, - "max_eo_gap_secondary": 0.16927964353494512, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0784011589081359, - "robust_max_eo_gap": 0.16927964353494512, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.42000000000000026 - }, - { - "threshold": 0.7600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0007789870862907768, - "max_eo_gap": 0.0025230484296530136, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.08679314135135802, - "max_eo_gap_secondary": 0.18726792639656736, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.08679314135135802, - "robust_max_eo_gap": 0.18726792639656736, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.41000000000000025 - }, - { - "threshold": 0.7500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.000980838350744769, - "max_eo_gap": 0.0031446297324874244, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.09614310307967008, - "max_eo_gap_secondary": 0.20703028302665236, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.09614310307967008, - "robust_max_eo_gap": 0.20703028302665236, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.40000000000000024 - }, - { - "threshold": 0.7400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.000980838350744769, - "max_eo_gap": 0.0031446297324874244, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.10509965518345735, - "max_eo_gap_secondary": 0.22638006436174601, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.10509965518345735, - "robust_max_eo_gap": 0.22638006436174601, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.39000000000000024 - }, - { - "threshold": 0.7300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0010385101405887668, - "max_eo_gap": 0.0031446297324874244, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11608467754849316, - "max_eo_gap_secondary": 0.24511098275435267, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.11608467754849316, - "robust_max_eo_gap": 0.24511098275435267, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3800000000000002 - }, - { - "threshold": 0.7200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0010385101405887668, - "max_eo_gap": 0.0031446297324874244, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1273899629562414, - "max_eo_gap_secondary": 0.26474956679594025, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1273899629562414, - "robust_max_eo_gap": 0.26474956679594025, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3700000000000002 - }, - { - "threshold": 0.7100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0010385101405887668, - "max_eo_gap": 0.0031446297324874244, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1392503709713578, - "max_eo_gap_secondary": 0.2855433616635036, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1392503709713578, - "robust_max_eo_gap": 0.2855433616635036, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3600000000000002 - }, - { - "threshold": 0.7000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012407793284395657, - "max_eo_gap": 0.003554301732503441, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1516659015938424, - "max_eo_gap_secondary": 0.30571829358857994, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1516659015938424, - "robust_max_eo_gap": 0.30571829358857994, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3500000000000002 - }, - { - "threshold": 0.6900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0013272870132055624, - "max_eo_gap": 0.0036786179930703234, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.16505289677922133, - "max_eo_gap_secondary": 0.3270896938691311, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.16505289677922133, - "robust_max_eo_gap": 0.3270896938691311, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3400000000000002 - }, - { - "threshold": 0.6800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.001442630592893558, - "max_eo_gap": 0.0039272505142040875, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.17893096196342595, - "max_eo_gap_secondary": 0.34858486673817973, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.17893096196342595, - "robust_max_eo_gap": 0.34858486673817973, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3300000000000002 - }, - { - "threshold": 0.6700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0014430485162903644, - "max_eo_gap": 0.004026552346296447, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19320401823364255, - "max_eo_gap_secondary": 0.36999752454823004, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.19320401823364255, - "robust_max_eo_gap": 0.36999752454823004, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3200000000000002 - }, - { - "threshold": 0.6600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.003589897337454852, - "max_eo_gap": 0.009966280609252344, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.207722609503272, - "max_eo_gap_secondary": 0.38951233600132024, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.207722609503272, - "robust_max_eo_gap": 0.38951233600132024, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.31000000000000016 - }, - { - "threshold": 0.6500000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.011297820701325921, - "max_eo_gap": 0.029325945787121138, - "min_dir": 0.12153111188053232, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22213444642533067, - "max_eo_gap_secondary": 0.409811040514894, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.22213444642533067, - "robust_max_eo_gap": 0.409811040514894, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.30000000000000016 - }, - { - "threshold": 0.6400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.011355492491169919, - "max_eo_gap": 0.029325945787121138, - "min_dir": 0.12098856227392281, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23719748486757122, - "max_eo_gap_secondary": 0.4302335176169651, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23719748486757122, - "robust_max_eo_gap": 0.4302335176169651, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.29000000000000015 - }, - { - "threshold": 0.6300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.012339256305692332, - "max_eo_gap": 0.030582011450403068, - "min_dir": 0.12670802158141753, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2532533387422203, - "max_eo_gap_secondary": 0.4521825233105042, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2532533387422203, - "robust_max_eo_gap": 0.4521825233105042, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.28000000000000014 - }, - { - "threshold": 0.6200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01239692809553633, - "max_eo_gap": 0.030582011450403068, - "min_dir": 0.1261929483229565, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.26978958718093793, - "max_eo_gap_secondary": 0.4757405726545095, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26978958718093793, - "robust_max_eo_gap": 0.4757405726545095, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.27000000000000013 - }, - { - "threshold": 0.6100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.012541525493543133, - "max_eo_gap": 0.030867964624536677, - "min_dir": 0.1266514807418007, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2866567740971251, - "max_eo_gap_secondary": 0.4973595181120555, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2866567740971251, - "robust_max_eo_gap": 0.4973595181120555, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2600000000000001 - }, - { - "threshold": 0.6000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.012541525493543133, - "max_eo_gap": 0.030967407743072876, - "min_dir": 0.1266514807418007, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3035026101437981, - "max_eo_gap_secondary": 0.5178232527436257, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3035026101437981, - "robust_max_eo_gap": 0.5178232527436257, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2500000000000002 - }, - { - "threshold": 0.08000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.1574909610212114, - "max_eo_gap": 0.17734333726059515, - "min_dir": 0.8345880059923615, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.03935030951415652, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9605154072388612, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1574909610212114, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8345880059923615, - "distance_from_fallback": 0.26999999999999996 - }, - { - "threshold": 0.09000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.19460664923286375, - "max_eo_gap": 0.21452754668404517, - "min_dir": 0.7861066180037244, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.04905289923119993, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.950671373379108, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.19460664923286375, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7861066180037244, - "distance_from_fallback": 0.25999999999999995 - }, - { - "threshold": 0.1, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.20906261707002327, - "max_eo_gap": 0.22341656329346438, - "min_dir": 0.7680641576067263, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.05833398144488822, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.941194752853011, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.20906261707002327, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7680641576067263, - "distance_from_fallback": 0.24999999999999997 - }, - { - "threshold": 0.11000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2285259963647991, - "max_eo_gap": 0.23921459120299715, - "min_dir": 0.7424924282578999, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.06699327867926164, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9322331690378856, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2285259963647991, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7424924282578999, - "distance_from_fallback": 0.23999999999999996 - }, - { - "threshold": 0.28, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.22767700710954317, - "max_eo_gap": 0.2195646449296827, - "min_dir": 0.4036998014083641, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23098373641235648, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7373824800453503, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23098373641235648, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4036998014083641, - "distance_from_fallback": 0.06999999999999995 - }, - { - "threshold": 0.29000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.22139537589608171, - "max_eo_gap": 0.22070936085899245, - "min_dir": 0.3983881648095736, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2408731003566248, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7237160763813221, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2408731003566248, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3983881648095736, - "distance_from_fallback": 0.05999999999999994 - }, - { - "threshold": 0.30000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.22142337594421013, - "max_eo_gap": 0.22113317946462924, - "min_dir": 0.39826492795476326, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2488683799471707, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7114277740677366, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2488683799471707, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.39826492795476326, - "distance_from_fallback": 0.04999999999999993 - }, - { - "threshold": 0.5900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.018793482506916055, - "max_eo_gap": 0.04583589933592687, - "min_dir": 0.1535854807047494, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2491285788685242, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.38842816769853783, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2491285788685242, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1535854807047494, - "distance_from_fallback": 0.2400000000000002 - }, - { - "threshold": 0.27, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.25054639243439136, - "max_eo_gap": 0.22894049879719608, - "min_dir": 0.42565452516973407, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22083450530123727, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7511464116778677, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.25054639243439136, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.42565452516973407, - "distance_from_fallback": 0.07999999999999996 - }, - { - "threshold": 0.26000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.25242406981095344, - "max_eo_gap": 0.22973997959306036, - "min_dir": 0.42669367234435407, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2531142167011732, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7468857832988267, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2531142167011732, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.42669367234435407, - "distance_from_fallback": 0.08999999999999991 - }, - { - "threshold": 0.12000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2531514138572636, - "max_eo_gap": 0.2633081473287928, - "min_dir": 0.7079980747993997, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.0756379351768407, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9231707430908312, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2531514138572636, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7079980747993997, - "distance_from_fallback": 0.22999999999999998 - }, - { - "threshold": 0.13, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.254495834090823, - "max_eo_gap": 0.2647553635186287, - "min_dir": 0.7059877063616612, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.08473810079100796, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9135829086469778, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.254495834090823, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7059877063616612, - "distance_from_fallback": 0.21999999999999997 - }, - { - "threshold": 0.5800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.028860971964722565, - "max_eo_gap": 0.06073048224171591, - "min_dir": 0.1631524692603552, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.25739520539113037, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.3983072502211952, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.25739520539113037, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1631524692603552, - "distance_from_fallback": 0.2300000000000002 - }, - { - "threshold": 0.31000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.20384893583235386, - "max_eo_gap": 0.21060117647071225, - "min_dir": 0.38931606377589817, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2578435467836605, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.69775489641709, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2578435467836605, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38931606377589817, - "distance_from_fallback": 0.039999999999999925 - }, - { - "threshold": 0.32000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.19570425374176095, - "max_eo_gap": 0.20840650316035697, - "min_dir": 0.3786663173569046, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2662839697004159, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6845075972070769, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2662839697004159, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3786663173569046, - "distance_from_fallback": 0.029999999999999916 - }, - { - "threshold": 0.5700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.028860971964722565, - "max_eo_gap": 0.06073048224171591, - "min_dir": 0.1631524692603552, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2666712645368764, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4057656764403998, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2666712645368764, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1631524692603552, - "distance_from_fallback": 0.2200000000000002 - }, - { - "threshold": 0.25000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.27037630611786156, - "max_eo_gap": 0.24274316602622986, - "min_dir": 0.44234685857848677, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23773291925465834, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7622670807453414, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.27037630611786156, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.44234685857848677, - "distance_from_fallback": 0.09999999999999992 - }, - { - "threshold": 0.33, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.18712787071304698, - "max_eo_gap": 0.20387782432897933, - "min_dir": 0.375598246083156, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2740071984350957, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6713969689807595, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2740071984350957, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.375598246083156, - "distance_from_fallback": 0.019999999999999962 - }, - { - "threshold": 0.5600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.028860971964722565, - "max_eo_gap": 0.06073048224171591, - "min_dir": 0.1631524692603552, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2741232671915205, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.41617067344241004, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2741232671915205, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1631524692603552, - "distance_from_fallback": 0.2100000000000002 - }, - { - "threshold": 0.34, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16492840184304183, - "max_eo_gap": 0.1911650048488347, - "min_dir": 0.3562687622380584, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2811633288480262, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6584012428303371, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2811633288480262, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3562687622380584, - "distance_from_fallback": 0.009999999999999953 - }, - { - "threshold": 0.5500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.03216456299830589, - "max_eo_gap": 0.06556738320673045, - "min_dir": 0.17313945128372768, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28146972894906713, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4259097076205352, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28146972894906713, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.17313945128372768, - "distance_from_fallback": 0.20000000000000018 - }, - { - "threshold": 0.14, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.28226686593396266, - "max_eo_gap": 0.2906425884347771, - "min_dir": 0.6618277950762145, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.3333333333333333, - "max_dpd_secondary": 0.09353691714037538, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9041456395709153, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28226686593396266, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6618277950762145, - "distance_from_fallback": 0.20999999999999996 - }, - { - "threshold": 0.24000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2875729598681577, - "max_eo_gap": 0.2646352011342974, - "min_dir": 0.4596183866015799, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22295548654244302, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7770445134575568, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2875729598681577, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4596183866015799, - "distance_from_fallback": 0.10999999999999993 - }, - { - "threshold": 0.5400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.03891592454008253, - "max_eo_gap": 0.07853842434105532, - "min_dir": 0.1870088270328171, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2876199194800825, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4359426015590087, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2876199194800825, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1870088270328171, - "distance_from_fallback": 0.19000000000000017 - }, - { - "threshold": 0.35000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16492840184304183, - "max_eo_gap": 0.1911650048488347, - "min_dir": 0.3562687622380584, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2878056757123377, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6455253620499563, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2878056757123377, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3562687622380584, - "distance_from_fallback": 5.551115123125783e-17 - }, - { - "threshold": 0.5300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.03891592454008253, - "max_eo_gap": 0.07853842434105532, - "min_dir": 0.1870088270328171, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.29358623836414854, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4452636332509244, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.29358623836414854, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1870088270328171, - "distance_from_fallback": 0.18000000000000016 - }, - { - "threshold": 0.36000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16489956594811986, - "max_eo_gap": 0.19104068858826784, - "min_dir": 0.35630886453007077, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2937447324592446, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6330615696282876, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2937447324592446, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.35630886453007077, - "distance_from_fallback": 0.010000000000000064 - }, - { - "threshold": 0.23000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2987228520761669, - "max_eo_gap": 0.2794050273886324, - "min_dir": 0.4796096957276638, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.20849723947550036, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7915027605244994, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2987228520761669, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4796096957276638, - "distance_from_fallback": 0.11999999999999994 - }, - { - "threshold": 0.5200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.04753785876067548, - "max_eo_gap": 0.09169007837693702, - "min_dir": 0.21309527257209218, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2991095912203319, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4552505302860175, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2991095912203319, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.21309527257209218, - "distance_from_fallback": 0.17000000000000015 - }, - { - "threshold": 0.37000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.13015397196883377, - "max_eo_gap": 0.17585355665686572, - "min_dir": 0.3225859832046842, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2994312773367929, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6203969174266234, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2994312773367929, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3225859832046842, - "distance_from_fallback": 0.020000000000000073 - }, - { - "threshold": 0.5100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.047509440789150294, - "max_eo_gap": 0.09147816907411863, - "min_dir": 0.21356568156231756, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3038319421543746, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4649417630548159, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3038319421543746, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.21356568156231756, - "distance_from_fallback": 0.16000000000000014 - }, - { - "threshold": 0.38000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.12615539317388721, - "max_eo_gap": 0.17317818853689748, - "min_dir": 0.3175880705229704, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30388465031312173, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6083804664920393, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30388465031312173, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3175880705229704, - "distance_from_fallback": 0.030000000000000082 - }, - { - "threshold": 0.22000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3040483065165834, - "max_eo_gap": 0.2876545964792003, - "min_dir": 0.4918510254607904, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19395272601794344, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8060472739820563, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3040483065165834, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4918510254607904, - "distance_from_fallback": 0.12999999999999995 - }, - { - "threshold": 0.5000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.047509440789150294, - "max_eo_gap": 0.09147816907411863, - "min_dir": 0.21356568156231756, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30693554241024934, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4759652048989477, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30693554241024934, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.21356568156231756, - "distance_from_fallback": 0.15000000000000013 - }, - { - "threshold": 0.15000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30784552818003336, - "max_eo_gap": 0.31125118281766617, - "min_dir": 0.6092322448112965, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.3333333333333333, - "max_dpd_secondary": 0.10387336093857835, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8951258320197172, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30784552818003336, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6092322448112965, - "distance_from_fallback": 0.19999999999999996 - }, - { - "threshold": 0.39000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10946483810146249, - "max_eo_gap": 0.15643308592316324, - "min_dir": 0.3017967407539784, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3082424887830414, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5961028063654177, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3082424887830414, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3017967407539784, - "distance_from_fallback": 0.04000000000000009 - }, - { - "threshold": 0.16000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30976875183172853, - "max_eo_gap": 0.31187727911305574, - "min_dir": 0.6015108485506151, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11612318840579705, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8838768115942027, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30976875183172853, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6015108485506151, - "distance_from_fallback": 0.18999999999999995 - }, - { - "threshold": 0.49000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.056888630898857734, - "max_eo_gap": 0.10591542553253146, - "min_dir": 0.23355057072980218, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3104592700156127, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.48586795737121896, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3104592700156127, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23355057072980218, - "distance_from_fallback": 0.14000000000000007 - }, - { - "threshold": 0.4000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10946483810146249, - "max_eo_gap": 0.15643308592316324, - "min_dir": 0.3017967407539784, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31107586450041574, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.584743559220507, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31107586450041574, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3017967407539784, - "distance_from_fallback": 0.0500000000000001 - }, - { - "threshold": 0.21000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3119464260949251, - "max_eo_gap": 0.2990105886940282, - "min_dir": 0.504511972218847, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1799344375431332, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8200655624568666, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3119464260949251, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.504511972218847, - "distance_from_fallback": 0.13999999999999996 - }, - { - "threshold": 0.4800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.06641074671298475, - "max_eo_gap": 0.12057302130083651, - "min_dir": 0.24613476750913255, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31303219882955696, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.49627477128444974, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31303219882955696, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.24613476750913255, - "distance_from_fallback": 0.13000000000000012 - }, - { - "threshold": 0.4100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10568566335363576, - "max_eo_gap": 0.15435943590186163, - "min_dir": 0.2997567597553044, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31360434429284134, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5733647615582308, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31360434429284134, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2997567597553044, - "distance_from_fallback": 0.06000000000000011 - }, - { - "threshold": 0.2, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3136398082675296, - "max_eo_gap": 0.3018842957732031, - "min_dir": 0.5118390148148798, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.16645100069013108, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8335489993098687, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3136398082675296, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5118390148148798, - "distance_from_fallback": 0.14999999999999997 - }, - { - "threshold": 0.4700000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.07278724244023305, - "max_eo_gap": 0.12872101829427446, - "min_dir": 0.2621488510421386, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31471624550349175, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5071127667955715, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31471624550349175, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2621488510421386, - "distance_from_fallback": 0.1200000000000001 - }, - { - "threshold": 0.42000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10548381208918177, - "max_eo_gap": 0.15423511964129477, - "min_dir": 0.30015819409972516, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3153008722735208, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5616486244963211, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3153008722735208, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.30015819409972516, - "distance_from_fallback": 0.07000000000000006 - }, - { - "threshold": 0.43000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.09422067575196022, - "max_eo_gap": 0.14770352381671942, - "min_dir": 0.2884410247381903, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3162792416101878, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5509724215294817, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3162792416101878, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2884410247381903, - "distance_from_fallback": 0.08000000000000007 - }, - { - "threshold": 0.4600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.07287375012499905, - "max_eo_gap": 0.12872101829427446, - "min_dir": 0.261919164548819, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31652600378752405, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5171572880667701, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31652600378752405, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.261919164548819, - "distance_from_fallback": 0.1100000000000001 - }, - { - "threshold": 0.44000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0802310842029921, - "max_eo_gap": 0.13440526668906944, - "min_dir": 0.2725924786730548, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3166040537918443, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5400565291700105, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3166040537918443, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2725924786730548, - "distance_from_fallback": 0.09000000000000008 - }, - { - "threshold": 0.45000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.07595584831393612, - "max_eo_gap": 0.1306336288366938, - "min_dir": 0.2664792916516311, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3166241279801753, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5289919613310016, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3166241279801753, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2664792916516311, - "distance_from_fallback": 0.10000000000000009 - }, - { - "threshold": 0.17000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.317654726228376, - "max_eo_gap": 0.31700544174678064, - "min_dir": 0.5747394900064139, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.128416149068323, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8715838509316768, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.317654726228376, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5747394900064139, - "distance_from_fallback": 0.17999999999999994 - }, - { - "threshold": 0.18000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.31926198750022095, - "max_eo_gap": 0.31127384880474346, - "min_dir": 0.5341375719717173, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.14103692201518292, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8589630779848169, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31926198750022095, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5341375719717173, - "distance_from_fallback": 0.16999999999999993 - }, - { - "threshold": 0.19, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3194024056642597, - "max_eo_gap": 0.31142376838330765, - "min_dir": 0.5338542199296833, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.15356280193236715, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8464371980676326, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3194024056642597, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5338542199296833, - "distance_from_fallback": 0.15999999999999998 - } - ], - "source": "validation_fairness_search", - "fairness_policy_path": "configs/fairness_policy.yaml", - "validation_rows": 119782, - "secondary_validation_rows": 237584, - "calibration_method": "venn_abers", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T14:38:35.246585+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - }, - "decision_threshold_semantics": { - "pd_internal_selected_threshold": 0.05, - "fairness_primary_threshold": 0.35, - "decision_policy_global_threshold": 0.35 - }, - "set_prediction": { - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-30T16:09:12.760437+00:00", - "run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "status": "promoted_guardrail", - "promoted": true, - "method": "lac", - "alpha": 0.1, - "confidence_level": 0.9, - "summary": { - "n_obs": 276869.0, - "set_coverage": 0.8978072662522709, - "singleton_rate": 0.4526183863126605, - "ambiguity_rate": 0.5473816136873395, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2389065211510165, - "default_rate_singleton_positive": 0.45251658213031604, - "default_rate_overall": 0.21977902907151034 - }, - "promotion_gate": { - "coverage": 0.8978072662522709, - "min_coverage": 0.85, - "grade_a_singleton_rate": 0.8644112283803945, - "min_grade_a_singleton": 0.8, - "grades_with_singleton_above_40pct": 5, - "min_grades_above_40pct": 3, - "pass": true - }, - "promotion_rationale": "Conformal triage guardrail: binary set prediction provides distribution-free abstention mechanism. Grade A has 86% singleton rate (clear decisions); 5 grades have >40% singleton rate. Overall coverage 89.8%. Promoted as complementary guardrail to PD point + conformal intervals.", - "artifact_path": "data/processed/pd_set_prediction_cases.parquet", - "by_slice_path": "data/processed/pd_set_prediction_by_slice.parquet", - "calibration_size_sensitivity_path": "data/processed/pd_set_prediction_sensitivity.parquet", - "slice_metrics": { - "grade": [ - { - "slice_name": "grade", - "slice_value": "A", - "n_obs": 72388, - "set_coverage": 0.9356246891750014, - "singleton_rate": 0.8644112283803945, - "ambiguity_rate": 0.13558877161960545, - "default_rate": 0.08658893739293806, - "default_rate_ambiguous": 0.16362710137544575 - }, - { - "slice_name": "grade", - "slice_value": "B", - "n_obs": 75311, - "set_coverage": 0.9601651817131628, - "singleton_rate": 0.3047098033487804, - "ambiguity_rate": 0.6952901966512196, - "default_rate": 0.17861932519817822, - "default_rate_ambiguous": 0.20405629929530394 - }, - { - "slice_name": "grade", - "slice_value": "C", - "n_obs": 70568, - "set_coverage": 0.9118155537920871, - "singleton_rate": 0.17473925858746173, - "ambiguity_rate": 0.8252607414125382, - "default_rate": 0.2760741412538261, - "default_rate_ambiguous": 0.2571045898655494 - }, - { - "slice_name": "grade", - "slice_value": "D", - "n_obs": 45637, - "set_coverage": 0.7728159169095251, - "singleton_rate": 0.41740254617963496, - "ambiguity_rate": 0.5825974538203651, - "default_rate": 0.34745929837631745, - "default_rate_ambiguous": 0.28095381375056416 - }, - { - "slice_name": "grade", - "slice_value": "E", - "n_obs": 10828, - "set_coverage": 0.7051163649796823, - "singleton_rate": 0.6031584780199483, - "ambiguity_rate": 0.39684152198005174, - "default_rate": 0.4314739564093092, - "default_rate_ambiguous": 0.3223178962066558 - }, - { - "slice_name": "grade", - "slice_value": "F", - "n_obs": 1731, - "set_coverage": 0.5915655690352397, - "singleton_rate": 0.8937030618139804, - "ambiguity_rate": 0.10629693818601964, - "default_rate": 0.5210860774119006, - "default_rate_ambiguous": 0.3532608695652174 - }, - { - "slice_name": "grade", - "slice_value": "G", - "n_obs": 406, - "set_coverage": 0.6477832512315271, - "singleton_rate": 0.8300492610837439, - "ambiguity_rate": 0.16995073891625614, - "default_rate": 0.5344827586206896, - "default_rate_ambiguous": 0.3333333333333333 - } - ], - "term": [ - { - "slice_name": "term", - "slice_value": "36", - "n_obs": 204234, - "set_coverage": 0.9457925712662926, - "singleton_rate": 0.4465906754017451, - "ambiguity_rate": 0.553409324598255, - "default_rate": 0.18582116591752598, - "default_rate_ambiguous": 0.2411413404114134 - }, - { - "slice_name": "term", - "slice_value": "60", - "n_obs": 72635, - "set_coverage": 0.7628829076891306, - "singleton_rate": 0.4695670131479314, - "ambiguity_rate": 0.5304329868520685, - "default_rate": 0.31526123769532594, - "default_rate_ambiguous": 0.23235049833887042 - } - ], - "issue_quarter": [ - { - "slice_name": "issue_quarter", - "slice_value": "2018Q1", - "n_obs": 53924, - "set_coverage": 0.8959276018099548, - "singleton_rate": 0.464524145093094, - "ambiguity_rate": 0.535475854906906, - "default_rate": 0.2329945849714413, - "default_rate_ambiguous": 0.2536103896103896 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q2", - "n_obs": 56915, - "set_coverage": 0.8948783273302293, - "singleton_rate": 0.46479838355442327, - "ambiguity_rate": 0.5352016164455767, - "default_rate": 0.2556268119125011, - "default_rate_ambiguous": 0.27786349758707857 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q3", - "n_obs": 47168, - "set_coverage": 0.8923634667571235, - "singleton_rate": 0.46431902985074625, - "ambiguity_rate": 0.5356809701492538, - "default_rate": 0.23289094301221167, - "default_rate_ambiguous": 0.25309692484268015 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q4", - "n_obs": 39171, - "set_coverage": 0.8998238492762503, - "singleton_rate": 0.44415511475326136, - "ambiguity_rate": 0.5558448852467387, - "default_rate": 0.23305506624798958, - "default_rate_ambiguous": 0.24962109034124833 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q1", - "n_obs": 27508, - "set_coverage": 0.9020285007997674, - "singleton_rate": 0.42805729242402213, - "ambiguity_rate": 0.5719427075759779, - "default_rate": 0.21241093500072705, - "default_rate_ambiguous": 0.23231424394584632 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q2", - "n_obs": 22737, - "set_coverage": 0.9052645467739807, - "singleton_rate": 0.43466596296784976, - "ambiguity_rate": 0.5653340370321502, - "default_rate": 0.20191757927606985, - "default_rate_ambiguous": 0.2265442663762253 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q3", - "n_obs": 16603, - "set_coverage": 0.9044148647834729, - "singleton_rate": 0.4385352044811179, - "ambiguity_rate": 0.5614647955188822, - "default_rate": 0.1451544901523821, - "default_rate_ambiguous": 0.16659515125509547 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q4", - "n_obs": 8557, - "set_coverage": 0.8926025476218301, - "singleton_rate": 0.4260839079116513, - "ambiguity_rate": 0.5739160920883487, - "default_rate": 0.08425850181138249, - "default_rate_ambiguous": 0.08979841172877215 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q1", - "n_obs": 2977, - "set_coverage": 0.9012428619415519, - "singleton_rate": 0.42089351696338595, - "ambiguity_rate": 0.579106483036614, - "default_rate": 0.0141081625797783, - "default_rate_ambiguous": 0.011020881670533642 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q2", - "n_obs": 824, - "set_coverage": 0.9514563106796117, - "singleton_rate": 0.5242718446601942, - "ambiguity_rate": 0.47572815533980584, - "default_rate": 0.01820388349514563, - "default_rate_ambiguous": 0.025510204081632654 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q3", - "n_obs": 485, - "set_coverage": 0.9814432989690721, - "singleton_rate": 0.5030927835051546, - "ambiguity_rate": 0.49690721649484537, - "default_rate": 0.002061855670103093, - "default_rate_ambiguous": 0.0 - } - ] - }, - "decision_use_case": { - "probability_first": true, - "set_first": false, - "recommended_guardrail": "selective_ambiguity_defer" - }, - "promotion_note": "Binary conformal set prediction promoted as triage guardrail. When predicted set = {{0,1}}, the loan is ambiguous and may be deferred to human review — especially informative for Grade A (low ambiguity rate)." - }, - "rare_event_calibration": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-07T17:41:19.460102+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "summary": { - "n_obs": 276869, - "prevalence": 0.21977902907151034, - "pr_auc": 0.39707597765141317, - "brier": 0.15463051782916892, - "ece": 0.006379675621599359 - }, - "global": { - "n_obs": 276869, - "prevalence": 0.21977902907151034, - "pr_auc": 0.39707597765141317, - "brier": 0.15463051782916892, - "ece": 0.006379675621599359 - }, - "worst_protected_group_ece": 0.021540776718198477, - "worst_grade_brier": 0.24104901112917315, - "max_decile_gap": 0.016875745454345503, - "worst_protected_group": { - "slice_name": "home_ownership", - "slice_value": "RENT", - "ece": 0.021540776718198477, - "brier": 0.17641635233321967, - "pr_auc": 0.4548848383534659, - "n": 103518 - }, - "worst_grade_group": { - "slice_value": "F", - "ece": 0.021597948225972937, - "brier": 0.24104901112917315, - "pr_auc": 0.6160751280992285, - "n": 1731 - }, - "report_path": "data/processed/pd_rare_event_calibration_report.parquet", - "notes": [ - "Rare-event calibration audit is diagnostic; it should inform PD selection but not replace the core champion metric stack.", - "Coverage-by-group should be interpreted jointly with fairness/governance, without collapsing validity into fairness claims." - ] - }, - "calibration_diagnostics": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-03-13T23:00:22.826913+00:00", - "selected_method": "venn_abers", - "candidate_comparison": [ - { - "method": "platt", - "auc": 0.712893562122109, - "brier": 0.15460779745505918, - "ece": 0.008388765396486227 - }, - { - "method": "isotonic", - "auc": 0.7127913494994733, - "brier": 0.15453854630270586, - "ece": 0.006247440054484605 - }, - { - "method": "venn_abers", - "auc": 0.7128133359856964, - "brier": 0.15453739375452677, - "ece": 0.0061293170581817056 - } - ], - "venn_abers": { - "mean_p0": 0.21644999110967125, - "mean_p1": 0.2169528448444764, - "prevalence_observed": 0.21977902907151034, - "unbiasedness_in_the_large": false, - "avg_width": 0.0005028537348051775, - "median_width": 0.00026392187912377407, - "selector_ece": 0.0062228836249861785, - "selector_reliability_bins": [ - { - "bin": "(0.999, 27687.8]", - "n": 27687, - "empirical_rate": 0.04344999458229494, - "mean_selector_prob": 0.03586890118501795 - }, - { - "bin": "(110748.2, 138435.0]", - "n": 27687, - "empirical_rate": 0.1818904178856503, - "mean_selector_prob": 0.17437626091554378 - }, - { - "bin": "(138435.0, 166121.8]", - "n": 27686, - "empirical_rate": 0.24597269378024994, - "mean_selector_prob": 0.21290472279492598 - }, - { - "bin": "(166121.8, 193808.6]", - "n": 27687, - "empirical_rate": 0.250225737710839, - "mean_selector_prob": 0.2520684294541225 - }, - { - "bin": "(193808.6, 221495.4]", - "n": 27687, - "empirical_rate": 0.3133600606782967, - "mean_selector_prob": 0.3001931651732964 - }, - { - "bin": "(221495.4, 249182.2]", - "n": 27687, - "empirical_rate": 0.35500415357387943, - "mean_selector_prob": 0.3674842752610861 - }, - { - "bin": "(249182.2, 276869.0]", - "n": 27687, - "empirical_rate": 0.4915303210893199, - "mean_selector_prob": 0.49226632808875687 - }, - { - "bin": "(27687.8, 55374.6]", - "n": 27687, - "empirical_rate": 0.08343265792610251, - "mean_selector_prob": 0.0766410468347081 - }, - { - "bin": "(55374.6, 83061.4]", - "n": 27687, - "empirical_rate": 0.11369957019539856, - "mean_selector_prob": 0.10866640959629549 - }, - { - "bin": "(83061.4, 110748.2]", - "n": 27687, - "empirical_rate": 0.11922562935673782, - "mean_selector_prob": 0.1453369268637383 - } - ], - "temporal_stability": [ - { - "issue_quarter": "2018Q1", - "n_obs": 53924, - "brier": 0.15980530336223178, - "ece": 0.01789519036976101 - }, - { - "issue_quarter": "2018Q2", - "n_obs": 56915, - "brier": 0.1696101412796089, - "ece": 0.03514702791535815 - }, - { - "issue_quarter": "2018Q3", - "n_obs": 47168, - "brier": 0.15966140401978957, - "ece": 0.014219782776526387 - }, - { - "issue_quarter": "2018Q4", - "n_obs": 39171, - "brier": 0.15998317341352006, - "ece": 0.012831631173337394 - }, - { - "issue_quarter": "2019Q1", - "n_obs": 27508, - "brier": 0.15355504188944324, - "ece": 0.008701116060326712 - }, - { - "issue_quarter": "2019Q2", - "n_obs": 22737, - "brier": 0.1480348392667114, - "ece": 0.011107598571590603 - }, - { - "issue_quarter": "2019Q3", - "n_obs": 16603, - "brier": 0.12292350301111363, - "ece": 0.06162276831062438 - }, - { - "issue_quarter": "2019Q4", - "n_obs": 8557, - "brier": 0.0987770092004663, - "ece": 0.12603520307748503 - }, - { - "issue_quarter": "2020Q1", - "n_obs": 2977, - "brier": 0.0629698193954347, - "ece": 0.18908696961992824 - }, - { - "issue_quarter": "2020Q2", - "n_obs": 824, - "brier": 0.04644638147749954, - "ece": 0.14535032085322505 - }, - { - "issue_quarter": "2020Q3", - "n_obs": 485, - "brier": 0.03417385525796483, - "ece": 0.15326379812230886 - } - ] - }, - "selected_calibrator": "venn_abers", - "calibrators": { - "platt": { - "auc": 0.712893562122109, - "brier": 0.15460779745505918, - "ece": 0.008388765396486227 - }, - "isotonic": { - "auc": 0.7127913494994733, - "brier": 0.15453854630270586, - "ece": 0.006247440054484605 - }, - "venn_abers": { - "auc": 0.7128133359856964, - "brier": 0.15453739375452677, - "ece": 0.0061293170581817056 - } - } - } - }, - "conformal": { - "policy_status": { - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9293384235866059, - "coverage_95": 0.9623395902033092, - "avg_width_90": 0.764155715329014, - "min_group_coverage_90": 0.9003509420022164, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.193742537952152, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.1305594074149068, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2919.8495778179786, - "p_value": 0.0, - "reject": true, - "n_violations": 19564, - "n_total": 276869, - "violation_rate": 0.07066157641339406, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 966.7782855685655, - "p_value": 0.0, - "reject": true, - "n_violations": 10427, - "n_total": 276869, - "violation_rate": 0.03766040979669085, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2919.8495778179786, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 2.290243347553769, - "p_ind": 0.13018950970882026, - "reject_ind": false, - "lr_cc": 2922.1398211655323, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239175, - "n01": 18129, - "n10": 18129, - "n11": 1435 - } - }, - "christoffersen_95": { - "lr_uc": 966.7782855685655, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.07731377833988518, - "p_ind": 0.7809712495852495, - "reject_ind": false, - "lr_cc": 966.8555993469054, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 256412, - "n01": 10029, - "n10": 10029, - "n11": 398 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07066157641339406, - "violation_rate_95": 0.03766040979669085, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.13018950970882026, - "christoffersen_independence_pvalue_95": 0.7809712495852495 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.029338423586605877, - "coverage_deviation_95": 0.012339590203309214, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "configs/conformal_policy.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T14:39:38.072894+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - }, - "method_registry": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-03-30T16:09:15.677245+00:00", - "run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "project_scope": "credit_risk_tabular", - "decision_principles": [ - "roi_high", - "baseline_compatibility", - "reproducibility", - "release_stable_over_colab_notebooks" - ], - "libraries": { - "mapie": { - "version_family": "1.3.x", - "role": [ - "primary_conformal_intervals", - "classification_sets", - "time_series_adaptive_benchmark" - ], - "status": "adopted", - "notes": "Primary conformal library for PD intervals, LGD/EAD candidates, and ACI/EnbPI benchmarks." - }, - "crepes": { - "version_family": "0.9.x", - "role": [ - "p_values", - "predictive_systems", - "research_only" - ], - "status": "research_only", - "notes": "Kept for p-values/predictive systems. Not used as a Venn-Abers substitute." - }, - "venn_abers": { - "version_family": "1.5.x", - "role": [ - "probability_calibration" - ], - "status": "adopted", - "notes": "Canonical implementation for Venn-Abers calibration." - }, - "nonconformist": { - "status": "rejected", - "reason": "legacy_unmaintained_api" - }, - "fortuna": { - "status": "rejected_for_canonical_use", - "reason": "archived_project_and_extra_dependency_surface" - }, - "neuralprophet": { - "status": "excluded_from_canonical_stack", - "reason": "beta_ts_stack_outside_current_tabular_credit_scope" - } - }, - "methods": { - "pd_core": { - "calibration_default": "venn_abers", - "calibration_candidates": [ - "platt", - "isotonic", - "venn_abers" - ], - "binary_classification_sets": { - "status": "research_sidecar", - "method": "lac", - "artifact": "models/pd_set_prediction_status.json" - } - }, - "pd_conformal": { - "operational_family": "mondrian_group_conditional", - "implemented_variants": [ - "global_split", - "mondrian_unscaled", - "mondrian_scaled", - "mondrian_selected_cfg", - "cross_conformal_score_space" - ], - "strict_diagnostics": [ - "kupiec", - "christoffersen" - ], - "selector_artifact": "models/conformal_variant_selection_status.json" - }, - "lgd_ead": { - "current_operational_variants": [ - "two_stage_split", - "direct_split", - "direct_cqr", - "direct_adaptive_grade_temporal" - ], - "research_extensions": [ - "mapie_cqr_short_benchmark", - "jackknife_after_bootstrap_short_benchmark" - ] - }, - "time_series": { - "official_baseline": "statsforecast_native_intervals", - "research_shortlist": [ - "enbpi", - "aci", - "online_conformal" - ], - "status_artifact": "models/time_series_status.json" - }, - "multiclass_extension": { - "status": "backlog_p2", - "candidate_targets": [ - "loan_status_multiclass", - "stage_migration", - "delinquency_bucket" - ] - } - } - }, - "variant_selection": { - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-07T14:39:29.493223+00:00", - "artifact_namespace": "", - "calibrator_override_path": "", - "selected_variant": "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "selection_rank": 1, - "promotion_pass": true, - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time" - ], - "strict_diagnostics_role": "Kupiec/Christoffersen remain strict diagnostics in validate_conformal_policy.py; they are not the primary selector here.", - "variants_tested": [ - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "global_split", - "cross_conformal_score_space", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "mondrian_selected_cfg" - ], - "report_path": "data/processed/conformal_variant_selection_report.parquet", - "summary_path": "data/processed/conformal_variant_benchmark.parquet", - "temporal_diagnostics_path": "data/processed/conformal_temporal_diagnostics.parquet", - "local_diagnostics_path": "data/processed/conformal_local_diagnostics.parquet", - "selected_metrics": { - "coverage": 0.9151837150421318, - "coverage_gap": 0.015183715042131785, - "avg_width": 0.7799199594252145, - "min_group_coverage": 0.8914942528735632, - "winkler_90": 1.1171291360673112, - "stability_over_time": 0.039565474559288674 - }, - "search_space": { - "partition_candidates": [ - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian" - ], - "partition_probability_sources": [ - "calibrated" - ], - "n_score_bins_candidates": [ - 10 - ], - "fallback_modes": [ - "global_only" - ], - "score_scale_families": [ - "bernoulli_sqrt" - ], - "min_group_sizes": [ - 100 - ], - "calibration_size_fractions": [ - 0.25, - 0.5, - 0.75, - 1.0 - ] - }, - "top_variants": [ - { - "variant": "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9151837150421318, - "coverage_gap": 0.015183715042131785, - "avg_width": 0.7799199594252145, - "median_width": 0.9942782944989191, - "winkler_90": 1.1171291360673112, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8914942528735632, - "max_group_coverage": 0.9532581996821269, - "std_group_coverage": 0.01745020926626937, - "min_monthly_coverage": 0.9028579890419073, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.039565474559288674, - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 1 - }, - { - "variant": "global_split", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.8987318912554313, - "coverage_gap": 0.00126810874456873, - "avg_width": 0.9188873625898447, - "median_width": 0.9336885786320458, - "winkler_90": 1.09175138019133, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8515356322449575, - "max_group_coverage": 0.9832466782206817, - "std_group_coverage": 0.047074916949785994, - "min_monthly_coverage": 0.8827681524260822, - "last_monthly_coverage": 1.0, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03740927094307107, - "partition": NaN, - "partition_probability_source": NaN, - "n_score_bins": NaN, - "fallback_mode": NaN, - "scaled_scores": NaN, - "score_scale_family": NaN, - "min_group_size": NaN, - "selected_alpha_used": NaN, - "fallback_groups_n": NaN, - "calibration_fraction": NaN, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 2 - }, - { - "variant": "cross_conformal_score_space", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9066, - "coverage_gap": 0.006599999999999939, - "avg_width": 0.9264959581932166, - "median_width": 0.9393232285128859, - "winkler_90": 1.0761021403896558, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8591549295774648, - "max_group_coverage": 1.0, - "std_group_coverage": 0.04756372971496412, - "min_monthly_coverage": 0.8623188405797102, - "last_monthly_coverage": 1.0, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.04566317874614079, - "partition": NaN, - "partition_probability_source": NaN, - "n_score_bins": NaN, - "fallback_mode": NaN, - "scaled_scores": NaN, - "score_scale_family": NaN, - "min_group_size": NaN, - "selected_alpha_used": NaN, - "fallback_groups_n": NaN, - "calibration_fraction": NaN, - "implementation_note": "Cross conformal executed on calibrated score space with a lightweight linear regressor.", - "evaluation_sample_n_cal": 5000.0, - "evaluation_sample_n_test": 5000.0, - "promotion_pass": false, - "selection_rank": 3 - }, - { - "variant": "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.891920005489961, - "coverage_gap": 0.008079994510039046, - "avg_width": 0.7309907182326675, - "median_width": 0.9127438586534876, - "winkler_90": 1.256966978198614, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8534978173177875, - "max_group_coverage": 0.9116287223086531, - "std_group_coverage": 0.02170023572723275, - "min_monthly_coverage": 0.87639648205372, - "last_monthly_coverage": 0.9583333333333334, - "max_monthly_gap": 0.07836538461538456, - "stability_over_time": 0.026479277810963868, - "partition": "grade", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "global_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 4 - }, - { - "variant": "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9146961198256215, - "coverage_gap": 0.014696119825621468, - "avg_width": 0.7795528428194203, - "median_width": 0.9951357285429856, - "winkler_90": 1.1176487443153287, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8636363636363636, - "max_group_coverage": 0.988009592326139, - "std_group_coverage": 0.023832037233754995, - "min_monthly_coverage": 0.9017226911496126, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03877850237253919, - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "global_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 23.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 5 - } - ] - } - }, - "fairness": { - "overall_pass": true, - "n_attributes": 6, - "n_base_attributes": 3, - "n_intersectional_attributes": 3, - "n_passed": 6, - "attributes": [ - { - "attribute": "home_ownership", - "dpd": 0.008742234380824332, - "eo_gap": 0.022086796811760956, - "dir": 0.9912481257421258, - "tpr_gap": 0.003461823899529315, - "fpr_gap": 0.022086796811760956, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "annual_inc_quartile", - "dpd": 0.008400704557627536, - "eo_gap": 0.022249134036549156, - "dir": 0.9915953879107448, - "tpr_gap": 0.003462939364403339, - "fpr_gap": 0.022249134036549156, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "verification_status", - "dpd": 0.01152366673334193, - "eo_gap": 0.024771462501377073, - "dir": 0.9884514222176747, - "tpr_gap": 0.00523918531670553, - "fpr_gap": 0.024771462501377073, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "home_ownership__x__annual_inc_quartile", - "dpd": 0.014363846487668086, - "eo_gap": 0.03637090327737813, - "dir": 0.9856351875292462, - "tpr_gap": 0.005516043679281046, - "fpr_gap": 0.03637090327737813, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "home_ownership__x__verification_status", - "dpd": 0.0343875474383778, - "eo_gap": 0.0636573811298401, - "dir": 0.9655997718523373, - "tpr_gap": 0.01650250650050855, - "fpr_gap": 0.0636573811298401, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "annual_inc_quartile__x__verification_status", - "dpd": 0.02054232397484057, - "eo_gap": 0.04371873973202878, - "dir": 0.9794546321542239, - "tpr_gap": 0.009328152306949944, - "fpr_gap": 0.04371873973202878, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - } - ], - "prediction_threshold": 0.35, - "primary_threshold": 0.35, - "prediction_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval", - "thresholds": { - "dpd": 0.1, - "eo_gap": 0.11, - "dir": 0.8 - }, - "threshold_frontier": { - "path": "data/processed/fairness_threshold_frontier.parquet", - "thresholds": [ - 0.35, - 0.4, - 0.45, - 0.5 - ], - "worst_primary_attribute": "home_ownership__x__verification_status", - "selected_threshold": 0.35, - "all_primary_pass": true - }, - "decision_policy": { - "path": "models/fairness_decision_policy.json", - "global_threshold": 0.35, - "n_overrides": 0 - }, - "policy_config": "configs/fairness_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:42:20.905091+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - }, - "portfolio": { - "selection_stage": "economic_actual_ab_v3", - "selection_universe_path": "data/processed/champion_candidate_universe.parquet", - "decision_scenario": "baseline", - "selection_outcome": "robust_selected", - "selected_policy": { - "source": "economic_actual_ab_v1", - "risk_tolerance": 0.08, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "policy_mode": "segment_relative_tail_blended_uncertainty", - "gamma": 0.05, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 0.9 - }, - "economic_metrics": { - "diff_total_return": 3080.8323454261117, - "passed_no_regression": true, - "funded_ratio": 0.948936170212766, - "total_allocated_ratio": 1.0000000000000002, - "return_per_funded_delta": 36.5139916744335 - }, - "robustness_metrics": { - "worst_case_pd_reduction_bps": 1102.9433765668173, - "price_of_robustness_pct": 0.0, - "allocation_similarity": 0.8352184652116703, - "breadth_score": 0.941511778148717 - }, - "research_alternatives": { - "promotion_first": { - "risk_tolerance": 0.12, - "policy_mode": "segment_tail_blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "robustness_aware": { - "risk_tolerance": 0.12, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "balanced_robustness": { - "risk_tolerance": 0.12, - "policy_mode": "segment_tail_blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "guardrail_robustness": { - "risk_tolerance": 0.08, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.5, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - } - }, - "schema_version": "2026-03-10.1", - "generated_at_utc": "2026-04-07T17:40:45.568249+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - }, - "survival": { - "cox_concordance_index": 0.6947846585338557, - "rsf_c_index_test": 0.6948291667286338, - "cox_training_time": 18.711559661023784, - "rsf_training_time": 6457.35460821999, - "n_loans": 500000, - "n_events": 91989, - "event_rate": 0.183978, - "median_time_default": 17.017082785808146, - "median_time_censored": 28.02233902759527, - "cox_features": [ - "loan_amnt", - "int_rate", - "installment", - "annual_inc", - "dti", - "loan_to_income", - "installment_burden", - "rev_utilization", - "revol_bal_to_income", - "open_acc_ratio", - "fico_score", - "credit_age_years", - "emp_length_num", - "open_acc", - "total_acc", - "revol_bal", - "pub_rec", - "delinq_severity", - "delinq_recency", - "il_ratio", - "high_util_pct", - "log_annual_inc", - "log_revol_bal", - "loan_to_income_sq", - "fico_x_dti", - "has_delinq_2yrs", - "has_pub_rec", - "has_bankruptcy", - "has_recent_inq", - "has_mortgage", - "many_recent_opens", - "recent_chargeoff", - "bc_util", - "bc_open_to_buy", - "percent_bc_gt_75", - "acc_open_past_24mths", - "tot_cur_bal", - "tot_hi_cred_lim", - "total_bal_ex_mort", - "total_bc_limit", - "total_il_high_credit_limit", - "avg_cur_bal", - "pct_tl_nvr_dlq", - "mths_since_recent_bc", - "num_accts_ever_120_pd", - "num_actv_bc_tl", - "num_actv_rev_tl", - "num_bc_sats", - "num_bc_tl", - "num_il_tl", - "num_op_rev_tl", - "num_rev_accts", - "num_rev_tl_bal_gt_0", - "num_sats", - "num_tl_30dpd", - "num_tl_90g_dpd_24m", - "num_tl_op_past_12m", - "mo_sin_old_il_acct", - "mo_sin_old_rev_tl_op", - "mo_sin_rcnt_rev_tl_op", - "mo_sin_rcnt_tl" - ], - "rsf_sample_size": 500000, - "rsf_artifact": { - "saved": false, - "size_mb": 23932.23, - "placeholder": true, - "reason": "max_size_exceeded" - }, - "dataset_scope": "full_data", - "sample_size_requested": null, - "rsf_params": { - "n_estimators": 200, - "min_samples_leaf": 5, - "max_depth": null, - "max_samples": 0.5, - "n_jobs": 12 - } - }, - "time_series": { - "schema_version": "2026-03-07.1", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "status": "warn", - "warnings": [ - "interval_champion_not_promotable", - "exogenous_available_but_disabled_by_policy" - ], - "config": { - "horizon": 12, - "freq": "MS", - "season_length": 12, - "rolling_origin": { - "min_train_periods": 72, - "step_months": 1, - "embargo_periods": 12, - "max_windows": 36 - }, - "exogenous_enabled": false - }, - "summary": { - "n_models_evaluated": 6, - "n_backtest_rows": 2592, - "point_model": "AutoARIMA", - "interval_model": "AutoARIMA", - "point_promotable": true, - "interval_promotable": false, - "recent_actual_mean_12m": 0.026911907006801378 - }, - "point_champion": { - "model": "AutoARIMA", - "promotable": true, - "reasons": [], - "mae": 0.049873139227617436, - "mase": 1.9066089897100431, - "rmsse": 1.5570058706800392, - "abs_bias": 0.048018836438593226 - }, - "interval_champion": { - "model": "AutoARIMA", - "promotable": false, - "reasons": [ - "no_model_within_coverage_gap_policy" - ], - "coverage_90": 0.8101851851851852, - "coverage_gap_90": 0.08981481481481479, - "winkler_90": 0.35975890823617446, - "wis_90": 0.07804457277384988, - "pinball_90": 0.008993972705904364, - "avg_interval_width_90": 0.17404434598567664, - "family": "statistical", - "interval_subfamily": "native_statistical" - }, - "panel_global_model": { - "available": true, - "method": "bottom_up_counts_with_global_catboost", - "n_bottom_series": 14, - "n_forecast_rows": 297 - }, - "diagnostics": { - "forecastability": { - "schema_version": "2026-03-07.1", - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "n_periods": 160, - "series_start": "2007-06-01", - "series_end": "2020-09-01", - "recent_actual_mean_12m": 0.026911907006801378, - "adf": { - "statistic": -1.9765602949487615, - "p_value": 0.2969442042501646, - "lags_used": 13 - }, - "kpss": { - "statistic": 0.25753454841249984, - "p_value": 0.1, - "lags_used": 8 - }, - "acf": { - "lag_1": 0.8349981729279056, - "lag_2": 0.7459033829126462, - "lag_3": 0.6874916772827628, - "lag_4": 0.6240369599316403, - "lag_5": 0.5635390436715605, - "lag_6": 0.5053567057440341, - "lag_7": 0.472785952224223, - "lag_8": 0.39282805197501364, - "lag_9": 0.3169792848261985, - "lag_10": 0.2531242623788613, - "lag_11": 0.24157832454873526, - "lag_12": 0.1604446770648665 - }, - "pacf": { - "lag_1": 0.8402497337639301, - "lag_2": 0.16778528544014598, - "lag_3": 0.11010702609123724, - "lag_4": 0.0069389561419676725, - "lag_5": -0.007538230205843105, - "lag_6": -0.020816650426219923, - "lag_7": 0.06304032265221003, - "lag_8": -0.1548066587007755, - "lag_9": -0.09023881373390952, - "lag_10": -0.05817758714140727, - "lag_11": 0.15329868994109364, - "lag_12": -0.2048329868372945 - }, - "stl": { - "seasonal_strength": 0.21165919268077193, - "trend_strength": 0.8078373840115486 - }, - "variance_ratio": { - "k": 12, - "value": 0.4236395667454792 - }, - "spectral_entropy": 0.5559378570303748, - "permutation_entropy": 0.9903056294119694 - }, - "residual_drift": { - "ks_statistic": 0.7731481481481481, - "p_value": 6.274375257076014e-64, - "early_mean": 0.022589284820660756, - "late_mean": -0.07240765274862622 - } - }, - "research_backlog": [ - "ACI", - "EnbPI", - "OnlineConformal" - ], - "artifacts": { - "config_path": "configs/time_series.yaml", - "time_series_full_path": "data/processed/time_series_full.parquet", - "time_series_panel_path": "data/processed/time_series_panel.parquet", - "backtest_predictions_path": "data/processed/ts_backtest_predictions.parquet", - "backtest_metrics_path": "data/processed/ts_backtest_metrics.parquet", - "forecasts_path": "data/processed/ts_forecasts.parquet", - "scenarios_path": "data/processed/ts_ifrs9_scenarios.parquet", - "diagnostics_path": "data/processed/ts_diagnostics.json", - "panel_forecasts_path": "data/processed/ts_panel_forecasts.parquet", - "status_path": "models/time_series_status.json", - "forecastability_report_path": "data/processed/ts_forecastability_report.parquet", - "forecastability_status_path": "models/time_series_forecastability_status.json", - "hierarchical_eval_path": "data/processed/ts_hierarchical_eval.parquet", - "hierarchy_status_path": "models/time_series_hierarchy_status.json", - "interval_eval_path": "data/processed/ts_interval_eval.parquet" - }, - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "forecastability_summary": { - "available": true, - "series_evaluated": 22, - "routes": { - "intermittent_counts": 14, - "exogenous_challenger": 5, - "structured_statistical": 3 - }, - "intermittency": { - "erratic": 21, - "lumpy": 1 - }, - "levels": { - "grade_term": 14, - "grade": 7, - "portfolio": 1 - } - }, - "interval_benchmark": { - "available": true, - "estimator_name": "linear", - "best_method": "enbpi", - "candidate_methods_tested": [ - "enbpi", - "aci" - ], - "results": [ - { - "method": "enbpi", - "n_eval": 24, - "coverage_90": 0.8333333333333334, - "coverage_gap_90": 0.06666666666666665, - "avg_interval_width_90": 0.08555778278379522, - "winkler_90": 0.1492063183872844, - "pinball_90": 0.0037301579596821116, - "wis_90": 0.03850617753044722, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - { - "method": "aci", - "n_eval": 24, - "coverage_90": 0.7083333333333334, - "coverage_gap_90": 0.19166666666666665, - "avg_interval_width_90": 0.09372322463850964, - "winkler_90": 0.17448460839509497, - "pinball_90": 0.004362115209877376, - "wis_90": 0.04080420389479363, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.0, - "last_rolling_coverage_6": 1.0 - } - } - ], - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - "exogenous_contract_available": true, - "exogenous_contract_version": "minimal_macro_covariates_v1", - "exogenous_active": false, - "ensemble_weights": { - "AutoARIMA": 0.2730550804361629, - "AutoETS": 0.27172184234396446, - "SARIMAX": 0.2282948755555333, - "STL_CatBoost": 0.22692820166433936 - }, - "candidate_methods_tested": [ - "AutoARIMA", - "AutoETS", - "AutoTheta", - "SARIMAX", - "STL_CatBoost", - "SeasonalNaive", - "enbpi", - "aci" - ], - "rolling_coverage_summary": { - "official_interval_model": "AutoARIMA", - "official_interval_coverage_gap_90": 0.08981481481481479, - "mapie_best_method": "enbpi", - "mapie_best_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - "rolling_coverage_by_horizon": { - "official_interval_model": { - "coverage_gap_90": 0.08981481481481479, - "avg_interval_width_90": 0.17404434598567664, - "winkler_90": 0.35975890823617446, - "wis_90": 0.07804457277384988, - "pinball_90": 0.008993972705904364 - }, - "adaptive_best_method": { - "method": "enbpi", - "coverage_gap_90": 0.06666666666666665, - "avg_interval_width_90": 0.08555778278379522, - "winkler_90": 0.1492063183872844, - "wis_90": 0.03850617753044722, - "pinball_90": 0.0037301579596821116, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - } - }, - "hierarchy_reconciliation": { - "available": false, - "reason": "hierarchy_reconciliation_disabled" - }, - "interval_policy": { - "eligible_interval_families": [ - "statistical" - ], - "max_coverage_gap": 0.03, - "max_winkler_90": null - }, - "interval_selector_reason": "interval champion selected from eligible families under the governed coverage-gap policy; adaptive and conformal statistical candidates remain diagnostic unless they satisfy the same thresholds.", - "adaptive_method_status": { - "best_method": "enbpi", - "candidate_methods_tested": [ - "enbpi", - "aci" - ], - "promotion_ready": false, - "notes": "Adaptive methods remain diagnostic until they beat the official interval policy." - }, - "final_interval_decision": { - "status": "research_only", - "official_family": "statistical", - "reason": "interval champion selected from eligible families under the governed coverage-gap policy; adaptive and conformal statistical candidates remain diagnostic unless they satisfy the same thresholds." - } - }, - "causal": { - "effect_status": { - "dataset_scope": "full_data", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:12.527380+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "treatment": "int_rate", - "treatment_unit": "percentage_points", - "identified_estimand": "linear_dml_const_marginal_effect", - "identification_strategy": "orthogonal_dml_with_dowhy_audit", - "ate": 0.008349399779009324, - "ate_ci": [ - 0.007376764960351769, - 0.009322034597666878 - ], - "ate_audit_dowhy": { - "ate": 0.01393374819424989, - "ate_ci": [ - 0.013556213125054806, - 0.01431128326344496 - ], - "identified_estimand": "Estimand type: EstimandType.NONPARAMETRIC_ATE\n\n### Estimand : 1\nEstimand name: backdoor\nEstimand expression:\n d ↪\n──────────(E[default_flag|annual_inc,purpose_woe,home_ownership_woe,fico_range ↪\nd[intᵣₐₜₑ] ↪\n\n↪ \n↪ _low,loan_amnt,grade_woe,dti])\n↪ \nEstimand assumption 1, Unconfoundedness: If U→{int_rate} and U→default_flag then P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti,U) = P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti)\n\n### Estimand : 2\nEstimand name: iv\nNo such variable(s) found!\n\n### Estimand : 3\nEstimand name: frontdoor\nNo such variable(s) found!\n\n### Estimand : 4\nEstimand name: general_adjustment\nEstimand expression:\n d ↪\n──────────(E[default_flag|annual_inc,purpose_woe,home_ownership_woe,fico_range ↪\nd[intᵣₐₜₑ] ↪\n\n↪ \n↪ _low,loan_amnt,grade_woe,dti])\n↪ \nEstimand assumption 1, Unconfoundedness: If U→{int_rate} and U→default_flag then P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti,U) = P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti)\n", - "identification_strategy": "backdoor" - }, - "cate_mean": 0.008261572107039006, - "cate_std": 0.018584600775105254, - "effect_modifiers": [ - "loan_amnt", - "annual_inc", - "dti", - "fico_range_low" - ], - "confounders": [ - "grade_woe", - "purpose_woe", - "home_ownership_woe" - ], - "ate_controls": [ - "annual_inc", - "dti", - "fico_range_low", - "grade_woe", - "home_ownership_woe", - "loan_amnt", - "purpose_woe" - ], - "refutation_summary": [ - { - "test": "placebo_treatment", - "estimated_effect": 0.01393374819424989, - "new_effect": 1.1313914378179579e-05, - "p_value": 0.8600000000000001, - "result": "Refute: Use a Placebo Treatment\nEstimated effect:0.01393374819424989\nNew effect:1.1313914378179579e-05\np value:0.8600000000000001\n" - }, - { - "test": "random_common_cause", - "estimated_effect": 0.01393374819424989, - "new_effect": 0.013933751461074069, - "p_value": 0.94, - "result": "Refute: Add a random common cause\nEstimated effect:0.01393374819424989\nNew effect:0.013933751461074069\np value:0.94\n" - }, - { - "test": "data_subset", - "estimated_effect": 0.01393374819424989, - "new_effect": 0.013939064210410735, - "p_value": 0.9199999999999999, - "result": "Refute: Use a subset of data\nEstimated effect:0.01393374819424989\nNew effect:0.013939064210410735\np value:0.9199999999999999\n" - } - ], - "n_obs": 1346311, - "n_rows_input": 1346311, - "n_rows_dropped_nonfinite": 0, - "drop_rate": 0.0, - "n_imputed_cells": 74, - "source_train_split": "data/processed/train_fe.parquet", - "source_test_split": "data/processed/test_fe.parquet", - "cate_artifact_path": "data/processed/cate_estimates.parquet", - "oot_cate_artifact_path": "data/processed/cate_estimates_oot.parquet", - "overlap_artifact_path": "data/processed/causal_overlap_diagnostics.parquet", - "overlap_pass": true, - "sensitivity_pass": false, - "identification_valid": true, - "missing_required_columns": [], - "continuous_treatment_semantics": { - "estimand": "const_marginal_effect", - "interpretation": "default_probability_delta_per_1pp_rate_change", - "policy_safe": false - }, - "policy_value_method": "local_cate_discrete_grid", - "policy_evaluation_consistent": false, - "role": "insights_only", - "promotion_eligible": false, - "promotion_state": "insights_only", - "estimator_family": "causal_forest_dml", - "ate_estimator_family": "linear_dml", - "official_method": { - "identification": "LinearDML with DoWhy audit/refutation", - "heterogeneity": "causal_forest_dml", - "policy_semantics": "research_grade_pricing_intervention" - }, - "environment_status": { - "environment": "causal_lane", - "compatible": true, - "packages": { - "dowhy": { - "installed": "0.14", - "expected": ">=0.14,<0.15", - "compatible": true - }, - "econml": { - "installed": "0.16.0", - "expected": ">=0.16,<0.17", - "compatible": true - }, - "statsmodels": { - "installed": "0.14.6", - "expected": ">=0.14,<0.15", - "compatible": true - }, - "scikit-learn": { - "installed": "1.6.1", - "expected": ">=1.0,<1.7", - "compatible": true - }, - "shap": { - "installed": "0.48.0", - "expected": ">=0.38.1,<0.49", - "compatible": true - } - } - } - }, - "policy_rule": { - "selection_reason": "best_feasible", - "selected_rule": "discount_100_only", - "selected_metrics": { - "rule_name": "discount_100_only", - "n_selected": 286409.0, - "action_rate": 0.21273613600423677, - "total_net_value": 24255084.366237763, - "total_loss_reduction": 69817578.61623776, - "total_revenue_impact": -45562494.25, - "bootstrap_mean_net": 24256557.58078567, - "bootstrap_p05_net": 24170337.916558467, - "bootstrap_p95_net": 24340805.21273278, - "min_grade_total_net": 304163.96874709026, - "worst_grade": "G", - "pass_action_rate": true, - "pass_bootstrap": true, - "pass_grade_floor": true, - "pass_all": true - }, - "constraints": { - "max_action_rate": 0.35, - "min_bootstrap_p05_net": 0.0, - "min_grade_total_net": 0.0 - }, - "source_simulation_path": "data/processed/causal_policy_simulation.parquet", - "source_effect_status_path": "models/causal_effect_status.json", - "effect_status_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "policy_semantics": "research_grade_pricing_intervention", - "policy_value_method": "local_cate_discrete_grid", - "role": "insights_only", - "promotion_eligible": false, - "promotion_state": "validated_research_policy", - "promotion_decider": "validate_causal_policy.py", - "policy_evaluation_consistent": false, - "overlap_pass": true, - "sensitivity_pass": false, - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:21.052543+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - }, - "oot_status": { - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:27.886061+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "rule_name": "discount_100_only", - "min_history_months": 12, - "n_months_evaluated": 106, - "avg_action_rate": 0.2054802605581573, - "total_net_value": 24239407.18821239, - "p05_monthly_net": 1697.3359334449603, - "worst_month": "2008-09-01 00:00:00", - "best_month": "2016-03-01 00:00:00", - "selected_rule_path": "models/causal_policy_rule.json", - "effect_status_path": "models/causal_effect_status.json", - "policy_semantics": "research_grade_pricing_intervention", - "policy_value_method": "local_cate_discrete_grid", - "promotion_state": "validated_research_policy" - }, - "cate_portfolio_status": { - "delta_rate": -1.0, - "baseline_objective": null, - "cate_adjusted_objective": null, - "baseline_n_funded": null, - "cate_adjusted_n_funded": null, - "objective_change_pct": null, - "n_candidates_available": 276869, - "n_candidates_used": 0, - "max_candidates_requested": 150000, - "dataset_scope": "blocked_by_policy_gate", - "solver_backend": "highs", - "feasible_baseline": false, - "feasible_adjusted": false, - "cate_policy_mode": "research_blocked_by_policy_gate", - "promotion_eligible": false, - "promotion_state": "research_blocked_by_policy_gate", - "fallback_applied": true, - "raw_objective_change_pct": null, - "shrunk_objective_change_pct": null, - "alignment_strategy": null, - "source_cate_artifact": null, - "n_missing_cate": null, - "cate_shrink": {}, - "constraint_binding_reason": "research_blocked_by_policy_gate", - "role": "insights_only", - "promotion_decider": "causal_policy_rule.json", - "policy_evaluation_consistent": false, - "warning": "CATE portfolio blocked because the causal policy did not pass overlap/sensitivity/policy gates.", - "source_effect_status_path": "models/causal_effect_status.json", - "source_policy_rule_path": "models/causal_policy_rule.json", - "effect_status_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "policy_rule_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:30.521517+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - } - }, - "governance": { - "overall_pass": true, - "checks": { - "pass_psi": true, - "pass_breach_ratio": true, - "pass_score_psi": true, - "pass_auc_delta": true, - "pass_brier_increase": true, - "pass_calibration_gap_delta": true, - "pass_predictive_drift": true, - "pass_fairness": true, - "pass_c2st": false, - "pass_explainability": true, - "pass_reason_code_stability": true, - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "thresholds": { - "psi_threshold": 0.25, - "ks_pvalue_min": 0.01, - "cvm_pvalue_min": 0.01, - "c2st_auc_max": 0.6, - "max_feature_breach_ratio": 0.15, - "score_psi_max": 0.15, - "auc_delta_max": 0.05, - "brier_increase_max": 0.02, - "calibration_gap_delta_max": 0.02, - "explanation_rank_overlap_top10_min": 0.6, - "explanation_shap_psi_max": 0.25, - "reason_code_stability_min": 0.55 - }, - "summary": { - "n_features": 33, - "max_psi": 0.13944722694322031, - "mean_psi": 0.024499996173148176, - "min_ks_pvalue": 0.0, - "min_cvm_pvalue": 8.80828621152574e-09, - "c2st_auc": 0.9853501044444442, - "psi_breaches": 0, - "ks_breaches": 32, - "cvm_breaches": 33, - "feature_breach_ratio": 0.0, - "distribution_warning_ratio": 0.9848484848484849, - "c2st_rows_used": 100000, - "c2st_materiality": "severe", - "c2st_effective_driver_count": 1, - "score_psi": 0.01430573726529544, - "auc_train_reference": 0.7464579629585804, - "auc_test_oot": 0.7118119107831851, - "auc_delta_train_to_test": 0.03464605217539529, - "brier_train_reference": 0.13796511693423574, - "brier_test_oot": 0.1550631955367288, - "brier_increase_train_to_test": 0.017098078602493044, - "calibration_gap_train_reference": 0.0303899887875225, - "calibration_gap_test_oot": 0.0034155097827143766, - "calibration_gap_delta": 0.0, - "train_eval_rows": 100000, - "test_eval_rows": 100000, - "n_explanation_segments": 1, - "min_rank_overlap_top10": 0.9, - "max_explanation_shap_psi": 0.06104327913317865, - "min_reason_code_stability": 1.0, - "fairness_overall_pass": true, - "fairness_primary_threshold": 0.35, - "challenger_promotable": false, - "model_shift_type": "mixed_shift", - "governance_posture": "candidate_gate" - }, - "warnings": { - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "c2st": { - "auc": 0.9853501044444442, - "materiality": "severe", - "effective_driver_count": 1, - "top_drivers": [ - { - "feature": "int_rate", - "importance_auc": 0.4170757822222219 - }, - { - "feature": "loan_to_income", - "importance_auc": 0.00718282296296265 - }, - { - "feature": "pub_rec", - "importance_auc": 0.004748391851851608 - }, - { - "feature": "has_bankruptcy", - "importance_auc": 0.003179000740740475 - }, - { - "feature": "loan_amnt", - "importance_auc": 0.0025205829629626844 - }, - { - "feature": "fico_score", - "importance_auc": 0.0018507688888886875 - }, - { - "feature": "inq_last_6mths", - "importance_auc": 0.0005079955555552887 - }, - { - "feature": "term", - "importance_auc": 0.000483162962962654 - }, - { - "feature": "credit_age_years", - "importance_auc": 0.000386952592592323 - }, - { - "feature": "revol_bal_to_income", - "importance_auc": 0.000345855555555256 - } - ] - }, - "model_shift": { - "shift_type": "mixed_shift", - "structural_shift_level": "severe", - "predictive_degradation_level": "moderate", - "governance_posture": "candidate_gate", - "c2st_materiality": "severe", - "pvalue_interpretation": "Both structural shift and predictive degradation are present. This combination deserves the strongest governance posture." - }, - "artifacts": { - "drift_monitoring_path": "data/processed/drift_monitoring.parquet", - "explanation_drift_path": "data/processed/explanation_drift.parquet", - "fairness_status_path": "models/fairness_audit_status.json", - "fairness_frontier_path": "data/processed/fairness_threshold_frontier.parquet", - "challenger_promotion_report_path": "models/challenger_promotion_report.json", - "model_shift_status_path": "models/model_shift_status.json" - }, - "top_drift_features": [ - { - "feature": "fico_score", - "train_n": 798550, - "test_n": 276869, - "psi": 0.13944722694322031, - "ks_statistic": 0.15457282396927496, - "ks_pvalue": 0.0, - "cvm_statistic": 3230.128652537591, - "cvm_pvalue": 4.865294239708717e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "int_rate", - "train_n": 798550, - "test_n": 276869, - "psi": 0.10118060476101726, - "ks_statistic": 0.09481768715722516, - "ks_pvalue": 0.0, - "cvm_statistic": 398.7076133590017, - "cvm_pvalue": 1.135480649505638e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.05347961324875766, - "ks_statistic": 0.09122311383287474, - "ks_pvalue": 0.0, - "cvm_statistic": 909.8105939357774, - "cvm_pvalue": 2.1943404671098676e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_severity", - "train_n": 798550, - "test_n": 276869, - "psi": 0.05178332227166686, - "ks_statistic": 0.0916698979201469, - "ks_pvalue": 0.0, - "cvm_statistic": 44606.0478113642, - "cvm_pvalue": 2.134860395242022e-06, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "log_revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670529338066, - "cvm_pvalue": 4.4769498996899415e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income_sq", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670714583946, - "cvm_pvalue": 4.476950465903684e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_recency", - "train_n": 798550, - "test_n": 276869, - "psi": 0.038966862981116755, - "ks_statistic": 0.08135488943770064, - "ks_pvalue": 0.0, - "cvm_statistic": 11755.072386931657, - "cvm_pvalue": 9.791550346083255e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "credit_age_years", - "train_n": 798550, - "test_n": 276869, - "psi": 0.029775277868375027, - "ks_statistic": 0.054973967117968536, - "ks_pvalue": 0.0, - "cvm_statistic": 290.1511620187666, - "cvm_pvalue": 1.0223037416601244e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - } - ], - "top_explanation_breaches": [ - { - "segment_type": "overall", - "segment": "all", - "reference_period": "2018Q1|2018Q2|2018Q3|2018Q4|2019Q1|2019Q2|2019Q3", - "comparison_period": "2019Q4|2020Q1|2020Q2|2020Q3", - "reference_n": 4763, - "comparison_n": 237, - "rank_overlap_top10": 0.9, - "avg_shap_psi_top5": 0.044564149041987314, - "max_shap_psi_top5": 0.06104327913317865, - "reason_code_match_rate": 1.0, - "pass_rank_overlap": true, - "pass_distribution_shift": true, - "pass_reason_code_stability": true, - "passed_all": true, - "feature_psi_details": "[{\"feature\": \"int_rate\", \"psi\": 0.0571152296148128}, {\"feature\": \"term\", \"psi\": 0.05398847136247015}, {\"feature\": \"fico_score\", \"psi\": 0.06104327913317865}, {\"feature\": \"home_ownership\", \"psi\": 0.03675845025796413}, {\"feature\": \"many_recent_opens\", \"psi\": 0.013915314841510859}]", - "reason_code_details": "[{\"band\": \"low\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}, {\"band\": \"near_threshold\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}]" - } - ], - "primary_threshold": 0.35, - "explainability_pass": true, - "explanation_drift_pass": true, - "reason_code_stability_pass": true, - "challenger_promotable": false, - "policy_config": "configs/mrm_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:43:53.829393+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - } -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612/conformal_reopen_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612/conformal_reopen_status.json deleted file mode 100644 index 975ac94..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612/conformal_reopen_status.json +++ /dev/null @@ -1,811 +0,0 @@ -{ - "schema_version": "2026-04-05.1", - "generated_at_utc": "2026-04-05T16:34:21.897096+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612", - "mode": "derived_resume", - "upstream_canonical_run_tag": "pd-hpo-local-2026-04-03-1325", - "pipeline_profile": "search_conformal_reopen_exhaustive", - "inner_search_winner": { - "partition": "grade", - "partition_probability_source": "calibrated", - "n_score_bins": 10, - "fallback_mode": "global_only", - "alpha_used_90": 0.1, - "alpha_used_95": 0.05, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100, - "calibration_fraction": 0.5 - }, - "inner_search_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149/conformal_reopen_inner_search.parquet", - "inner_search_aggregate_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149/conformal_reopen_inner_aggregate.parquet", - "phase1_shortlist_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612/conformal_reopen_phase1_shortlist.parquet", - "phase1_final_candidates_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612/conformal_reopen_phase1_final_candidates.parquet", - "inner_search_runs": [], - "phase1_oot_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "oot_confirmation_result": { - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9297140524941399, - "coverage_95": 0.9663884364085542, - "avg_width_90": 0.7842303100675323, - "min_group_coverage_90": 0.9189831313851271, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.110742432294828, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.0793923357931081, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2999.3495000789117, - "p_value": 0.0, - "reject": true, - "n_violations": 19460, - "n_total": 276869, - "violation_rate": 0.07028594750586017, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 1760.9205010859296, - "p_value": 0.0, - "reject": true, - "n_violations": 9306, - "n_total": 276869, - "violation_rate": 0.03361156359144577, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2999.3495000789117, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.008821488241665065, - "p_ind": 0.9251704995292243, - "reject_ind": false, - "lr_cc": 2999.3583215671533, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239319, - "n01": 18089, - "n10": 18089, - "n11": 1371 - } - }, - "christoffersen_95": { - "lr_uc": 1760.9205010859296, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.780246490277932, - "p_ind": 0.37706576490539123, - "reject_ind": false, - "lr_cc": 1761.7007475762075, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 258584, - "n01": 8978, - "n10": 8978, - "n11": 328 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07028594750586017, - "violation_rate_95": 0.03361156359144577, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.9251704995292243, - "christoffersen_independence_pvalue_95": 0.37706576490539123 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.02971405249413983, - "coverage_deviation_95": 0.016388436408554252, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_validation.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-05T16:17:44.230431+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" - }, - "sidecar_set_result": { - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:19:43.149211+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "status": "research_sidecar", - "promoted": false, - "method": "lac", - "selected_method": "lac", - "selected_partition": "score_decile_mondrian", - "selected_partition_probability_source": "calibrated", - "selected_calibrator": "VennAbersScoreCalibrator", - "requested_fallback_mode": "score_only", - "effective_fallback_mode": "global_only", - "alpha": 0.1, - "confidence_level": 0.9, - "summary": { - "n_obs": 276869.0, - "set_coverage": 0.919579295623562, - "singleton_rate": 0.5526765365569999, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - "promotion_gate": { - "coverage": 0.919579295623562, - "min_coverage": 0.85, - "grade_a_singleton_rate": 0.9509310935514174, - "min_grade_a_singleton": 0.8, - "grades_with_singleton_above_40pct": 2, - "min_grades_above_40pct": 3, - "pass": false - }, - "artifact_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_cases.parquet", - "by_slice_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_by_slice.parquet", - "calibration_size_sensitivity_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_sensitivity.parquet", - "benchmark_matrix_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_benchmark.parquet", - "slice_metrics": { - "grade": [ - { - "slice_name": "grade", - "slice_value": "A", - "n_obs": 72388, - "set_coverage": 0.9225285958998729, - "singleton_rate": 0.9509310935514174, - "ambiguity_rate": 0.04906890644858264, - "empty_set_rate": 0.0, - "default_rate": 0.08658893739293806, - "default_rate_ambiguous": 0.1858108108108108 - }, - { - "slice_name": "grade", - "slice_value": "B", - "n_obs": 75311, - "set_coverage": 0.9064545683897438, - "singleton_rate": 0.6199758335435727, - "ambiguity_rate": 0.38002416645642734, - "empty_set_rate": 0.0, - "default_rate": 0.17861932519817822, - "default_rate_ambiguous": 0.2238993710691824 - }, - { - "slice_name": "grade", - "slice_value": "C", - "n_obs": 70568, - "set_coverage": 0.9199920643917923, - "singleton_rate": 0.3359171295771454, - "ambiguity_rate": 0.6640828704228545, - "empty_set_rate": 0.0, - "default_rate": 0.2760741412538261, - "default_rate_ambiguous": 0.29492350041610654 - }, - { - "slice_name": "grade", - "slice_value": "D", - "n_obs": 45637, - "set_coverage": 0.9323575169270548, - "singleton_rate": 0.2405723426167364, - "ambiguity_rate": 0.7594276573832636, - "empty_set_rate": 0.0, - "default_rate": 0.34745929837631745, - "default_rate_ambiguous": 0.36404293381037567 - }, - { - "slice_name": "grade", - "slice_value": "E", - "n_obs": 10828, - "set_coverage": 0.9294421869227928, - "singleton_rate": 0.2208164019209457, - "ambiguity_rate": 0.7791835980790544, - "empty_set_rate": 0.0, - "default_rate": 0.4314739564093092, - "default_rate_ambiguous": 0.4372407253763186 - }, - { - "slice_name": "grade", - "slice_value": "F", - "n_obs": 1731, - "set_coverage": 0.9456961294049683, - "singleton_rate": 0.19410745233968804, - "ambiguity_rate": 0.8058925476603119, - "empty_set_rate": 0.0, - "default_rate": 0.5210860774119006, - "default_rate_ambiguous": 0.4953405017921147 - }, - { - "slice_name": "grade", - "slice_value": "G", - "n_obs": 406, - "set_coverage": 0.9458128078817734, - "singleton_rate": 0.19950738916256158, - "ambiguity_rate": 0.8004926108374384, - "empty_set_rate": 0.0, - "default_rate": 0.5344827586206896, - "default_rate_ambiguous": 0.5169230769230769 - } - ], - "term": [ - { - "slice_name": "term", - "slice_value": "36", - "n_obs": 204234, - "set_coverage": 0.9116601545286289, - "singleton_rate": 0.6512089074297129, - "ambiguity_rate": 0.348791092570287, - "empty_set_rate": 0.0, - "default_rate": 0.18582116591752598, - "default_rate_ambiguous": 0.27945532392784445 - }, - { - "slice_name": "term", - "slice_value": "60", - "n_obs": 72635, - "set_coverage": 0.9418462173883114, - "singleton_rate": 0.275624698836649, - "ambiguity_rate": 0.724375301163351, - "empty_set_rate": 0.0, - "default_rate": 0.31526123769532594, - "default_rate_ambiguous": 0.3449016440178656 - } - ], - "issue_quarter": [ - { - "slice_name": "issue_quarter", - "slice_value": "2018Q1", - "n_obs": 53924, - "set_coverage": 0.9131370076403827, - "singleton_rate": 0.5601216526963875, - "ambiguity_rate": 0.43987834730361247, - "empty_set_rate": 0.0, - "default_rate": 0.2329945849714413, - "default_rate_ambiguous": 0.326602023608769 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q2", - "n_obs": 56915, - "set_coverage": 0.9075463410348765, - "singleton_rate": 0.547324958271106, - "ambiguity_rate": 0.45267504172889395, - "empty_set_rate": 0.0, - "default_rate": 0.2556268119125011, - "default_rate_ambiguous": 0.35398230088495575 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q3", - "n_obs": 47168, - "set_coverage": 0.9174864314789688, - "singleton_rate": 0.5468962008141113, - "ambiguity_rate": 0.4531037991858887, - "empty_set_rate": 0.0, - "default_rate": 0.23289094301221167, - "default_rate_ambiguous": 0.3256597417181359 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q4", - "n_obs": 39171, - "set_coverage": 0.9163411707640857, - "singleton_rate": 0.5457864236297261, - "ambiguity_rate": 0.45421357637027393, - "empty_set_rate": 0.0, - "default_rate": 0.23305506624798958, - "default_rate_ambiguous": 0.32525854316546765 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q1", - "n_obs": 27508, - "set_coverage": 0.9213683292133198, - "singleton_rate": 0.5472589792060492, - "ambiguity_rate": 0.45274102079395084, - "empty_set_rate": 0.0, - "default_rate": 0.21241093500072705, - "default_rate_ambiguous": 0.2932391199614582 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q2", - "n_obs": 22737, - "set_coverage": 0.9247482077670757, - "singleton_rate": 0.5578572371025201, - "ambiguity_rate": 0.4421427628974799, - "empty_set_rate": 0.0, - "default_rate": 0.20191757927606985, - "default_rate_ambiguous": 0.28489008256241916 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q3", - "n_obs": 16603, - "set_coverage": 0.9434439559115823, - "singleton_rate": 0.5670059627778112, - "ambiguity_rate": 0.43299403722218877, - "empty_set_rate": 0.0, - "default_rate": 0.1451544901523821, - "default_rate_ambiguous": 0.20503547085825566 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q4", - "n_obs": 8557, - "set_coverage": 0.9631880331892019, - "singleton_rate": 0.5553348136028982, - "ambiguity_rate": 0.4446651863971018, - "empty_set_rate": 0.0, - "default_rate": 0.08425850181138249, - "default_rate_ambiguous": 0.1085413929040736 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q1", - "n_obs": 2977, - "set_coverage": 0.9939536446086664, - "singleton_rate": 0.5646624118239839, - "ambiguity_rate": 0.43533758817601614, - "empty_set_rate": 0.0, - "default_rate": 0.0141081625797783, - "default_rate_ambiguous": 0.019290123456790122 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q2", - "n_obs": 824, - "set_coverage": 0.9951456310679612, - "singleton_rate": 0.6747572815533981, - "ambiguity_rate": 0.32524271844660196, - "empty_set_rate": 0.0, - "default_rate": 0.01820388349514563, - "default_rate_ambiguous": 0.04477611940298507 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q3", - "n_obs": 485, - "set_coverage": 0.9979381443298969, - "singleton_rate": 0.7175257731958763, - "ambiguity_rate": 0.2824742268041237, - "empty_set_rate": 0.0, - "default_rate": 0.002061855670103093, - "default_rate_ambiguous": 0.0 - } - ] - }, - "benchmark_matrix": [ - { - "method": "lac", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.919579295623562, - "singleton_rate": 0.5526765365569999, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9185715988427741, - "singleton_rate": 0.5558115932083404, - "ambiguity_rate": 0.4441884067916596, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.30634564407799514, - "default_rate_singleton_positive": 0.6925427872860636, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8922522926004717, - "singleton_rate": 0.522080839675081, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.030595696881918887, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8916635665242407, - "singleton_rate": 0.5251689427129798, - "ambiguity_rate": 0.44456764751561206, - "empty_set_rate": 0.03026340977140814, - "default_rate_ambiguous": 0.3063036713869052, - "default_rate_singleton_positive": 0.6925427872860636, - "default_rate_overall": 0.21977902907151034 - } - ], - "decision_use_case": { - "probability_first": true, - "set_first": false, - "recommended_guardrail": "selective_ambiguity_defer" - }, - "promotion_rationale": "Binary conformal sets selected via lac + score_decile_mondrian with set coverage 92.0% and ambiguity 44.7%.", - "promotion_note": "Binary set prediction remains a sidecar triage/abstention signal; it does not replace the interval-first conformal stack." - }, - "promotion_decision": "promotable_for_followup", - "policy_review_needed": false, - "final_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "phase2": null, - "resume_meta": { - "source_seed_design": { - "partition": "grade", - "partition_probability_source": "calibrated", - "n_score_bins": 10, - "fallback_mode": "global_only", - "alpha_used_90": 0.1, - "alpha_used_95": 0.05, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100, - "calibration_fraction": 0.5 - }, - "source_local_winner_included": true, - "source_local_winner_design": { - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5, - "fallback_mode": "score_only", - "alpha_used_90": 0.1, - "alpha_used_95": 0.05, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100, - "calibration_fraction": 1.0 - }, - "source_aggregate_unique_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149/conformal_reopen_inner_aggregate.parquet" - } -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_status.json deleted file mode 100644 index 3203169..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_status.json +++ /dev/null @@ -1,359 +0,0 @@ -{ - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9297140524941399, - "coverage_95": 0.9663884364085542, - "avg_width_90": 0.7842303100675323, - "min_group_coverage_90": 0.9189831313851271, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.110742432294828, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.0793923357931081, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2999.3495000789117, - "p_value": 0.0, - "reject": true, - "n_violations": 19460, - "n_total": 276869, - "violation_rate": 0.07028594750586017, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 1760.9205010859296, - "p_value": 0.0, - "reject": true, - "n_violations": 9306, - "n_total": 276869, - "violation_rate": 0.03361156359144577, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2999.3495000789117, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.008821488241665065, - "p_ind": 0.9251704995292243, - "reject_ind": false, - "lr_cc": 2999.3583215671533, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239319, - "n01": 18089, - "n10": 18089, - "n11": 1371 - } - }, - "christoffersen_95": { - "lr_uc": 1760.9205010859296, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.780246490277932, - "p_ind": 0.37706576490539123, - "reject_ind": false, - "lr_cc": 1761.7007475762075, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 258584, - "n01": 8978, - "n10": 8978, - "n11": 328 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07028594750586017, - "violation_rate_95": 0.03361156359144577, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.9251704995292243, - "christoffersen_independence_pvalue_95": 0.37706576490539123 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.02971405249413983, - "coverage_deviation_95": 0.016388436408554252, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_validation.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-05T16:17:44.230431+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_validation.yaml b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_validation.yaml deleted file mode 100644 index 0750245..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_validation.yaml +++ /dev/null @@ -1,34 +0,0 @@ -policy: - target_coverage_90_min: 0.9 - target_coverage_95_min: 0.95 - min_group_coverage_90_min: 0.88 - max_avg_width_90: 0.8 - max_critical_alerts: 0 - max_total_alerts: 5 - max_warning_alerts: 5 - max_winkler_90: 1.2 - enable_compensated_winkler_90: true - compensated_winkler_90_max: 1.22 - compensated_min_coverage_90: 0.92 - compensated_min_group_coverage_90: 0.885 - compensated_max_avg_width_90: 0.8 - max_winkler_95: 1.5 - min_kupiec_pvalue_90: 0.01 - min_kupiec_pvalue_95: 0.01 - min_christoffersen_pvalue_90: 0.01 - min_christoffersen_pvalue_95: 0.01 - statistical_tests_role: diagnostic_informational - allow_methodological_justification: true - max_coverage_deviation_for_statistical_warning_90: 0.03 - max_coverage_deviation_for_statistical_warning_95: 0.03 - min_christoffersen_independence_pvalue_90: 0.01 - min_christoffersen_independence_pvalue_95: 0.01 -artifacts: - conformal_results_path: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_results_mondrian.pkl - group_metrics_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_group_metrics_mondrian.parquet - backtest_monthly_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_backtest_monthly.parquet - backtest_alerts_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_backtest_alerts.parquet - intervals_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet -output: - policy_status_json: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_status.json - policy_checks_parquet: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_policy_checks.parquet diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_results_mondrian.pkl.dvc b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_results_mondrian.pkl.dvc deleted file mode 100644 index 47e1b1a..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_results_mondrian.pkl.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: dc01e7297f66b44a7510e8ff5c6eaf44 - size: 3983 - isexec: true - hash: md5 - path: conformal_results_mondrian.pkl diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_variant_selection_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_variant_selection_status.json deleted file mode 100644 index edf6ec3..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_variant_selection_status.json +++ /dev/null @@ -1,435 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:17:36.292845+00:00", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "calibrator_override_path": "", - "selected_variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "selection_rank": 1, - "promotion_pass": true, - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time" - ], - "strict_diagnostics_role": "Kupiec/Christoffersen remain strict diagnostics in validate_conformal_policy.py; they are not the primary selector here.", - "variants_tested": [ - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "mondrian_selected_cfg", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "global_split", - "cross_conformal_score_space", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100" - ], - "report_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_variant_selection_report.parquet", - "summary_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_variant_benchmark.parquet", - "temporal_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_temporal_diagnostics.parquet", - "local_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_local_diagnostics.parquet", - "selected_metrics": { - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "min_group_coverage": 0.8961474675460837, - "winkler_90": 1.1149695862930542, - "stability_over_time": 0.03709437942840774 - }, - "search_space": { - "partition_candidates": [ - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian" - ], - "partition_probability_sources": [ - "calibrated", - "raw" - ], - "n_score_bins_candidates": [ - 5, - 10, - 15, - 20 - ], - "fallback_modes": [ - "grade_then_global", - "global_only" - ], - "score_scale_families": [ - "none", - "bernoulli_sqrt", - "bernoulli_sqrt_clipped_0.02", - "bernoulli_sqrt_clipped_0.05" - ], - "min_group_sizes": [ - 100 - ], - "calibration_size_fractions": [ - 0.25, - 0.5, - 0.75, - 1.0 - ] - }, - "top_variants": [ - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 1 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 2 - }, - { - "variant": "mondrian_selected_cfg", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 3 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 4 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 5 - } - ] -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_conformal_width_attribution_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_conformal_width_attribution_status.json deleted file mode 100644 index 86a08a8..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_conformal_width_attribution_status.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "selected_partition": "score_decile_mondrian", - "selected_partition_probability_source": "raw", - "selected_n_score_bins": 5, - "selected_fallback_mode": "score_only", - "selected_alpha_used_90": 0.1, - "selected_alpha_used_95": 0.05, - "selected_min_group_size": 100, - "selected_scaled_scores": true, - "selected_score_scale_family": "bernoulli_sqrt", - "group_factors_after_shrinkback": { - "score_q01": 1.05, - "score_q04": 1.02 - }, - "temporal_factors_after_shrinkback": {}, - "width_attribution_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_conformal_width_attribution.parquet", - "shrinkback_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_shrinkback_report.parquet", - "evaluation_scope": "test", - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-05T16:12:42.814139+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_status.json deleted file mode 100644 index 77a1c60..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_status.json +++ /dev/null @@ -1,397 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:19:43.149211+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "status": "research_sidecar", - "promoted": false, - "method": "lac", - "selected_method": "lac", - "selected_partition": "score_decile_mondrian", - "selected_partition_probability_source": "calibrated", - "selected_calibrator": "VennAbersScoreCalibrator", - "requested_fallback_mode": "score_only", - "effective_fallback_mode": "global_only", - "alpha": 0.1, - "confidence_level": 0.9, - "summary": { - "n_obs": 276869.0, - "set_coverage": 0.919579295623562, - "singleton_rate": 0.5526765365569999, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - "promotion_gate": { - "coverage": 0.919579295623562, - "min_coverage": 0.85, - "grade_a_singleton_rate": 0.9509310935514174, - "min_grade_a_singleton": 0.8, - "grades_with_singleton_above_40pct": 2, - "min_grades_above_40pct": 3, - "pass": false - }, - "artifact_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_cases.parquet", - "by_slice_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_by_slice.parquet", - "calibration_size_sensitivity_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_sensitivity.parquet", - "benchmark_matrix_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_set_prediction_benchmark.parquet", - "slice_metrics": { - "grade": [ - { - "slice_name": "grade", - "slice_value": "A", - "n_obs": 72388, - "set_coverage": 0.9225285958998729, - "singleton_rate": 0.9509310935514174, - "ambiguity_rate": 0.04906890644858264, - "empty_set_rate": 0.0, - "default_rate": 0.08658893739293806, - "default_rate_ambiguous": 0.1858108108108108 - }, - { - "slice_name": "grade", - "slice_value": "B", - "n_obs": 75311, - "set_coverage": 0.9064545683897438, - "singleton_rate": 0.6199758335435727, - "ambiguity_rate": 0.38002416645642734, - "empty_set_rate": 0.0, - "default_rate": 0.17861932519817822, - "default_rate_ambiguous": 0.2238993710691824 - }, - { - "slice_name": "grade", - "slice_value": "C", - "n_obs": 70568, - "set_coverage": 0.9199920643917923, - "singleton_rate": 0.3359171295771454, - "ambiguity_rate": 0.6640828704228545, - "empty_set_rate": 0.0, - "default_rate": 0.2760741412538261, - "default_rate_ambiguous": 0.29492350041610654 - }, - { - "slice_name": "grade", - "slice_value": "D", - "n_obs": 45637, - "set_coverage": 0.9323575169270548, - "singleton_rate": 0.2405723426167364, - "ambiguity_rate": 0.7594276573832636, - "empty_set_rate": 0.0, - "default_rate": 0.34745929837631745, - "default_rate_ambiguous": 0.36404293381037567 - }, - { - "slice_name": "grade", - "slice_value": "E", - "n_obs": 10828, - "set_coverage": 0.9294421869227928, - "singleton_rate": 0.2208164019209457, - "ambiguity_rate": 0.7791835980790544, - "empty_set_rate": 0.0, - "default_rate": 0.4314739564093092, - "default_rate_ambiguous": 0.4372407253763186 - }, - { - "slice_name": "grade", - "slice_value": "F", - "n_obs": 1731, - "set_coverage": 0.9456961294049683, - "singleton_rate": 0.19410745233968804, - "ambiguity_rate": 0.8058925476603119, - "empty_set_rate": 0.0, - "default_rate": 0.5210860774119006, - "default_rate_ambiguous": 0.4953405017921147 - }, - { - "slice_name": "grade", - "slice_value": "G", - "n_obs": 406, - "set_coverage": 0.9458128078817734, - "singleton_rate": 0.19950738916256158, - "ambiguity_rate": 0.8004926108374384, - "empty_set_rate": 0.0, - "default_rate": 0.5344827586206896, - "default_rate_ambiguous": 0.5169230769230769 - } - ], - "term": [ - { - "slice_name": "term", - "slice_value": "36", - "n_obs": 204234, - "set_coverage": 0.9116601545286289, - "singleton_rate": 0.6512089074297129, - "ambiguity_rate": 0.348791092570287, - "empty_set_rate": 0.0, - "default_rate": 0.18582116591752598, - "default_rate_ambiguous": 0.27945532392784445 - }, - { - "slice_name": "term", - "slice_value": "60", - "n_obs": 72635, - "set_coverage": 0.9418462173883114, - "singleton_rate": 0.275624698836649, - "ambiguity_rate": 0.724375301163351, - "empty_set_rate": 0.0, - "default_rate": 0.31526123769532594, - "default_rate_ambiguous": 0.3449016440178656 - } - ], - "issue_quarter": [ - { - "slice_name": "issue_quarter", - "slice_value": "2018Q1", - "n_obs": 53924, - "set_coverage": 0.9131370076403827, - "singleton_rate": 0.5601216526963875, - "ambiguity_rate": 0.43987834730361247, - "empty_set_rate": 0.0, - "default_rate": 0.2329945849714413, - "default_rate_ambiguous": 0.326602023608769 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q2", - "n_obs": 56915, - "set_coverage": 0.9075463410348765, - "singleton_rate": 0.547324958271106, - "ambiguity_rate": 0.45267504172889395, - "empty_set_rate": 0.0, - "default_rate": 0.2556268119125011, - "default_rate_ambiguous": 0.35398230088495575 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q3", - "n_obs": 47168, - "set_coverage": 0.9174864314789688, - "singleton_rate": 0.5468962008141113, - "ambiguity_rate": 0.4531037991858887, - "empty_set_rate": 0.0, - "default_rate": 0.23289094301221167, - "default_rate_ambiguous": 0.3256597417181359 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q4", - "n_obs": 39171, - "set_coverage": 0.9163411707640857, - "singleton_rate": 0.5457864236297261, - "ambiguity_rate": 0.45421357637027393, - "empty_set_rate": 0.0, - "default_rate": 0.23305506624798958, - "default_rate_ambiguous": 0.32525854316546765 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q1", - "n_obs": 27508, - "set_coverage": 0.9213683292133198, - "singleton_rate": 0.5472589792060492, - "ambiguity_rate": 0.45274102079395084, - "empty_set_rate": 0.0, - "default_rate": 0.21241093500072705, - "default_rate_ambiguous": 0.2932391199614582 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q2", - "n_obs": 22737, - "set_coverage": 0.9247482077670757, - "singleton_rate": 0.5578572371025201, - "ambiguity_rate": 0.4421427628974799, - "empty_set_rate": 0.0, - "default_rate": 0.20191757927606985, - "default_rate_ambiguous": 0.28489008256241916 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q3", - "n_obs": 16603, - "set_coverage": 0.9434439559115823, - "singleton_rate": 0.5670059627778112, - "ambiguity_rate": 0.43299403722218877, - "empty_set_rate": 0.0, - "default_rate": 0.1451544901523821, - "default_rate_ambiguous": 0.20503547085825566 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q4", - "n_obs": 8557, - "set_coverage": 0.9631880331892019, - "singleton_rate": 0.5553348136028982, - "ambiguity_rate": 0.4446651863971018, - "empty_set_rate": 0.0, - "default_rate": 0.08425850181138249, - "default_rate_ambiguous": 0.1085413929040736 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q1", - "n_obs": 2977, - "set_coverage": 0.9939536446086664, - "singleton_rate": 0.5646624118239839, - "ambiguity_rate": 0.43533758817601614, - "empty_set_rate": 0.0, - "default_rate": 0.0141081625797783, - "default_rate_ambiguous": 0.019290123456790122 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q2", - "n_obs": 824, - "set_coverage": 0.9951456310679612, - "singleton_rate": 0.6747572815533981, - "ambiguity_rate": 0.32524271844660196, - "empty_set_rate": 0.0, - "default_rate": 0.01820388349514563, - "default_rate_ambiguous": 0.04477611940298507 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q3", - "n_obs": 485, - "set_coverage": 0.9979381443298969, - "singleton_rate": 0.7175257731958763, - "ambiguity_rate": 0.2824742268041237, - "empty_set_rate": 0.0, - "default_rate": 0.002061855670103093, - "default_rate_ambiguous": 0.0 - } - ] - }, - "benchmark_matrix": [ - { - "method": "lac", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.919579295623562, - "singleton_rate": 0.5526765365569999, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9185715988427741, - "singleton_rate": 0.5558115932083404, - "ambiguity_rate": 0.4441884067916596, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.30634564407799514, - "default_rate_singleton_positive": 0.6925427872860636, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8922522926004717, - "singleton_rate": 0.522080839675081, - "ambiguity_rate": 0.4473234634430001, - "empty_set_rate": 0.030595696881918887, - "default_rate_ambiguous": 0.3072587807832055, - "default_rate_singleton_positive": 0.6983532934131736, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8916635665242407, - "singleton_rate": 0.5251689427129798, - "ambiguity_rate": 0.44456764751561206, - "empty_set_rate": 0.03026340977140814, - "default_rate_ambiguous": 0.3063036713869052, - "default_rate_singleton_positive": 0.6925427872860636, - "default_rate_overall": 0.21977902907151034 - } - ], - "decision_use_case": { - "probability_first": true, - "set_first": false, - "recommended_guardrail": "selective_ambiguity_defer" - }, - "promotion_rationale": "Binary conformal sets selected via lac + score_decile_mondrian with set coverage 92.0% and ambiguity 44.7%.", - "promotion_note": "Binary set prediction remains a sidecar triage/abstention signal; it does not replace the interval-first conformal stack." -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_status.json deleted file mode 100644 index a609302..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_status.json +++ /dev/null @@ -1,364 +0,0 @@ -{ - "overall_pass": false, - "strict_overall_pass": false, - "non_statistical_checks_pass": false, - "methodological_justification_pass": false, - "methodological_justification_status": "blocked_non_statistical_failures", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 7, - "checks_total": 13, - "failing_checks": [ - "min_group_coverage_90", - "winkler_90", - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [ - "min_group_coverage_90", - "winkler_90" - ], - "coverage_90": 0.9224326305942522, - "coverage_95": 0.9676561839714811, - "avg_width_90": 0.753274497961607, - "min_group_coverage_90": 0.8777244181751016, - "critical_alerts": 0, - "warning_alerts": 3, - "total_alerts": 3, - "winkler_90": 1.2047051555467814, - "winkler_90_raw_pass": false, - "winkler_90_policy_pass": false, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.10210781037295, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 1664.5264803378668, - "p_value": 0.0, - "reject": true, - "n_violations": 21476, - "n_total": 276869, - "violation_rate": 0.07756736940574785, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 2065.5975096133625, - "p_value": 0.0, - "reject": true, - "n_violations": 8955, - "n_total": 276869, - "violation_rate": 0.0323438160285189, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 1664.5264803378668, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 4.69790689938236, - "p_ind": 0.030199373958066045, - "reject_ind": true, - "lr_cc": 1669.2243872372492, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 235664, - "n01": 19728, - "n10": 19728, - "n11": 1748 - } - }, - "christoffersen_95": { - "lr_uc": 2065.5975096133625, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 1.8008314184990013, - "p_ind": 0.17961201292822215, - "reject_ind": false, - "lr_cc": 2067.3983410318615, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 259270, - "n01": 8643, - "n10": 8643, - "n11": 312 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07756736940574785, - "violation_rate_95": 0.0323438160285189, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.030199373958066045, - "christoffersen_independence_pvalue_95": 0.17961201292822215 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": false, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.022432630594252156, - "coverage_deviation_95": 0.017656183971481165, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": false, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": false, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_validation.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-05T16:24:50.648209+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_validation.yaml b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_validation.yaml deleted file mode 100644 index 551cca2..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_validation.yaml +++ /dev/null @@ -1,34 +0,0 @@ -policy: - target_coverage_90_min: 0.9 - target_coverage_95_min: 0.95 - min_group_coverage_90_min: 0.88 - max_avg_width_90: 0.8 - max_critical_alerts: 0 - max_total_alerts: 5 - max_warning_alerts: 5 - max_winkler_90: 1.2 - enable_compensated_winkler_90: true - compensated_winkler_90_max: 1.22 - compensated_min_coverage_90: 0.92 - compensated_min_group_coverage_90: 0.885 - compensated_max_avg_width_90: 0.8 - max_winkler_95: 1.5 - min_kupiec_pvalue_90: 0.01 - min_kupiec_pvalue_95: 0.01 - min_christoffersen_pvalue_90: 0.01 - min_christoffersen_pvalue_95: 0.01 - statistical_tests_role: diagnostic_informational - allow_methodological_justification: true - max_coverage_deviation_for_statistical_warning_90: 0.03 - max_coverage_deviation_for_statistical_warning_95: 0.03 - min_christoffersen_independence_pvalue_90: 0.01 - min_christoffersen_independence_pvalue_95: 0.01 -artifacts: - conformal_results_path: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_results_mondrian.pkl - group_metrics_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_group_metrics_mondrian.parquet - backtest_monthly_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_backtest_monthly.parquet - backtest_alerts_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_backtest_alerts.parquet - intervals_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_intervals_mondrian.parquet -output: - policy_status_json: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_status.json - policy_checks_parquet: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_policy_checks.parquet diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_results_mondrian.pkl.dvc b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_results_mondrian.pkl.dvc deleted file mode 100644 index d9e5edb..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_results_mondrian.pkl.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: d67ef81bc65ace86fe721692bc45f40f - size: 4155 - isexec: true - hash: md5 - path: conformal_results_mondrian.pkl diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_variant_selection_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_variant_selection_status.json deleted file mode 100644 index c5bcd14..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_variant_selection_status.json +++ /dev/null @@ -1,435 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:24:42.986741+00:00", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2", - "calibrator_override_path": "", - "selected_variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "selection_rank": 1, - "promotion_pass": true, - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time" - ], - "strict_diagnostics_role": "Kupiec/Christoffersen remain strict diagnostics in validate_conformal_policy.py; they are not the primary selector here.", - "variants_tested": [ - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "global_split", - "cross_conformal_score_space", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "mondrian_selected_cfg", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=100", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=100" - ], - "report_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_variant_selection_report.parquet", - "summary_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_variant_benchmark.parquet", - "temporal_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_temporal_diagnostics.parquet", - "local_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_local_diagnostics.parquet", - "selected_metrics": { - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "min_group_coverage": 0.8961474675460837, - "winkler_90": 1.1149695862930542, - "stability_over_time": 0.03709437942840774 - }, - "search_space": { - "partition_candidates": [ - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian" - ], - "partition_probability_sources": [ - "calibrated", - "raw" - ], - "n_score_bins_candidates": [ - 5, - 10, - 15, - 20 - ], - "fallback_modes": [ - "grade_then_global", - "global_only" - ], - "score_scale_families": [ - "none", - "bernoulli_sqrt", - "bernoulli_sqrt_clipped_0.02", - "bernoulli_sqrt_clipped_0.05" - ], - "min_group_sizes": [ - 100 - ], - "calibration_size_fractions": [ - 0.25, - 0.5, - 0.75, - 1.0 - ] - }, - "top_variants": [ - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 1 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 2 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 3 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 4 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7850360432132643, - "median_width": 0.9833675451648134, - "winkler_90": 1.1153438100824948, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.05", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 5 - } - ] -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_conformal_width_attribution_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_conformal_width_attribution_status.json deleted file mode 100644 index dab282e..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_conformal_width_attribution_status.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2", - "selected_partition": "grade", - "selected_partition_probability_source": "calibrated", - "selected_n_score_bins": 10, - "selected_fallback_mode": "global_only", - "selected_alpha_used_90": 0.1, - "selected_alpha_used_95": 0.05, - "selected_min_group_size": 100, - "selected_scaled_scores": true, - "selected_score_scale_family": "bernoulli_sqrt", - "group_factors_after_shrinkback": { - "A": 1.2, - "B": 1.05, - "C": 1.05, - "D": 1.02, - "F": 1.05, - "G": 1.05 - }, - "temporal_factors_after_shrinkback": { - "A|vintage=2017Q4": 1.2 - }, - "width_attribution_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_conformal_width_attribution.parquet", - "shrinkback_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/conformal_shrinkback_report.parquet", - "evaluation_scope": "test", - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-05T16:19:50.795487+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_status.json deleted file mode 100644 index 1111a43..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_status.json +++ /dev/null @@ -1,397 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:26:56.231881+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2", - "status": "research_sidecar", - "promoted": false, - "method": "lac", - "selected_method": "lac", - "selected_partition": "grade_x_scoreband_mondrian", - "selected_partition_probability_source": "calibrated", - "selected_calibrator": "VennAbersScoreCalibrator", - "requested_fallback_mode": "global_only", - "effective_fallback_mode": "global_only", - "alpha": 0.1, - "confidence_level": 0.9, - "summary": { - "n_obs": 276869.0, - "set_coverage": 0.920756747776024, - "singleton_rate": 0.5309659080648249, - "ambiguity_rate": 0.46903409193517515, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2928207853012067, - "default_rate_singleton_positive": 0.6155857740585774, - "default_rate_overall": 0.21977902907151034 - }, - "promotion_gate": { - "coverage": 0.920756747776024, - "min_coverage": 0.85, - "grade_a_singleton_rate": 0.8846493894015582, - "min_grade_a_singleton": 0.8, - "grades_with_singleton_above_40pct": 2, - "min_grades_above_40pct": 3, - "pass": false - }, - "artifact_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_cases.parquet", - "by_slice_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_by_slice.parquet", - "calibration_size_sensitivity_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_sensitivity.parquet", - "benchmark_matrix_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-2/pd_set_prediction_benchmark.parquet", - "slice_metrics": { - "grade": [ - { - "slice_name": "grade", - "slice_value": "A", - "n_obs": 72388, - "set_coverage": 0.9318533458584296, - "singleton_rate": 0.8846493894015582, - "ambiguity_rate": 0.11535061059844173, - "empty_set_rate": 0.0, - "default_rate": 0.08658893739293806, - "default_rate_ambiguous": 0.1598802395209581 - }, - { - "slice_name": "grade", - "slice_value": "B", - "n_obs": 75311, - "set_coverage": 0.9212731207924473, - "singleton_rate": 0.5209730318280198, - "ambiguity_rate": 0.4790269681719802, - "empty_set_rate": 0.0, - "default_rate": 0.17861932519817822, - "default_rate_ambiguous": 0.20861514580330415 - }, - { - "slice_name": "grade", - "slice_value": "C", - "n_obs": 70568, - "set_coverage": 0.9152873823829498, - "singleton_rate": 0.3715706835959642, - "ambiguity_rate": 0.6284293164040359, - "empty_set_rate": 0.0, - "default_rate": 0.2760741412538261, - "default_rate_ambiguous": 0.3036733037184026 - }, - { - "slice_name": "grade", - "slice_value": "D", - "n_obs": 45637, - "set_coverage": 0.9163398119946534, - "singleton_rate": 0.29631658522689924, - "ambiguity_rate": 0.7036834147731008, - "empty_set_rate": 0.0, - "default_rate": 0.34745929837631745, - "default_rate_ambiguous": 0.3656349255776297 - }, - { - "slice_name": "grade", - "slice_value": "E", - "n_obs": 10828, - "set_coverage": 0.8975803472478758, - "singleton_rate": 0.3158478019948282, - "ambiguity_rate": 0.6841521980051718, - "empty_set_rate": 0.0, - "default_rate": 0.4314739564093092, - "default_rate_ambiguous": 0.4287257019438445 - }, - { - "slice_name": "grade", - "slice_value": "F", - "n_obs": 1731, - "set_coverage": 0.9145002888503755, - "singleton_rate": 0.2732524552281918, - "ambiguity_rate": 0.7267475447718083, - "empty_set_rate": 0.0, - "default_rate": 0.5210860774119006, - "default_rate_ambiguous": 0.4904610492845787 - }, - { - "slice_name": "grade", - "slice_value": "G", - "n_obs": 406, - "set_coverage": 0.9384236453201971, - "singleton_rate": 0.2413793103448276, - "ambiguity_rate": 0.7586206896551724, - "empty_set_rate": 0.0, - "default_rate": 0.5344827586206896, - "default_rate_ambiguous": 0.5292207792207793 - } - ], - "term": [ - { - "slice_name": "term", - "slice_value": "36", - "n_obs": 204234, - "set_coverage": 0.9153519981981453, - "singleton_rate": 0.6220022131476639, - "ambiguity_rate": 0.37799778685233604, - "empty_set_rate": 0.0, - "default_rate": 0.18582116591752598, - "default_rate_ambiguous": 0.2675 - }, - { - "slice_name": "term", - "slice_value": "60", - "n_obs": 72635, - "set_coverage": 0.9359537413092861, - "singleton_rate": 0.2749913953328285, - "ambiguity_rate": 0.7250086046671714, - "empty_set_rate": 0.0, - "default_rate": 0.31526123769532594, - "default_rate_ambiguous": 0.32994056322515714 - } - ], - "issue_quarter": [ - { - "slice_name": "issue_quarter", - "slice_value": "2018Q1", - "n_obs": 53924, - "set_coverage": 0.915232549514131, - "singleton_rate": 0.5382575476596692, - "ambiguity_rate": 0.46174245234033084, - "empty_set_rate": 0.0, - "default_rate": 0.2329945849714413, - "default_rate_ambiguous": 0.31125748022008914 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q2", - "n_obs": 56915, - "set_coverage": 0.9099534393393657, - "singleton_rate": 0.5295967671088465, - "ambiguity_rate": 0.4704032328911535, - "empty_set_rate": 0.0, - "default_rate": 0.2556268119125011, - "default_rate_ambiguous": 0.339558510439622 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q3", - "n_obs": 47168, - "set_coverage": 0.9192672998643148, - "singleton_rate": 0.5274338534599728, - "ambiguity_rate": 0.47256614654002715, - "empty_set_rate": 0.0, - "default_rate": 0.23289094301221167, - "default_rate_ambiguous": 0.31121579183490355 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q4", - "n_obs": 39171, - "set_coverage": 0.9171325725664394, - "singleton_rate": 0.523167649536647, - "ambiguity_rate": 0.476832350463353, - "empty_set_rate": 0.0, - "default_rate": 0.23305506624798958, - "default_rate_ambiguous": 0.3077952671592248 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q1", - "n_obs": 27508, - "set_coverage": 0.9203504435073433, - "singleton_rate": 0.5225025447142649, - "ambiguity_rate": 0.47749745528573506, - "empty_set_rate": 0.0, - "default_rate": 0.21241093500072705, - "default_rate_ambiguous": 0.2764370003806624 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q2", - "n_obs": 22737, - "set_coverage": 0.9262875489290584, - "singleton_rate": 0.5270704138628667, - "ambiguity_rate": 0.4729295861371333, - "empty_set_rate": 0.0, - "default_rate": 0.20191757927606985, - "default_rate_ambiguous": 0.270529154654515 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q3", - "n_obs": 16603, - "set_coverage": 0.9440462567005963, - "singleton_rate": 0.5454435945311088, - "ambiguity_rate": 0.45455640546889114, - "empty_set_rate": 0.0, - "default_rate": 0.1451544901523821, - "default_rate_ambiguous": 0.20007950178879025 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q4", - "n_obs": 8557, - "set_coverage": 0.958396634334463, - "singleton_rate": 0.5365198083440458, - "ambiguity_rate": 0.46348019165595417, - "empty_set_rate": 0.0, - "default_rate": 0.08425850181138249, - "default_rate_ambiguous": 0.1043872919818457 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q1", - "n_obs": 2977, - "set_coverage": 0.9895868323815922, - "singleton_rate": 0.5512260665099094, - "ambiguity_rate": 0.4487739334900907, - "empty_set_rate": 0.0, - "default_rate": 0.0141081625797783, - "default_rate_ambiguous": 0.017964071856287425 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q2", - "n_obs": 824, - "set_coverage": 0.9927184466019418, - "singleton_rate": 0.6274271844660194, - "ambiguity_rate": 0.3725728155339806, - "empty_set_rate": 0.0, - "default_rate": 0.01820388349514563, - "default_rate_ambiguous": 0.035830618892508145 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q3", - "n_obs": 485, - "set_coverage": 0.9979381443298969, - "singleton_rate": 0.6350515463917525, - "ambiguity_rate": 0.3649484536082474, - "empty_set_rate": 0.0, - "default_rate": 0.002061855670103093, - "default_rate_ambiguous": 0.0 - } - ] - }, - "benchmark_matrix": [ - { - "method": "lac", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.920756747776024, - "singleton_rate": 0.5309659080648249, - "ambiguity_rate": 0.46903409193517515, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2928207853012067, - "default_rate_singleton_positive": 0.6155857740585774, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9199043591012356, - "singleton_rate": 0.5266172810968364, - "ambiguity_rate": 0.4733827189031636, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2880326555525884, - "default_rate_singleton_positive": 0.6134743053848045, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8938306563753977, - "singleton_rate": 0.5003124221202084, - "ambiguity_rate": 0.4694241681083834, - "empty_set_rate": 0.03026340977140814, - "default_rate_ambiguous": 0.2927621201978933, - "default_rate_singleton_positive": 0.6155055002619172, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8925773560781453, - "singleton_rate": 0.49602158421491754, - "ambiguity_rate": 0.4733827189031636, - "empty_set_rate": 0.030595696881918887, - "default_rate_ambiguous": 0.2880326555525884, - "default_rate_singleton_positive": 0.6134743053848045, - "default_rate_overall": 0.21977902907151034 - } - ], - "decision_use_case": { - "probability_first": true, - "set_first": false, - "recommended_guardrail": "selective_ambiguity_defer" - }, - "promotion_rationale": "Binary conformal sets selected via lac + grade_x_scoreband_mondrian with set coverage 92.1% and ambiguity 46.9%.", - "promotion_note": "Binary set prediction remains a sidecar triage/abstention signal; it does not replace the interval-first conformal stack." -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_status.json deleted file mode 100644 index 1f7b9d8..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_status.json +++ /dev/null @@ -1,364 +0,0 @@ -{ - "overall_pass": false, - "strict_overall_pass": false, - "non_statistical_checks_pass": false, - "methodological_justification_pass": false, - "methodological_justification_status": "blocked_non_statistical_failures", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 7, - "checks_total": 13, - "failing_checks": [ - "min_group_coverage_90", - "winkler_90", - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [ - "min_group_coverage_90", - "winkler_90" - ], - "coverage_90": 0.9224326305942522, - "coverage_95": 0.9676561839714811, - "avg_width_90": 0.753274497961607, - "min_group_coverage_90": 0.8777244181751016, - "critical_alerts": 0, - "warning_alerts": 3, - "total_alerts": 3, - "winkler_90": 1.2047051555467814, - "winkler_90_raw_pass": false, - "winkler_90_policy_pass": false, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.10210781037295, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 1664.5264803378668, - "p_value": 0.0, - "reject": true, - "n_violations": 21476, - "n_total": 276869, - "violation_rate": 0.07756736940574785, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 2065.5975096133625, - "p_value": 0.0, - "reject": true, - "n_violations": 8955, - "n_total": 276869, - "violation_rate": 0.0323438160285189, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 1664.5264803378668, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 4.69790689938236, - "p_ind": 0.030199373958066045, - "reject_ind": true, - "lr_cc": 1669.2243872372492, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 235664, - "n01": 19728, - "n10": 19728, - "n11": 1748 - } - }, - "christoffersen_95": { - "lr_uc": 2065.5975096133625, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 1.8008314184990013, - "p_ind": 0.17961201292822215, - "reject_ind": false, - "lr_cc": 2067.3983410318615, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 259270, - "n01": 8643, - "n10": 8643, - "n11": 312 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07756736940574785, - "violation_rate_95": 0.0323438160285189, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.030199373958066045, - "christoffersen_independence_pvalue_95": 0.17961201292822215 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": false, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.022432630594252156, - "coverage_deviation_95": 0.017656183971481165, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": false, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": false, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_validation.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-05T16:32:20.908349+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_validation.yaml b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_validation.yaml deleted file mode 100644 index 41af27d..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_validation.yaml +++ /dev/null @@ -1,34 +0,0 @@ -policy: - target_coverage_90_min: 0.9 - target_coverage_95_min: 0.95 - min_group_coverage_90_min: 0.88 - max_avg_width_90: 0.8 - max_critical_alerts: 0 - max_total_alerts: 5 - max_warning_alerts: 5 - max_winkler_90: 1.2 - enable_compensated_winkler_90: true - compensated_winkler_90_max: 1.22 - compensated_min_coverage_90: 0.92 - compensated_min_group_coverage_90: 0.885 - compensated_max_avg_width_90: 0.8 - max_winkler_95: 1.5 - min_kupiec_pvalue_90: 0.01 - min_kupiec_pvalue_95: 0.01 - min_christoffersen_pvalue_90: 0.01 - min_christoffersen_pvalue_95: 0.01 - statistical_tests_role: diagnostic_informational - allow_methodological_justification: true - max_coverage_deviation_for_statistical_warning_90: 0.03 - max_coverage_deviation_for_statistical_warning_95: 0.03 - min_christoffersen_independence_pvalue_90: 0.01 - min_christoffersen_independence_pvalue_95: 0.01 -artifacts: - conformal_results_path: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_results_mondrian.pkl - group_metrics_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_group_metrics_mondrian.parquet - backtest_monthly_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_backtest_monthly.parquet - backtest_alerts_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_backtest_alerts.parquet - intervals_path: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_intervals_mondrian.parquet -output: - policy_status_json: models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_status.json - policy_checks_parquet: data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_policy_checks.parquet diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_results_mondrian.pkl.dvc b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_results_mondrian.pkl.dvc deleted file mode 100644 index 7afc62e..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_results_mondrian.pkl.dvc +++ /dev/null @@ -1,6 +0,0 @@ -outs: -- md5: 27016ab471056b8c57b4f6764758444c - size: 4158 - isexec: true - hash: md5 - path: conformal_results_mondrian.pkl diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_variant_selection_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_variant_selection_status.json deleted file mode 100644 index 157560d..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_variant_selection_status.json +++ /dev/null @@ -1,435 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:32:13.412404+00:00", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3", - "calibrator_override_path": "", - "selected_variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "selection_rank": 1, - "promotion_pass": true, - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time" - ], - "strict_diagnostics_role": "Kupiec/Christoffersen remain strict diagnostics in validate_conformal_policy.py; they are not the primary selector here.", - "variants_tested": [ - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "global_split", - "cross_conformal_score_space", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=none::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=none::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=1000", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=raw::bins=5::fallback=global_only::scale=none::mgs=1000", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=raw::bins=10::fallback=global_only::scale=none::mgs=1000", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=raw::bins=15::fallback=global_only::scale=none::mgs=1000", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "grade::prob=raw::bins=20::fallback=global_only::scale=none::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "mondrian_selected_cfg", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=10::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=10::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "score_decile_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=raw::bins=15::fallback=global_only::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "grade_x_scoreband_mondrian::prob=calibrated::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=20::fallback=global_only::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=grade_then_global::scale=none::mgs=1000", - "score_decile_mondrian::prob=raw::bins=15::fallback=global_only::scale=none::mgs=1000" - ], - "report_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_variant_selection_report.parquet", - "summary_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_variant_benchmark.parquet", - "temporal_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_temporal_diagnostics.parquet", - "local_diagnostics_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_local_diagnostics.parquet", - "selected_metrics": { - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "min_group_coverage": 0.8961474675460837, - "winkler_90": 1.1149695862930542, - "stability_over_time": 0.03709437942840774 - }, - "search_space": { - "partition_candidates": [ - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian" - ], - "partition_probability_sources": [ - "calibrated", - "raw" - ], - "n_score_bins_candidates": [ - 5, - 10, - 15, - 20 - ], - "fallback_modes": [ - "grade_then_global", - "global_only" - ], - "score_scale_families": [ - "none", - "bernoulli_sqrt", - "bernoulli_sqrt_clipped_0.02", - "bernoulli_sqrt_clipped_0.05" - ], - "min_group_sizes": [ - 1000 - ], - "calibration_size_fractions": [ - 0.25, - 0.5, - 0.75, - 1.0 - ] - }, - "top_variants": [ - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt::mgs=1000", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 1000.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 1 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt::mgs=1000", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.783799494309143, - "median_width": 0.9833675451648134, - "winkler_90": 1.1149695862930542, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 1000.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 2 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 1000.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 3 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=global_only::scale=bernoulli_sqrt_clipped_0.02::mgs=1000", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7839095531622903, - "median_width": 0.9833675451648134, - "winkler_90": 1.115030740746861, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.02", - "min_group_size": 1000.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 4 - }, - { - "variant": "score_decile_mondrian::prob=raw::bins=5::fallback=grade_then_global::scale=bernoulli_sqrt_clipped_0.05::mgs=1000", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9115104977444206, - "coverage_gap": 0.01151049774442059, - "avg_width": 0.7850360432132643, - "median_width": 0.9833675451648134, - "winkler_90": 1.1153438100824948, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8961474675460837, - "max_group_coverage": 0.9263745604883892, - "std_group_coverage": 0.01052363775798202, - "min_monthly_coverage": 0.895898119354361, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03709437942840774, - "partition": "score_decile_mondrian", - "partition_probability_source": "raw", - "n_score_bins": 5.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt_clipped_0.05", - "min_group_size": 1000.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 5 - } - ] -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_conformal_width_attribution_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_conformal_width_attribution_status.json deleted file mode 100644 index 5c43076..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_conformal_width_attribution_status.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3", - "selected_partition": "grade", - "selected_partition_probability_source": "calibrated", - "selected_n_score_bins": 10, - "selected_fallback_mode": "global_only", - "selected_alpha_used_90": 0.1, - "selected_alpha_used_95": 0.05, - "selected_min_group_size": 1000, - "selected_scaled_scores": true, - "selected_score_scale_family": "bernoulli_sqrt", - "group_factors_after_shrinkback": { - "A": 1.2, - "B": 1.05, - "C": 1.05, - "D": 1.02, - "F": 1.05, - "G": 1.05 - }, - "temporal_factors_after_shrinkback": { - "A|vintage=2017Q4": 1.2 - }, - "width_attribution_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_conformal_width_attribution.parquet", - "shrinkback_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/conformal_shrinkback_report.parquet", - "evaluation_scope": "test", - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-05T16:27:03.477482+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612" -} diff --git a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_status.json b/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_status.json deleted file mode 100644 index f5e0a77..0000000 --- a/models/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_status.json +++ /dev/null @@ -1,397 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-05T16:34:21.292387+00:00", - "run_tag": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612", - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3", - "status": "research_sidecar", - "promoted": false, - "method": "lac", - "selected_method": "lac", - "selected_partition": "score_decile_mondrian", - "selected_partition_probability_source": "calibrated", - "selected_calibrator": "VennAbersScoreCalibrator", - "requested_fallback_mode": "global_only", - "effective_fallback_mode": "global_only", - "alpha": 0.1, - "confidence_level": 0.9, - "summary": { - "n_obs": 276869.0, - "set_coverage": 0.9199043591012356, - "singleton_rate": 0.5266172810968364, - "ambiguity_rate": 0.4733827189031636, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2880326555525884, - "default_rate_singleton_positive": 0.6134743053848045, - "default_rate_overall": 0.21977902907151034 - }, - "promotion_gate": { - "coverage": 0.9199043591012356, - "min_coverage": 0.85, - "grade_a_singleton_rate": 0.8981322871194121, - "min_grade_a_singleton": 0.8, - "grades_with_singleton_above_40pct": 2, - "min_grades_above_40pct": 3, - "pass": false - }, - "artifact_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_cases.parquet", - "by_slice_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_by_slice.parquet", - "calibration_size_sensitivity_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_sensitivity.parquet", - "benchmark_matrix_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-3/pd_set_prediction_benchmark.parquet", - "slice_metrics": { - "grade": [ - { - "slice_name": "grade", - "slice_value": "A", - "n_obs": 72388, - "set_coverage": 0.9292700447587998, - "singleton_rate": 0.8981322871194121, - "ambiguity_rate": 0.10186771288058795, - "empty_set_rate": 0.0, - "default_rate": 0.08658893739293806, - "default_rate_ambiguous": 0.155682126390019 - }, - { - "slice_name": "grade", - "slice_value": "B", - "n_obs": 75311, - "set_coverage": 0.9149924977758893, - "singleton_rate": 0.5435726520694188, - "ambiguity_rate": 0.4564273479305812, - "empty_set_rate": 0.0, - "default_rate": 0.17861932519817822, - "default_rate_ambiguous": 0.20518415081165997 - }, - { - "slice_name": "grade", - "slice_value": "C", - "n_obs": 70568, - "set_coverage": 0.9174129917242942, - "singleton_rate": 0.340097494615123, - "ambiguity_rate": 0.659902505384877, - "empty_set_rate": 0.0, - "default_rate": 0.2760741412538261, - "default_rate_ambiguous": 0.29241109774952756 - }, - { - "slice_name": "grade", - "slice_value": "D", - "n_obs": 45637, - "set_coverage": 0.9230010736902075, - "singleton_rate": 0.2627035081184127, - "ambiguity_rate": 0.7372964918815873, - "empty_set_rate": 0.0, - "default_rate": 0.34745929837631745, - "default_rate_ambiguous": 0.35800047551117453 - }, - { - "slice_name": "grade", - "slice_value": "E", - "n_obs": 10828, - "set_coverage": 0.902936830439601, - "singleton_rate": 0.287125969708164, - "ambiguity_rate": 0.7128740302918359, - "empty_set_rate": 0.0, - "default_rate": 0.4314739564093092, - "default_rate_ambiguous": 0.41896618732996505 - }, - { - "slice_name": "grade", - "slice_value": "F", - "n_obs": 1731, - "set_coverage": 0.8752166377816292, - "singleton_rate": 0.36164067013287116, - "ambiguity_rate": 0.6383593298671288, - "empty_set_rate": 0.0, - "default_rate": 0.5210860774119006, - "default_rate_ambiguous": 0.465158371040724 - }, - { - "slice_name": "grade", - "slice_value": "G", - "n_obs": 406, - "set_coverage": 0.8891625615763546, - "singleton_rate": 0.31773399014778325, - "ambiguity_rate": 0.6822660098522167, - "empty_set_rate": 0.0, - "default_rate": 0.5344827586206896, - "default_rate_ambiguous": 0.5018050541516246 - } - ], - "term": [ - { - "slice_name": "term", - "slice_value": "36", - "n_obs": 204234, - "set_coverage": 0.9166691148388613, - "singleton_rate": 0.6079643937836011, - "ambiguity_rate": 0.3920356062163988, - "empty_set_rate": 0.0, - "default_rate": 0.18582116591752598, - "default_rate_ambiguous": 0.2612686874742403 - }, - { - "slice_name": "term", - "slice_value": "60", - "n_obs": 72635, - "set_coverage": 0.9290011702347353, - "singleton_rate": 0.29788669374268606, - "ambiguity_rate": 0.7021133062573139, - "empty_set_rate": 0.0, - "default_rate": 0.31526123769532594, - "default_rate_ambiguous": 0.3300521589081925 - } - ], - "issue_quarter": [ - { - "slice_name": "issue_quarter", - "slice_value": "2018Q1", - "n_obs": 53924, - "set_coverage": 0.9146391217268749, - "singleton_rate": 0.5318225650916104, - "ambiguity_rate": 0.4681774349083896, - "empty_set_rate": 0.0, - "default_rate": 0.2329945849714413, - "default_rate_ambiguous": 0.3054345242810742 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q2", - "n_obs": 56915, - "set_coverage": 0.9087586752174295, - "singleton_rate": 0.5249407010454186, - "ambiguity_rate": 0.4750592989545814, - "empty_set_rate": 0.0, - "default_rate": 0.2556268119125011, - "default_rate_ambiguous": 0.3334936015977513 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q3", - "n_obs": 47168, - "set_coverage": 0.9184828697421981, - "singleton_rate": 0.5235117028493894, - "ambiguity_rate": 0.4764882971506106, - "empty_set_rate": 0.0, - "default_rate": 0.23289094301221167, - "default_rate_ambiguous": 0.30589543937708563 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2018Q4", - "n_obs": 39171, - "set_coverage": 0.9163922289448827, - "singleton_rate": 0.5188021750785019, - "ambiguity_rate": 0.48119782492149804, - "empty_set_rate": 0.0, - "default_rate": 0.23305506624798958, - "default_rate_ambiguous": 0.3034113215555202 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q1", - "n_obs": 27508, - "set_coverage": 0.9188236149483786, - "singleton_rate": 0.5202486549367457, - "ambiguity_rate": 0.47975134506325434, - "empty_set_rate": 0.0, - "default_rate": 0.21241093500072705, - "default_rate_ambiguous": 0.27195574751837537 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q2", - "n_obs": 22737, - "set_coverage": 0.9255398689360953, - "singleton_rate": 0.5249593174121476, - "ambiguity_rate": 0.4750406825878524, - "empty_set_rate": 0.0, - "default_rate": 0.20191757927606985, - "default_rate_ambiguous": 0.26775298583464496 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q3", - "n_obs": 16603, - "set_coverage": 0.9433234957537794, - "singleton_rate": 0.5417695597181232, - "ambiguity_rate": 0.45823044028187676, - "empty_set_rate": 0.0, - "default_rate": 0.1451544901523821, - "default_rate_ambiguous": 0.19689800210304942 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2019Q4", - "n_obs": 8557, - "set_coverage": 0.9586303611078649, - "singleton_rate": 0.5281056445015777, - "ambiguity_rate": 0.47189435549842235, - "empty_set_rate": 0.0, - "default_rate": 0.08425850181138249, - "default_rate_ambiguous": 0.1030212976721149 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q1", - "n_obs": 2977, - "set_coverage": 0.9895868323815922, - "singleton_rate": 0.5448438024857238, - "ambiguity_rate": 0.4551561975142761, - "empty_set_rate": 0.0, - "default_rate": 0.0141081625797783, - "default_rate_ambiguous": 0.017712177121771217 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q2", - "n_obs": 824, - "set_coverage": 0.991504854368932, - "singleton_rate": 0.6492718446601942, - "ambiguity_rate": 0.35072815533980584, - "empty_set_rate": 0.0, - "default_rate": 0.01820388349514563, - "default_rate_ambiguous": 0.03460207612456748 - }, - { - "slice_name": "issue_quarter", - "slice_value": "2020Q3", - "n_obs": 485, - "set_coverage": 0.9979381443298969, - "singleton_rate": 0.6515463917525773, - "ambiguity_rate": 0.34845360824742266, - "empty_set_rate": 0.0, - "default_rate": 0.002061855670103093, - "default_rate_ambiguous": 0.0 - } - ] - }, - "benchmark_matrix": [ - { - "method": "lac", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9199043591012356, - "singleton_rate": 0.5266172810968364, - "ambiguity_rate": 0.4733827189031636, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.2880326555525884, - "default_rate_singleton_positive": 0.6134743053848045, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9193409157399348, - "singleton_rate": 0.5392622503783377, - "ambiguity_rate": 0.46073774962166225, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.29499702110313253, - "default_rate_singleton_positive": 0.6162516382699869, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "global", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.9043554894191838, - "singleton_rate": 0.6666618509114419, - "ambiguity_rate": 0.33333814908855813, - "empty_set_rate": 0.0, - "default_rate_ambiguous": 0.3719864342135203, - "default_rate_singleton_positive": 0.8275862068965517, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.893346672975306, - "singleton_rate": 0.5100823855325082, - "ambiguity_rate": 0.4607594205201738, - "empty_set_rate": 0.029158193947318046, - "default_rate_ambiguous": 0.2950145018421259, - "default_rate_singleton_positive": 0.6161722236807561, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "lac", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "grade", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8927182169184704, - "singleton_rate": 0.6196107184264038, - "ambiguity_rate": 0.3617234143222968, - "empty_set_rate": 0.018665867251299352, - "default_rate_ambiguous": 0.34914628057913133, - "default_rate_singleton_positive": 0.72, - "default_rate_overall": 0.21977902907151034 - }, - { - "method": "margin", - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "calibrator": "VennAbersScoreCalibrator", - "alpha": 0.1, - "n_obs": 276869.0, - "set_coverage": 0.8925773560781453, - "singleton_rate": 0.49602158421491754, - "ambiguity_rate": 0.4733827189031636, - "empty_set_rate": 0.030595696881918887, - "default_rate_ambiguous": 0.2880326555525884, - "default_rate_singleton_positive": 0.6134743053848045, - "default_rate_overall": 0.21977902907151034 - } - ], - "decision_use_case": { - "probability_first": true, - "set_first": false, - "recommended_guardrail": "selective_ambiguity_defer" - }, - "promotion_rationale": "Binary conformal sets selected via lac + score_decile_mondrian with set coverage 92.0% and ambiguity 47.3%.", - "promotion_note": "Binary set prediction remains a sidecar triage/abstention signal; it does not replace the interval-first conformal stack." -} diff --git a/models/conformal_gap_summary.json b/models/conformal_gap_summary.json deleted file mode 100644 index 0ddec7c..0000000 --- a/models/conformal_gap_summary.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-16T00:30:00+00:00", - "experiment_name": "conformal_gap_stage_c", - "purpose": "Stage C gap-correction experiments: test whether post-hoc group-coverage gap correction can push min_group_coverage_90 above 0.88 threshold while keeping winkler <= 1.22.", - "n_candidates": 12, - "decision": "REJECTED — score_decile_mondrian retained as champion", - "decision_rationale": "All 12 Stage C candidates fail min_group_coverage_90 gate (0.8624 < 0.88 target). The existing champion score_decile_mondrian achieves min_group_coverage_90=0.8873, which passes. Gap correction did not improve group coverage sufficiently.", - "champion_comparison": { - "champion_variant": "score_decile_mondrian", - "champion_min_group_coverage_90": 0.8873, - "champion_coverage_90": 0.9283, - "champion_winkler_90": 1.2032, - "champion_methodological_justification_pass": true, - "best_stage_c_min_group_coverage_90": 0.8624, - "best_stage_c_coverage_90": 0.9038, - "best_stage_c_winkler_90": 1.1908, - "best_stage_c_namespace": "gap_stage_c_1_grade_scaled0_mgs200_alpha009_floor092_t1_ts250" - }, - "candidates_ranked": [ - { - "rank": 1, - "namespace": "gap_stage_c_1_grade_scaled0_mgs200_alpha009_floor092_t1_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 2, - "namespace": "gap_stage_c_1_grade_scaled0_mgs200_alpha009_floor092_t0_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 3, - "namespace": "gap_stage_c_1_grade_scaled0_mgs200_alpha009_floor092_t1_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 4, - "namespace": "gap_stage_c_1_grade_scaled0_mgs200_alpha009_floor092_t0_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 5, - "namespace": "gap_stage_c_2_grade_scaled0_mgs200_alpha009_floor092_t1_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 6, - "namespace": "gap_stage_c_2_grade_scaled0_mgs200_alpha009_floor092_t0_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 7, - "namespace": "gap_stage_c_2_grade_scaled0_mgs200_alpha009_floor092_t1_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 8, - "namespace": "gap_stage_c_2_grade_scaled0_mgs200_alpha009_floor092_t0_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 9, - "namespace": "gap_stage_c_3_grade_scaled0_mgs200_alpha009_floor092_t1_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 10, - "namespace": "gap_stage_c_3_grade_scaled0_mgs200_alpha009_floor092_t0_ts250", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 11, - "namespace": "gap_stage_c_3_grade_scaled0_mgs200_alpha009_floor092_t1_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - }, - { - "rank": 12, - "namespace": "gap_stage_c_3_grade_scaled0_mgs200_alpha009_floor092_t0_ts500", - "coverage_90": 0.9038, - "min_group_coverage_90": 0.8624, - "avg_width_90": 0.7795, - "winkler_90": 1.1908, - "strict_overall_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": ["min_group_coverage_90"] - } - ], - "archived_at_utc": "2026-03-16T00:30:00+00:00", - "archived_by": "P2.3 audit — declarative-chasing-kahan plan" -} diff --git a/models/conformal_method_registry.json b/models/conformal_method_registry.json deleted file mode 100644 index 798c4dc..0000000 --- a/models/conformal_method_registry.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-03-30T16:09:15.677245+00:00", - "run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "project_scope": "credit_risk_tabular", - "decision_principles": [ - "roi_high", - "baseline_compatibility", - "reproducibility", - "release_stable_over_colab_notebooks" - ], - "libraries": { - "mapie": { - "version_family": "1.3.x", - "role": [ - "primary_conformal_intervals", - "classification_sets", - "time_series_adaptive_benchmark" - ], - "status": "adopted", - "notes": "Primary conformal library for PD intervals, LGD/EAD candidates, and ACI/EnbPI benchmarks." - }, - "crepes": { - "version_family": "0.9.x", - "role": [ - "p_values", - "predictive_systems", - "research_only" - ], - "status": "research_only", - "notes": "Kept for p-values/predictive systems. Not used as a Venn-Abers substitute." - }, - "venn_abers": { - "version_family": "1.5.x", - "role": [ - "probability_calibration" - ], - "status": "adopted", - "notes": "Canonical implementation for Venn-Abers calibration." - }, - "nonconformist": { - "status": "rejected", - "reason": "legacy_unmaintained_api" - }, - "fortuna": { - "status": "rejected_for_canonical_use", - "reason": "archived_project_and_extra_dependency_surface" - }, - "neuralprophet": { - "status": "excluded_from_canonical_stack", - "reason": "beta_ts_stack_outside_current_tabular_credit_scope" - } - }, - "methods": { - "pd_core": { - "calibration_default": "venn_abers", - "calibration_candidates": [ - "platt", - "isotonic", - "venn_abers" - ], - "binary_classification_sets": { - "status": "research_sidecar", - "method": "lac", - "artifact": "models/pd_set_prediction_status.json" - } - }, - "pd_conformal": { - "operational_family": "mondrian_group_conditional", - "implemented_variants": [ - "global_split", - "mondrian_unscaled", - "mondrian_scaled", - "mondrian_selected_cfg", - "cross_conformal_score_space" - ], - "strict_diagnostics": [ - "kupiec", - "christoffersen" - ], - "selector_artifact": "models/conformal_variant_selection_status.json" - }, - "lgd_ead": { - "current_operational_variants": [ - "two_stage_split", - "direct_split", - "direct_cqr", - "direct_adaptive_grade_temporal" - ], - "research_extensions": [ - "mapie_cqr_short_benchmark", - "jackknife_after_bootstrap_short_benchmark" - ] - }, - "time_series": { - "official_baseline": "statsforecast_native_intervals", - "research_shortlist": [ - "enbpi", - "aci", - "online_conformal" - ], - "status_artifact": "models/time_series_status.json" - }, - "multiclass_extension": { - "status": "backlog_p2", - "candidate_targets": [ - "loan_status_multiclass", - "stage_migration", - "delinquency_bucket" - ] - } - } -} diff --git a/models/conformal_policy_sensitivity_status.json b/models/conformal_policy_sensitivity_status.json deleted file mode 100644 index 835fe39..0000000 --- a/models/conformal_policy_sensitivity_status.json +++ /dev/null @@ -1,425 +0,0 @@ -{ - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-03-14T00:56:07.110893+00:00", - "run_tag": "untracked", - "overall_pass": false, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "strict_diagnostics", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9282909968252133, - "coverage_95": 0.9605445174432674, - "avg_width_90": 0.7569209206934802, - "min_group_coverage_90": 0.8873291466568156, - "critical_alerts": 0, - "warning_alerts": 5, - "total_alerts": 5, - "winkler_90": 1.2032086933790644, - "winkler_90_raw_pass": false, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "compensated_band", - "winkler_90_compensated_pass": true, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.131933531105045, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2704.433247593697, - "p_value": 0.0, - "reject": true, - "n_violations": 19854, - "n_total": 276869, - "violation_rate": 0.07170900317478664, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 696.4865357655508, - "p_value": 0.0, - "reject": true, - "n_violations": 10924, - "n_total": 276869, - "violation_rate": 0.039455482556732605, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2704.433247593697, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 2.205384103988763, - "p_ind": 0.13752963415074881, - "reject_ind": false, - "lr_cc": 2706.638631697686, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 238636, - "n01": 18378, - "n10": 18378, - "n11": 1476 - } - }, - "christoffersen_95": { - "lr_uc": 696.4865357655508, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 3.5346932247193763, - "p_ind": 0.06009745052932036, - "reject_ind": false, - "lr_cc": 700.0212289902702, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 255489, - "n01": 10455, - "n10": 10455, - "n11": 469 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07170900317478664, - "violation_rate_95": 0.039455482556732605, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.13752963415074881, - "christoffersen_independence_pvalue_95": 0.06009745052932036 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.0282909968252133, - "coverage_deviation_95": 0.010544517443267454, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "compensated_band", - "winkler_90_raw_pass": false, - "winkler_90_compensated_pass": true, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "diagnostic_warning_not_blocking_for_promotion" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_intervals_mondrian.parquet", - "policy_config": "configs/conformal_policy_sensitivity.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-13T22:03:58.134690+00:00", - "run_tag": "untracked", - "lgd": { - "available": true, - "n_train": 4632, - "n_cal": 5443, - "n_test": 5514, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "fallback_min_shortfall", - "model_metrics": { - "two_stage": { - "stage1_auc": NaN, - "lgd_mae": 0.19767328004785892, - "lgd_rmse": 0.2315233281759835 - }, - "direct": { - "lgd_mae": 0.216261627304772, - "lgd_rmse": 0.2430344524892947, - "lgd_bias": -0.2029258690300959 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9051505259339863, - "target_coverage": 0.9, - "coverage_gap": 0.005150525933986239, - "avg_interval_width": 0.5186727871531672, - "median_interval_width": 0.5201016360361282 - }, - "metrics_95": { - "empirical_coverage": 0.9555676459920203, - "target_coverage": 0.95, - "coverage_gap": 0.005567645992020309, - "avg_interval_width": 0.5815355057336601, - "median_interval_width": 0.5770183981808585 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.29965081801807014, - "q_start_95": 0.33830919909043555, - "q_end_by_grade_90": { - "A": 0.43705081801806067, - "B": 0.30025081801805936, - "C": 0.3044508180180598, - "D": 0.25345081801806435, - "F": 0.25585081801807014, - "G": 0.21445081801807014, - "E": 0.2324508180180695 - }, - "q_end_by_grade_95": { - "A": 0.5297091990904265, - "B": 0.3449091990904159, - "C": 0.27110919909040715, - "D": 0.1901091990904133, - "F": 0.21650919909043448, - "G": 0.24110919909043527, - "E": 0.19910919909042846 - }, - "offset_end_by_grade_90": { - "A": 0.14790124507994787, - "B": 0.1995765218858127, - "C": 0.19835245647123723, - "D": 0.18133436003830944, - "E": 0.11374929200271823, - "F": 0.10593248620654751, - "G": 0.1147169381751545 - }, - "offset_end_by_grade_95": { - "A": 0.14790124507994787, - "B": 0.1995765218858127, - "C": 0.19835245647123723, - "D": 0.18133436003830944, - "E": 0.11374929200271823, - "F": 0.10593248620654751, - "G": 0.1147169381751545 - } - } - }, - "guardrails": { - "overall_pass": false, - "coverage_90": 0.9051505259339863, - "coverage_95": 0.9555676459920203, - "min_grade_coverage_90": 0.90311986863711, - "width_inflation_90": 0.8812901731685944, - "width_inflation_95": 0.847775209760914, - "abs_bias": 0.10295197519048196, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7898077620602104, - "coverage_95": 0.886108088501995, - "avg_width_90": 0.5885380354217827, - "avg_width_95": 0.6859548368932163, - "bias": -0.18137919436929215, - "min_grade_coverage_90": 0.5123152709359606, - "min_year_coverage_90": 0.5201238390092879, - "n_test": 5514.0, - "n_grades_eval": 4.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7533550961189699, - "coverage_95": 0.843126586869786, - "avg_width_90": 0.5993016360361402, - "avg_width_95": 0.676618398180871, - "bias": -0.2029258690300959, - "min_grade_coverage_90": 0.6141215106732348, - "min_year_coverage_90": 0.4117647058823529, - "n_test": 5514.0, - "n_grades_eval": 4.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0182887085736838, - "width_inflation_95": 0.9863891349543779, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7874501269495828, - "coverage_95": 0.8980776206021037, - "avg_width_90": 0.5758490409346204, - "avg_width_95": 0.6823317624655033, - "bias": -0.17172900110001055, - "min_grade_coverage_90": 0.7454808387563269, - "min_year_coverage_90": 0.4938080495356037, - "n_test": 5514.0, - "n_grades_eval": 4.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9784398055461809, - "width_inflation_95": 0.9947182026675073, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9051505259339863, - "coverage_95": 0.9555676459920203, - "avg_width_90": 0.5186727871531672, - "avg_width_95": 0.5815355057336601, - "bias": -0.10295197519048196, - "min_grade_coverage_90": 0.90311986863711, - "min_year_coverage_90": 0.903250773993808, - "n_test": 5514.0, - "n_grades_eval": 4.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8812901731685944, - "width_inflation_95": 0.847775209760914, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "jackknife_after_bootstrap_short_benchmark", - "coverage_90": 0.8937250634747914, - "coverage_95": 0.9497642364889373, - "avg_width_90": 0.5427707674376765, - "avg_width_95": 0.6232091397772214, - "bias": -0.11317406381283228, - "min_grade_coverage_90": 0.7044334975369458, - "min_year_coverage_90": 0.7260061919504643, - "n_test": 5514.0, - "n_grades_eval": 4.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9222356666357059, - "width_inflation_95": 0.9085279471164911, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - } - ] - }, - "ead": { - "available": true, - "n_train": 4632, - "n_cal": 5443, - "n_test": 5514, - "model_metrics": { - "ead_mae": 105.07354706547333, - "ead_r2": 0.9994724107152437 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.8900979325353645, - "target_coverage": 0.9, - "coverage_gap": 0.009902067464635511, - "avg_interval_width": 392.6879621325351, - "median_interval_width": 392.6879621325352 - }, - "metrics_95": { - "empirical_coverage": 0.9345302865433442, - "target_coverage": 0.95, - "coverage_gap": 0.01546971345665571, - "avg_interval_width": 550.3975134776584, - "median_interval_width": 550.3975134776592 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": true, - "effective_sample_size_per_split": 25000 - } - }, - "policy_sensitivity": { - "metric": "winkler_90", - "results": [ - { - "max_winkler_90": 1.2, - "strict_overall_pass": false, - "non_statistical_checks_pass": false, - "methodological_justification_pass": false, - "failing_non_statistical_checks": [ - "winkler_90" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ] - }, - { - "max_winkler_90": 1.22, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "failing_non_statistical_checks": [], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ] - }, - { - "max_winkler_90": 1.25, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "failing_non_statistical_checks": [], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ] - } - ] - } -} diff --git a/models/conformal_variant_selection_status.json b/models/conformal_variant_selection_status.json deleted file mode 100644 index 2107909..0000000 --- a/models/conformal_variant_selection_status.json +++ /dev/null @@ -1,238 +0,0 @@ -{ - "schema_version": "2026-04-03.1", - "generated_at_utc": "2026-04-07T14:39:29.493223+00:00", - "artifact_namespace": "", - "calibrator_override_path": "", - "selected_variant": "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "selection_rank": 1, - "promotion_pass": true, - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time" - ], - "strict_diagnostics_role": "Kupiec/Christoffersen remain strict diagnostics in validate_conformal_policy.py; they are not the primary selector here.", - "variants_tested": [ - "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "global_split", - "cross_conformal_score_space", - "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "mondrian_selected_cfg" - ], - "report_path": "data/processed/conformal_variant_selection_report.parquet", - "summary_path": "data/processed/conformal_variant_benchmark.parquet", - "temporal_diagnostics_path": "data/processed/conformal_temporal_diagnostics.parquet", - "local_diagnostics_path": "data/processed/conformal_local_diagnostics.parquet", - "selected_metrics": { - "coverage": 0.9151837150421318, - "coverage_gap": 0.015183715042131785, - "avg_width": 0.7799199594252145, - "min_group_coverage": 0.8914942528735632, - "winkler_90": 1.1171291360673112, - "stability_over_time": 0.039565474559288674 - }, - "search_space": { - "partition_candidates": [ - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian" - ], - "partition_probability_sources": [ - "calibrated" - ], - "n_score_bins_candidates": [ - 10 - ], - "fallback_modes": [ - "global_only" - ], - "score_scale_families": [ - "bernoulli_sqrt" - ], - "min_group_sizes": [ - 100 - ], - "calibration_size_fractions": [ - 0.25, - 0.5, - 0.75, - 1.0 - ] - }, - "top_variants": [ - { - "variant": "score_decile_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9151837150421318, - "coverage_gap": 0.015183715042131785, - "avg_width": 0.7799199594252145, - "median_width": 0.9942782944989191, - "winkler_90": 1.1171291360673112, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8914942528735632, - "max_group_coverage": 0.9532581996821269, - "std_group_coverage": 0.01745020926626937, - "min_monthly_coverage": 0.9028579890419073, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.039565474559288674, - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "score_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": true, - "selection_rank": 1 - }, - { - "variant": "global_split", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.8987318912554313, - "coverage_gap": 0.00126810874456873, - "avg_width": 0.9188873625898447, - "median_width": 0.9336885786320458, - "winkler_90": 1.09175138019133, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8515356322449575, - "max_group_coverage": 0.9832466782206817, - "std_group_coverage": 0.047074916949785994, - "min_monthly_coverage": 0.8827681524260822, - "last_monthly_coverage": 1.0, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03740927094307107, - "partition": NaN, - "partition_probability_source": NaN, - "n_score_bins": NaN, - "fallback_mode": NaN, - "scaled_scores": NaN, - "score_scale_family": NaN, - "min_group_size": NaN, - "selected_alpha_used": NaN, - "fallback_groups_n": NaN, - "calibration_fraction": NaN, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 2 - }, - { - "variant": "cross_conformal_score_space", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9066, - "coverage_gap": 0.006599999999999939, - "avg_width": 0.9264959581932166, - "median_width": 0.9393232285128859, - "winkler_90": 1.0761021403896558, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8591549295774648, - "max_group_coverage": 1.0, - "std_group_coverage": 0.04756372971496412, - "min_monthly_coverage": 0.8623188405797102, - "last_monthly_coverage": 1.0, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.04566317874614079, - "partition": NaN, - "partition_probability_source": NaN, - "n_score_bins": NaN, - "fallback_mode": NaN, - "scaled_scores": NaN, - "score_scale_family": NaN, - "min_group_size": NaN, - "selected_alpha_used": NaN, - "fallback_groups_n": NaN, - "calibration_fraction": NaN, - "implementation_note": "Cross conformal executed on calibrated score space with a lightweight linear regressor.", - "evaluation_sample_n_cal": 5000.0, - "evaluation_sample_n_test": 5000.0, - "promotion_pass": false, - "selection_rank": 3 - }, - { - "variant": "grade::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.891920005489961, - "coverage_gap": 0.008079994510039046, - "avg_width": 0.7309907182326675, - "median_width": 0.9127438586534876, - "winkler_90": 1.256966978198614, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8534978173177875, - "max_group_coverage": 0.9116287223086531, - "std_group_coverage": 0.02170023572723275, - "min_monthly_coverage": 0.87639648205372, - "last_monthly_coverage": 0.9583333333333334, - "max_monthly_gap": 0.07836538461538456, - "stability_over_time": 0.026479277810963868, - "partition": "grade", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "global_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 0.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 4 - }, - { - "variant": "grade_x_scoreband_mondrian::prob=calibrated::bins=10::fallback=global_only::scale=bernoulli_sqrt::mgs=100", - "alpha": 0.1, - "target_coverage": 0.9, - "coverage": 0.9146961198256215, - "coverage_gap": 0.014696119825621468, - "avg_width": 0.7795528428194203, - "median_width": 0.9951357285429856, - "winkler_90": 1.1176487443153287, - "p90_width": 1.0, - "p95_width": 1.0, - "min_group_coverage": 0.8636363636363636, - "max_group_coverage": 0.988009592326139, - "std_group_coverage": 0.023832037233754995, - "min_monthly_coverage": 0.9017226911496126, - "last_monthly_coverage": 0.9895833333333334, - "max_monthly_gap": 0.09999999999999998, - "stability_over_time": 0.03877850237253919, - "partition": "grade_x_scoreband_mondrian", - "partition_probability_source": "calibrated", - "n_score_bins": 10.0, - "fallback_mode": "global_only", - "scaled_scores": true, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100.0, - "selected_alpha_used": 0.1, - "fallback_groups_n": 23.0, - "calibration_fraction": 1.0, - "implementation_note": NaN, - "evaluation_sample_n_cal": NaN, - "evaluation_sample_n_test": NaN, - "promotion_pass": false, - "selection_rank": 5 - } - ] -} diff --git a/models/cqr_comparison_status.json b/models/cqr_comparison_status.json deleted file mode 100644 index a56c5c9..0000000 --- a/models/cqr_comparison_status.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-17T14:11:11.921734+00:00", - "run_tag": "untracked", - "alpha": 0.1, - "low_risk_threshold": 0.1, - "n_test": 276869, - "cqr_status": "success", - "methods": { - "global_splitconf": { - "method": "global_splitconf", - "empirical_coverage": 0.899064178365942, - "min_group_coverage": 0.5880993645291739, - "avg_width": 0.9560809238980794, - "median_width": 1.0, - "std_width": 0.06814587264074508, - "n_eligible": 0, - "n_test": 276869, - "eligible_pct": 0.0 - }, - "mondrian_splitconf": { - "method": "mondrian_splitconf", - "empirical_coverage": 0.8729001802296393, - "min_group_coverage": 0.8502237939990054, - "avg_width": 0.8703068871888459, - "median_width": 0.9841398534896959, - "std_width": 0.18198601796418745, - "n_eligible": 0, - "n_test": 276869, - "eligible_pct": 0.0 - }, - "cqr_asymmetric": { - "method": "cqr_asymmetric", - "empirical_coverage": 0.9826271630265577, - "min_group_coverage": 0.9522572802121899, - "avg_width": 0.9695708112572741, - "median_width": 1.0, - "std_width": 0.09458606914238259, - "n_eligible": 0, - "n_test": 276869, - "eligible_pct": 0.0 - } - }, - "per_group_coverage": { - "global_splitconf": { - "A": 0.9404873735978339, - "B": 0.9662731871838112, - "C": 0.9190426255526585, - "D": 0.7541687665709841, - "E": 0.6940339859623199, - "F": 0.5880993645291739, - "G": 0.6551724137931034 - }, - "mondrian_splitconf": { - "A": 0.8502237939990054, - "B": 0.8763660023104195, - "C": 0.8840692665230699, - "D": 0.8789359510923155, - "E": 0.893424455116365, - "F": 0.9081455805892548, - "G": 0.9556650246305419 - }, - "cqr_asymmetric": { - "A": 0.9522572802121899, - "B": 0.9839731247759291, - "C": 0.9989797075161546, - "D": 0.9988824857023906, - "E": 0.9980605836719616, - "F": 0.9982668977469671, - "G": 1.0 - } - }, - "summary": { - "best_coverage_method": "cqr_asymmetric", - "tightest_method": "mondrian_splitconf", - "most_eligible_method": "global_splitconf" - } -} diff --git a/models/cqr_mondrian_status.json b/models/cqr_mondrian_status.json deleted file mode 100644 index 3ba4222..0000000 --- a/models/cqr_mondrian_status.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-17T15:03:27.268880+00:00", - "run_tag": "untracked", - "alpha": 0.1, - "n_numeric_features": 35, - "n_test": 276869, - "cqr_mondrian_group_offsets": { - "A": 9.999298880321093e-07, - "B": 1.000000465989359e-06, - "C": 1.000000465989359e-06, - "D": 1.000000465989359e-06, - "E": 1.000000465989359e-06, - "F": 1.000000465989359e-06, - "G": 1.000000465989359e-06 - }, - "methods": { - "symmetric_global": { - "method": "symmetric_global", - "alpha": 0.1, - "target_coverage": 0.9, - "empirical_coverage": 0.899064178365942, - "coverage_gap": 0.0009358216340580316, - "min_group_coverage": 0.5880993645291739, - "avg_width": 0.9560809238980794, - "median_width": 1.0, - "std_width": 0.06814587264074508, - "n_eligible": 0, - "eligible_pct": 0.0 - }, - "symmetric_mondrian": { - "method": "symmetric_mondrian", - "alpha": 0.1, - "target_coverage": 0.9, - "empirical_coverage": 0.8729001802296393, - "coverage_gap": 0.02709981977036069, - "min_group_coverage": 0.8502237939990054, - "avg_width": 0.8703068871888459, - "median_width": 0.9841398534896959, - "std_width": 0.18198601796418745, - "n_eligible": 0, - "eligible_pct": 0.0 - }, - "cqr_global": { - "method": "cqr_global", - "alpha": 0.1, - "target_coverage": 0.9, - "empirical_coverage": 0.9826271630265577, - "coverage_gap": -0.0826271630265577, - "min_group_coverage": 0.9522572802121899, - "avg_width": 0.9695708112572983, - "median_width": 1.0, - "std_width": 0.09458606914237175, - "n_eligible": 0, - "eligible_pct": 0.0 - }, - "cqr_mondrian": { - "method": "cqr_mondrian", - "alpha": 0.1, - "target_coverage": 0.9, - "empirical_coverage": 0.976692948650806, - "coverage_gap": -0.076692948650806, - "min_group_coverage": 0.9295601480908438, - "avg_width": 0.9695708112413847, - "median_width": 1.0, - "std_width": 0.09458606915953721, - "n_eligible": 0, - "eligible_pct": 0.0 - } - }, - "summary": { - "best_coverage": "cqr_global", - "tightest": "symmetric_mondrian", - "best_min_group_cov": "cqr_global" - } -} diff --git a/models/crpto_distribution_robustness_status.json b/models/crpto_distribution_robustness_status.json deleted file mode 100644 index 6e283f5..0000000 --- a/models/crpto_distribution_robustness_status.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "champion_promotion_changed": false, - "elapsed_sec": 1.066145, - "generated_artifacts": [ - "reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv", - "reports/crpto/tables/crpto_tableA23_multidistribution_robustness.tex", - "reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv", - "reports/crpto/tables/crpto_tableA24_online_conformal_stability.tex" - ], - "generated_at_utc": "2026-05-29T04:16:46.996425+00:00", - "multidistribution_robustness": { - "all_grades_meet_target": true, - "all_supported_cells_meet_target": false, - "marginal_coverage_90": 0.9293384235866059, - "min_grade_coverage_90": 0.9003509420022164, - "min_grade_period_cell_coverage_90": 0.8762765121759623, - "min_period_coverage_90": 0.9225192106405249, - "n_supported_cells": 47, - "robust_coverage_gap": 0.02372348782403777, - "worst_grade": "E", - "worst_grade_period_cell": "E|2018Q4", - "worst_period": "2018Q4" - }, - "notes": [ - "Read-only diagnostics on frozen Mondrian conformal intervals.", - "A23 stresses coverage across unknown groups (grade, grade x vintage).", - "A24 is a static-OOT online-control diagnostic, not streaming validation." - ], - "online_conformal_stability": { - "aci_alpha_target_max_abs_deviation": 0.02292023539471473, - "aci_step_gamma": 0.05, - "all_periods_meet_target": true, - "default_rate_first": 0.2329945849714413, - "default_rate_last": 0.002061855670103093, - "final_cumulative_coverage_90": 0.9293384235866059, - "first_period": "2018Q1", - "last_period": "2020Q3", - "max_period_coverage_90": 0.9979381443298969, - "min_period_coverage_90": 0.9225192106405249, - "n_periods": 11 - }, - "schema_version": "2026-05-28.1", - "source_intervals": "data/processed/conformal_intervals_mondrian.parquet", - "target_coverage_90": 0.9 -} diff --git a/models/crpto_multidataset_external_status.json b/models/crpto_multidataset_external_status.json deleted file mode 100644 index df59cc3..0000000 --- a/models/crpto_multidataset_external_status.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "duration_seconds": 2.203823, - "external_replications": [ - { - "alpha01_coverage": 0.9943090202029784, - "auc_roc": 0.7074290432845566, - "brier": 0.1781741704802637, - "coverage_90": 0.9205159821682634, - "credit_product": "Marketplace personal loans", - "dataset": "Prosper", - "default_rate": 0.3091758352035324, - "external_role": "main external economic replication", - "gate": "pass", - "lp_candidate_cap": "all", - "min_group_coverage_90": 0.8850308641975309, - "n_rows": 54807, - "oot_candidates": 10531, - "pr_auc": 0.4484817219126147, - "price_of_robustness_pct": 0.094619019478566, - "robust_objective": 199419.08877567088 - }, - { - "alpha01_coverage": 0.9907116706887202, - "auc_roc": 0.7838634726333499, - "brier": 0.0128541034734516, - "coverage_90": 0.974523173547136, - "credit_product": "Single-family mortgages", - "dataset": "Freddie FM48", - "default_rate": 0.0145435981792141, - "external_role": "main external economic replication", - "gate": "pass", - "lp_candidate_cap": "all", - "min_group_coverage_90": 0.5, - "n_rows": 3173355, - "oot_candidates": 1396053, - "pr_auc": 0.0517822051269566, - "price_of_robustness_pct": 0.0108636254963404, - "robust_objective": 1291227.5801824057 - } - ], - "freddie_all_candidate_lp": { - "available_candidates": 1396053, - "max_funded_rank": 551, - "robust_objective": 1291227.5801824057 - }, - "freddie_segment_sensitivity": [ - { - "all_lp_solved": true, - "alpha01_coverage": 0.9907116706887202, - "alpha01_pass": true, - "auc_roc": 0.7838634726333499, - "available_oot_candidates": 1396053, - "coverage90_pass": true, - "coverage_90": 0.974523173547136, - "default_rate": 0.0145435981792141, - "funded_outside_top250k": 0, - "max_funded_rank": 551, - "min_group_coverage_90": 0.5, - "price_of_robustness_pct": 0.0108636254963404, - "robust_objective_all_candidates": 1291227.5801824057, - "rows_total": 3173355, - "segment": "both" - }, - { - "all_lp_solved": true, - "alpha01_coverage": 0.9850218288730144, - "alpha01_pass": false, - "auc_roc": 0.700087721623812, - "available_oot_candidates": 597603, - "coverage90_pass": true, - "coverage_90": 0.970528929740982, - "default_rate": 0.0297209890995989, - "funded_outside_top250k": 0, - "max_funded_rank": 155, - "min_group_coverage_90": 0.891597510373444, - "price_of_robustness_pct": 0.0236887045711261, - "robust_objective_all_candidates": 1236591.1430666302, - "rows_total": 1158306, - "segment": "red" - }, - { - "all_lp_solved": true, - "alpha01_coverage": 0.9950629344354688, - "alpha01_pass": true, - "auc_roc": 0.6997060916716304, - "available_oot_candidates": 798450, - "coverage90_pass": true, - "coverage_90": 0.98100319368777, - "default_rate": 0.0058192133293036, - "funded_outside_top250k": 0, - "max_funded_rank": 745, - "min_group_coverage_90": 0.5, - "price_of_robustness_pct": 0.0100393498751964, - "robust_objective_all_candidates": 1287497.9769414272, - "rows_total": 2015049, - "segment": "green" - } - ], - "freddie_selected_variant": "freddie_fm48_both", - "freddie_top_screen_stable_through_candidates": "all", - "generated_at_utc": "2026-07-06T04:03:37.119346+00:00", - "home_credit_archive_metrics": { - "alpha01_coverage": 0.9922389871661464, - "auc_roc": 0.7520619425836951, - "coverage_90": 0.9014264654873396, - "rows_total": 307511 - }, - "home_credit_policy": "discarded_from_main_claim_archived_only", - "outputs": [ - "reports/crpto/tables/crpto_tableA25_external_replication_gate.csv", - "reports/crpto/tables/crpto_tableA25_external_replication_gate.tex", - "reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.csv", - "reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.tex", - "reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.csv", - "reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.tex", - "reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv", - "reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.tex", - "reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.csv", - "reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.tex", - "reports/crpto/tables/crpto_tableA30_external_metric_intervals.csv", - "reports/crpto/tables/crpto_tableA30_external_metric_intervals.tex", - "reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.csv", - "reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.tex", - "reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.csv", - "reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.tex", - "reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv", - "reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.tex", - "reports/crpto/figures/crpto_fig22_external_replication.png", - "reports/crpto/figures/crpto_fig22_external_replication.pdf", - "reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png", - "reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.pdf", - "reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png", - "reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.pdf", - "reports/crpto/multidataset/source/source_log.md" - ], - "prosper_default_definition_sensitivity": [ - { - "alpha01_coverage": 0.9943090202029784, - "alpha01_pass": true, - "auc_roc": 0.7074290432845566, - "available_oot_candidates": 10531, - "coverage90_pass": true, - "coverage_90": 0.9205159821682634, - "default_rate": 0.3091758352035324, - "min_group_coverage_90": 0.8850308641975309, - "price_of_robustness_pct": 0.0946190194785686, - "robust_objective_all_candidates": 199419.0887756553, - "rows_total": 54807, - "variant": "main_chargedoff_or_defaulted" - }, - { - "alpha01_coverage": 0.9947702652222636, - "alpha01_pass": true, - "auc_roc": 0.6796193367985865, - "available_oot_candidates": 8019, - "coverage90_pass": true, - "coverage_90": 0.9148300336197236, - "default_rate": 0.1166534459427931, - "min_group_coverage_90": 0.8671328671328671, - "price_of_robustness_pct": 0.0338142847144336, - "robust_objective_all_candidates": 272969.2286232216, - "rows_total": 42862, - "variant": "defaulted_only" - }, - { - "alpha01_coverage": 0.99212753395001, - "alpha01_pass": true, - "auc_roc": 0.7131638982556929, - "available_oot_candidates": 10150, - "coverage90_pass": true, - "coverage_90": 0.9035622908876204, - "default_rate": 0.2398257273074066, - "min_group_coverage_90": 0.8355855855855856, - "price_of_robustness_pct": 0.0942093080550645, - "robust_objective_all_candidates": 205828.04819793423, - "rows_total": 49807, - "variant": "chargedoff_only" - } - ], - "source_dir": "reports/crpto/multidataset/source" -} diff --git a/models/decision_threshold.json b/models/decision_threshold.json deleted file mode 100644 index 6c2358c..0000000 --- a/models/decision_threshold.json +++ /dev/null @@ -1,1763 +0,0 @@ -{ - "enabled": true, - "selected_threshold": 0.05, - "fallback_threshold": 0.35, - "selection_metrics": { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.6666666666666666, - "max_dpd": 0.11599234857293383, - "max_eo_gap": 0.13196437729927657, - "min_dir": 0.8810703143159722, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01064184660225942, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9893565406437804, - "robust_overall_pass": false, - "robust_pass_ratio": 0.6666666666666666, - "robust_max_dpd": 0.11599234857293383, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8810703143159722, - "distance_from_fallback": 0.3 - }, - "search_summary": [ - { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.6666666666666666, - "max_dpd": 0.11599234857293383, - "max_eo_gap": 0.13196437729927657, - "min_dir": 0.8810703143159722, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01064184660225942, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9893565406437804, - "robust_overall_pass": false, - "robust_pass_ratio": 0.6666666666666666, - "robust_max_dpd": 0.11599234857293383, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8810703143159722, - "distance_from_fallback": 0.3 - }, - { - "threshold": 0.060000000000000005, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.1237250991956449, - "max_eo_gap": 0.14081716308101877, - "min_dir": 0.8727226124638846, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01935904815967404, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9806295374995029, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.1237250991956449, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8727226124638846, - "distance_from_fallback": 0.29 - }, - { - "threshold": 0.07, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.14854215851361985, - "max_eo_gap": 0.16781138359217107, - "min_dir": 0.8445113919152774, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.02838253866291185, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9715686365159658, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.14854215851361985, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8445113919152774, - "distance_from_fallback": 0.27999999999999997 - }, - { - "threshold": 0.9400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0, - "max_eo_gap_secondary": 0.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0, - "robust_max_eo_gap": 0.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5900000000000002 - }, - { - "threshold": 0.9500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0, - "max_eo_gap_secondary": 0.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0, - "robust_max_eo_gap": 0.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.6000000000000002 - }, - { - "threshold": 0.9300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 5.01898850651632e-05, - "max_eo_gap_secondary": 0.00021159542953872197, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 5.01898850651632e-05, - "robust_max_eo_gap": 0.00021159542953872197, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5800000000000002 - }, - { - "threshold": 0.9200000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.00023421946363742828, - "max_eo_gap_secondary": 0.000711490572749911, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.00023421946363742828, - "robust_max_eo_gap": 0.000711490572749911, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5700000000000003 - }, - { - "threshold": 0.9100000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0010491043380628667, - "max_eo_gap_secondary": 0.0027515068267510373, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0010491043380628667, - "robust_max_eo_gap": 0.0027515068267510373, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5600000000000003 - }, - { - "threshold": 0.9000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00017301536953199343, - "max_eo_gap": 0.0007458975634012929, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0024976095552908875, - "max_eo_gap_secondary": 0.006333931449391483, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0024976095552908875, - "robust_max_eo_gap": 0.006333931449391483, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5500000000000003 - }, - { - "threshold": 0.8900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.004381055791832544, - "max_eo_gap_secondary": 0.01034532571305053, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.004381055791832544, - "robust_max_eo_gap": 0.01034532571305053, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5400000000000003 - }, - { - "threshold": 0.8800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.007053935115793775, - "max_eo_gap_secondary": 0.01654179711285209, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.007053935115793775, - "robust_max_eo_gap": 0.01654179711285209, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5300000000000002 - }, - { - "threshold": 0.8700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.011175597535644762, - "max_eo_gap_secondary": 0.024981129168733143, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.011175597535644762, - "robust_max_eo_gap": 0.024981129168733143, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5200000000000002 - }, - { - "threshold": 0.8600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.015467368597198794, - "max_eo_gap_secondary": 0.03294879019478193, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.015467368597198794, - "robust_max_eo_gap": 0.03294879019478193, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5100000000000002 - }, - { - "threshold": 0.8500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.020359855835502108, - "max_eo_gap_secondary": 0.043237890915092, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.020359855835502108, - "robust_max_eo_gap": 0.043237890915092, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5000000000000002 - }, - { - "threshold": 0.8400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.025619612572834495, - "max_eo_gap_secondary": 0.055862694941826885, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.025619612572834495, - "robust_max_eo_gap": 0.055862694941826885, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4900000000000002 - }, - { - "threshold": 0.8300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.032507178789284705, - "max_eo_gap_secondary": 0.0692713920290453, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.032507178789284705, - "robust_max_eo_gap": 0.0692713920290453, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4800000000000002 - }, - { - "threshold": 0.8200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.03945118466693076, - "max_eo_gap_secondary": 0.08490799570921694, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.03945118466693076, - "robust_max_eo_gap": 0.08490799570921694, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4700000000000002 - }, - { - "threshold": 0.8100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.046921789508703314, - "max_eo_gap_secondary": 0.1014522650383695, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.046921789508703314, - "robust_max_eo_gap": 0.1014522650383695, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4600000000000002 - }, - { - "threshold": 0.8000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0548686454979141, - "max_eo_gap_secondary": 0.1171713837775394, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0548686454979141, - "robust_max_eo_gap": 0.1171713837775394, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4500000000000002 - }, - { - "threshold": 0.7900000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.06186094608114822, - "max_eo_gap_secondary": 0.1342520009901807, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.06186094608114822, - "robust_max_eo_gap": 0.1342520009901807, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.44000000000000017 - }, - { - "threshold": 0.7800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0697766773769321, - "max_eo_gap_secondary": 0.15112633055532634, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0697766773769321, - "robust_max_eo_gap": 0.15112633055532634, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.43000000000000016 - }, - { - "threshold": 0.7700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0010961819304327648, - "max_eo_gap": 0.0032816229116945108, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0788936830945915, - "max_eo_gap_secondary": 0.16915587094644774, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0788936830945915, - "robust_max_eo_gap": 0.16915587094644774, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.42000000000000026 - }, - { - "threshold": 0.7600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011250178253547636, - "max_eo_gap": 0.0033932622536211885, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.08641695411080393, - "max_eo_gap_secondary": 0.18735044145556565, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.08641695411080393, - "robust_max_eo_gap": 0.18735044145556565, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.41000000000000025 - }, - { - "threshold": 0.7500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011250178253547636, - "max_eo_gap": 0.0033932622536211885, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.09568842715764136, - "max_eo_gap_secondary": 0.20554501196468355, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.09568842715764136, - "robust_max_eo_gap": 0.20554501196468355, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.40000000000000024 - }, - { - "threshold": 0.7400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011826896151987615, - "max_eo_gap": 0.003641894774754953, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.10498365272821508, - "max_eo_gap_secondary": 0.22605000412575296, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.10498365272821508, - "robust_max_eo_gap": 0.22605000412575296, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.39000000000000024 - }, - { - "threshold": 0.7300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011826896151987615, - "max_eo_gap": 0.003641894774754953, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11612737928752148, - "max_eo_gap_secondary": 0.24535852793134746, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.11612737928752148, - "robust_max_eo_gap": 0.24535852793134746, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3800000000000002 - }, - { - "threshold": 0.7200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012119434335175667, - "max_eo_gap": 0.0036793953858393, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.12719780513061393, - "max_eo_gap_secondary": 0.26425447644195066, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.12719780513061393, - "robust_max_eo_gap": 0.26425447644195066, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3700000000000002 - }, - { - "threshold": 0.7100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012119434335175667, - "max_eo_gap": 0.0036793953858393, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1393037481451432, - "max_eo_gap_secondary": 0.2850895288390131, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1393037481451432, - "robust_max_eo_gap": 0.2850895288390131, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3600000000000002 - }, - { - "threshold": 0.7000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1519968400713119, - "max_eo_gap_secondary": 0.3056357785295816, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1519968400713119, - "robust_max_eo_gap": 0.3056357785295816, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3500000000000002 - }, - { - "threshold": 0.6900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.16554396677804703, - "max_eo_gap_secondary": 0.32787358692961466, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.16554396677804703, - "robust_max_eo_gap": 0.32787358692961466, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3400000000000002 - }, - { - "threshold": 0.6800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1791231197890534, - "max_eo_gap_secondary": 0.3489149269741728, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1791231197890534, - "robust_max_eo_gap": 0.3489149269741728, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3300000000000002 - }, - { - "threshold": 0.6700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19308658845131468, - "max_eo_gap_secondary": 0.37057512996121794, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.19308658845131468, - "robust_max_eo_gap": 0.37057512996121794, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3200000000000002 - }, - { - "threshold": 0.6600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.003810517363671919, - "max_eo_gap": 0.009844868735083532, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.20726356580871755, - "max_eo_gap_secondary": 0.3908325769452925, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.20726356580871755, - "robust_max_eo_gap": 0.3908325769452925, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.31000000000000016 - }, - { - "threshold": 0.6500000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22218782359911607, - "max_eo_gap_secondary": 0.4103473883983827, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.22218782359911607, - "robust_max_eo_gap": 0.4103473883983827, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.30000000000000016 - }, - { - "threshold": 0.6400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23735761638892744, - "max_eo_gap_secondary": 0.43151250103143823, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23735761638892744, - "robust_max_eo_gap": 0.43151250103143823, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.29000000000000015 - }, - { - "threshold": 0.6300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.25342414569833355, - "max_eo_gap_secondary": 0.45243006848749895, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.25342414569833355, - "robust_max_eo_gap": 0.45243006848749895, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.28000000000000014 - }, - { - "threshold": 0.6200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013294184225292747, - "max_eo_gap": 0.033975453662108766, - "min_dir": 0.13177210028450426, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.26933054348638347, - "max_eo_gap_secondary": 0.4743378166515389, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26933054348638347, - "robust_max_eo_gap": 0.4743378166515389, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.27000000000000013 - }, - { - "threshold": 0.6100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013294184225292747, - "max_eo_gap": 0.033975453662108766, - "min_dir": 0.13177210028450426, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28583476562082993, - "max_eo_gap_secondary": 0.49624556481557885, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28583476562082993, - "robust_max_eo_gap": 0.49624556481557885, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2600000000000001 - }, - { - "threshold": 0.08000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.15809517248092775, - "max_eo_gap": 0.17768726765109522, - "min_dir": 0.8335097310976739, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.038032644055798204, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9618395032461408, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.15809517248092775, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8335097310976739, - "distance_from_fallback": 0.26999999999999996 - }, - { - "threshold": 0.09000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.17822307820545347, - "max_eo_gap": 0.19842899991154916, - "min_dir": 0.8083789577331744, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.048055230404599025, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.951691008308314, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.17822307820545347, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8083789577331744, - "distance_from_fallback": 0.25999999999999995 - }, - { - "threshold": 0.1, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2042612352913714, - "max_eo_gap": 0.21937034125278632, - "min_dir": 0.7743940576256616, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.05696641127357405, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.942562648084245, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2042612352913714, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7743940576256616, - "distance_from_fallback": 0.24999999999999997 - }, - { - "threshold": 0.11000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.23930768904618782, - "max_eo_gap": 0.2502639734208494, - "min_dir": 0.728783576311881, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.06567555833555871, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9335706300381653, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23930768904618782, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.728783576311881, - "distance_from_fallback": 0.23999999999999996 - }, - { - "threshold": 0.12000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2515219805200316, - "max_eo_gap": 0.26250067384950704, - "min_dir": 0.7122606464849844, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.07507937839900314, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9237654752566465, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2515219805200316, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7122606464849844, - "distance_from_fallback": 0.22999999999999998 - }, - { - "threshold": 0.26000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2636190895512414, - "max_eo_gap": 0.23624550091862243, - "min_dir": 0.4338939620690133, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2536145617667357, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7463854382332642, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2636190895512414, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4338939620690133, - "distance_from_fallback": 0.08999999999999991 - }, - { - "threshold": 0.13, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2658596768553525, - "max_eo_gap": 0.2763477190371312, - "min_dir": 0.6906747724059996, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.08392722129581132, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9144127938313642, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2658596768553525, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6906747724059996, - "distance_from_fallback": 0.21999999999999997 - }, - { - "threshold": 0.5700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.030159423083006127, - "max_eo_gap": 0.0648470654460438, - "min_dir": 0.15856908037363568, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2686213316291236, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4030004515535375, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2686213316291236, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.15856908037363568, - "distance_from_fallback": 0.2200000000000002 - }, - { - "threshold": 0.27, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2511577971553087, - "max_eo_gap": 0.2292484311493398, - "min_dir": 0.4255819265614355, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.26916839199447895, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7308316080055209, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26916839199447895, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4255819265614355, - "distance_from_fallback": 0.07999999999999996 - }, - { - "threshold": 0.25000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.26956096051550626, - "max_eo_gap": 0.2427376643337971, - "min_dir": 0.44247005727218675, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23872498274672183, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.761275017253278, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26956096051550626, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.44247005727218675, - "distance_from_fallback": 0.09999999999999992 - }, - { - "threshold": 0.32000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.201926959395188, - "max_eo_gap": 0.21361606312557896, - "min_dir": 0.38557295561413285, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2697091161904569, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6806583800080782, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2697091161904569, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38557295561413285, - "distance_from_fallback": 0.029999999999999916 - }, - { - "threshold": 0.5600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.031642591425932266, - "max_eo_gap": 0.06722593441096221, - "min_dir": 0.1711983171752973, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2752640463465945, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.413803696655444, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2752640463465945, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1711983171752973, - "distance_from_fallback": 0.2100000000000002 - }, - { - "threshold": 0.33, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.19656097294094244, - "max_eo_gap": 0.21158785611115977, - "min_dir": 0.38037105893837414, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.27759151645861113, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6673120721562222, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.27759151645861113, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38037105893837414, - "distance_from_fallback": 0.019999999999999962 - }, - { - "threshold": 0.14, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2795550429578503, - "max_eo_gap": 0.28873971751859007, - "min_dir": 0.6686369301454252, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.09191548746205047, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9058039909366099, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2795550429578503, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6686369301454252, - "distance_from_fallback": 0.20999999999999996 - }, - { - "threshold": 0.5500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.03512295913961884, - "max_eo_gap": 0.0724216375520566, - "min_dir": 0.18362660857718285, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28246822446397823, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.42383383903612554, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28246822446397823, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.18362660857718285, - "distance_from_fallback": 0.20000000000000018 - }, - { - "threshold": 0.34, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16349914397938847, - "max_eo_gap": 0.194086581862903, - "min_dir": 0.3554635882617693, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28442742549731403, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6547373021617968, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28442742549731403, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3554635882617693, - "distance_from_fallback": 0.009999999999999953 - }, - { - "threshold": 0.28, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.24345610157346592, - "max_eo_gap": 0.22376766205028947, - "min_dir": 0.4197378593494001, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.285024154589372, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7149758454106279, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.285024154589372, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4197378593494001, - "distance_from_fallback": 0.06999999999999995 - }, - { - "threshold": 0.5400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.035209884747781646, - "max_eo_gap": 0.0723340445098051, - "min_dir": 0.18379438959336827, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28900820730917065, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.43362881297363814, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28900820730917065, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.18379438959336827, - "distance_from_fallback": 0.19000000000000017 - }, - { - "threshold": 0.24000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.29029064114694414, - "max_eo_gap": 0.2677009068058277, - "min_dir": 0.4636660019001129, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22358523119392681, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.776414768806073, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.29029064114694414, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4636660019001129, - "distance_from_fallback": 0.10999999999999993 - }, - { - "threshold": 0.35000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16344147218954447, - "max_eo_gap": 0.194086581862903, - "min_dir": 0.3555444213688215, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2905400316372354, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6425921113181214, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2905400316372354, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3555444213688215, - "distance_from_fallback": 5.551115123125783e-17 - }, - { - "threshold": 0.23000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2905539255118134, - "max_eo_gap": 0.2680437922395825, - "min_dir": 0.46369386934084617, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2089803312629399, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7910196687370599, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2905539255118134, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.46369386934084617, - "distance_from_fallback": 0.11999999999999994 - }, - { - "threshold": 0.5300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0403146748152267, - "max_eo_gap": 0.08114334921441291, - "min_dir": 0.19789293865906576, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.295013762097875, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.44370934092948394, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.295013762097875, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.19789293865906576, - "distance_from_fallback": 0.18000000000000016 - }, - { - "threshold": 0.36000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.14868292125743604, - "max_eo_gap": 0.1916209892002126, - "min_dir": 0.3406425797587434, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2960028372367042, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6302871616902891, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2960028372367042, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3406425797587434, - "distance_from_fallback": 0.010000000000000064 - }, - { - "threshold": 0.29000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.21437153485524849, - "max_eo_gap": 0.21858784420377475, - "min_dir": 0.397699873835764, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3003450655624569, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.699654934437543, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3003450655624569, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.397699873835764, - "distance_from_fallback": 0.05999999999999994 - }, - { - "threshold": 0.5200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.041731812900372714, - "max_eo_gap": 0.08390088355957892, - "min_dir": 0.19688260845059552, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30087657632374554, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.45233279747246347, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30087657632374554, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.19688260845059552, - "distance_from_fallback": 0.17000000000000015 - }, - { - "threshold": 0.37000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.13517643440493854, - "max_eo_gap": 0.17736763861486973, - "min_dir": 0.3288785155721022, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3020048992841837, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.617580314276046, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3020048992841837, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3288785155721022, - "distance_from_fallback": 0.020000000000000073 - }, - { - "threshold": 0.6000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013554125202987545, - "max_eo_gap": 0.03467155549186215, - "min_dir": 0.13115802603621796, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3033424786224419, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3033424786224419, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2500000000000002 - }, - { - "threshold": 0.5100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05926571198436584, - "max_eo_gap": 0.11093322918059595, - "min_dir": 0.23481920107750384, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3037722196734903, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4646368414157465, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3037722196734903, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23481920107750384, - "distance_from_fallback": 0.16000000000000014 - }, - { - "threshold": 0.15000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30462552756201483, - "max_eo_gap": 0.3094632067079493, - "min_dir": 0.6205829590804469, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.3333333333333333, - "max_dpd_secondary": 0.10364044168391995, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8963595583160798, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30462552756201483, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6205829590804469, - "distance_from_fallback": 0.19999999999999996 - }, - { - "threshold": 0.38000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.13514759851001654, - "max_eo_gap": 0.17724332235430287, - "min_dir": 0.3289256058521096, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3066923337925269, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6054087376171872, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3066923337925269, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3289256058521096, - "distance_from_fallback": 0.030000000000000082 - }, - { - "threshold": 0.21000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30725954057676075, - "max_eo_gap": 0.2912496167144003, - "min_dir": 0.49312845553888834, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.18075396825396828, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8192460317460315, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30725954057676075, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.49312845553888834, - "distance_from_fallback": 0.13999999999999996 - }, - { - "threshold": 0.22000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30728377931431783, - "max_eo_gap": 0.2912774517047823, - "min_dir": 0.4928472214524447, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19492753623188408, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8050724637681157, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30728377931431783, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4928472214524447, - "distance_from_fallback": 0.12999999999999995 - }, - { - "threshold": 0.5000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.059294965802684646, - "max_eo_gap": 0.1111818617017297, - "min_dir": 0.23501111641692612, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30795201446763537, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.47449994973238624, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30795201446763537, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23501111641692612, - "distance_from_fallback": 0.15000000000000013 - }, - { - "threshold": 0.39000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.11439494192524596, - "max_eo_gap": 0.16190271160661288, - "min_dir": 0.3091079430467421, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3099330214438822, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5938075230054995, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3099330214438822, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3091079430467421, - "distance_from_fallback": 0.04000000000000009 - }, - { - "threshold": 0.49000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0597906090205171, - "max_eo_gap": 0.11147564744825267, - "min_dir": 0.23713078358259218, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.311376924746901, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4845910408768376, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.311376924746901, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23713078358259218, - "distance_from_fallback": 0.14000000000000007 - }, - { - "threshold": 0.4000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10845641762598612, - "max_eo_gap": 0.15556287209919506, - "min_dir": 0.3040044213820179, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31326673356787554, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5819683328744976, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31326673356787554, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3040044213820179, - "distance_from_fallback": 0.0500000000000001 - }, - { - "threshold": 0.4800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05987753462867991, - "max_eo_gap": 0.11151237066656802, - "min_dir": 0.2371443705407816, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31471630691044583, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4942863107436398, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31471630691044583, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2371443705407816, - "distance_from_fallback": 0.13000000000000012 - }, - { - "threshold": 0.4700000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05987753462867991, - "max_eo_gap": 0.11151237066656802, - "min_dir": 0.2371443705407816, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3161739632131047, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.50468703176158, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3161739632131047, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2371443705407816, - "distance_from_fallback": 0.1200000000000001 - }, - { - "threshold": 0.30000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.21419517609854205, - "max_eo_gap": 0.218514397767144, - "min_dir": 0.3975120032507631, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31633022774327124, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6836697722567286, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31633022774327124, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3975120032507631, - "distance_from_fallback": 0.04999999999999993 - }, - { - "threshold": 0.4100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10102803820945128, - "max_eo_gap": 0.15089301699310287, - "min_dir": 0.29491822558551767, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31638687073610067, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5696087295286993, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31638687073610067, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.29491822558551767, - "distance_from_fallback": 0.06000000000000011 - }, - { - "threshold": 0.42000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10094069467789164, - "max_eo_gap": 0.15085629377478754, - "min_dir": 0.29481822104459976, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3174257522117674, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.55915589379507, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3174257522117674, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.29481822104459976, - "distance_from_fallback": 0.07000000000000006 - }, - { - "threshold": 0.16000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3174942526580365, - "max_eo_gap": 0.3179596496142747, - "min_dir": 0.5808442520204031, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11575224292615593, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8842477570738438, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3174942526580365, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5808442520204031, - "distance_from_fallback": 0.18999999999999995 - }, - { - "threshold": 0.4600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.07810433185705226, - "max_eo_gap": 0.13166759554807111, - "min_dir": 0.2679513177106169, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3179965369126446, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5150996963983899, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3179965369126446, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2679513177106169, - "distance_from_fallback": 0.1100000000000001 - }, - { - "threshold": 0.2, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3183726637204274, - "max_eo_gap": 0.3091432706631215, - "min_dir": 0.5213367898569017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1672532781228433, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8327467218771565, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3183726637204274, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5213367898569017, - "distance_from_fallback": 0.14999999999999997 - }, - { - "threshold": 0.44000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08336458533082108, - "max_eo_gap": 0.13890594209069865, - "min_dir": 0.273435422295163, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3188279089878116, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5372644569753188, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3188279089878116, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.273435422295163, - "distance_from_fallback": 0.09000000000000008 - }, - { - "threshold": 0.19, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.31895565210873483, - "max_eo_gap": 0.3099594741515642, - "min_dir": 0.5239891957017335, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1538388543823327, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8461611456176671, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31895565210873483, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5239891957017335, - "distance_from_fallback": 0.15999999999999998 - }, - { - "threshold": 0.17000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.31906768589901163, - "max_eo_gap": 0.31915227852930955, - "min_dir": 0.5747685223745502, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.12853692201518285, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8714630779848169, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31906768589901163, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5747685223745502, - "distance_from_fallback": 0.17999999999999994 - }, - { - "threshold": 0.45000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08254715011148175, - "max_eo_gap": 0.1375300332173891, - "min_dir": 0.2710332012436776, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31913239252484515, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5259214032261954, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31913239252484515, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2710332012436776, - "distance_from_fallback": 0.10000000000000009 - }, - { - "threshold": 0.43000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08347992891050908, - "max_eo_gap": 0.13903025835126553, - "min_dir": 0.2731608198123157, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31916464098515046, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5473812407030914, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31916464098515046, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2731608198123157, - "distance_from_fallback": 0.08000000000000007 - }, - { - "threshold": 0.5900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0198019029823924, - "max_eo_gap": 0.04539471874809337, - "min_dir": 0.1448191861439761, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.320732761841726, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.320732761841726, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2400000000000002 - }, - { - "threshold": 0.18000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.32298016264072443, - "max_eo_gap": 0.32001828065149646, - "min_dir": 0.5564282974845477, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.14108868184955137, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8589113181504484, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.32298016264072443, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5564282974845477, - "distance_from_fallback": 0.16999999999999993 - }, - { - "threshold": 0.31000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.20381591988400619, - "max_eo_gap": 0.21522645791440148, - "min_dir": 0.38521942370553597, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3324534161490683, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6675465838509316, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3324534161490683, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38521942370553597, - "distance_from_fallback": 0.039999999999999925 - }, - { - "threshold": 0.5800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.022402566529530786, - "max_eo_gap": 0.051319459438441406, - "min_dir": 0.1424960213271532, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.33828317658236634, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.33828317658236634, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2300000000000002 - } - ], - "source": "validation_fairness_search", - "fairness_policy_path": "configs/crpto_fairness_policy.yaml", - "validation_rows": 119782, - "secondary_validation_rows": 237584, - "calibration_method": "venn_abers", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-06-07T22:28:25.252712+00:00", - "run_tag": "ijds-rebaseline-2026-06-07" -} diff --git a/models/decision_threshold_v2.json b/models/decision_threshold_v2.json deleted file mode 100644 index 6c2358c..0000000 --- a/models/decision_threshold_v2.json +++ /dev/null @@ -1,1763 +0,0 @@ -{ - "enabled": true, - "selected_threshold": 0.05, - "fallback_threshold": 0.35, - "selection_metrics": { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.6666666666666666, - "max_dpd": 0.11599234857293383, - "max_eo_gap": 0.13196437729927657, - "min_dir": 0.8810703143159722, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01064184660225942, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9893565406437804, - "robust_overall_pass": false, - "robust_pass_ratio": 0.6666666666666666, - "robust_max_dpd": 0.11599234857293383, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8810703143159722, - "distance_from_fallback": 0.3 - }, - "search_summary": [ - { - "threshold": 0.05, - "overall_pass": false, - "pass_ratio": 0.6666666666666666, - "max_dpd": 0.11599234857293383, - "max_eo_gap": 0.13196437729927657, - "min_dir": 0.8810703143159722, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01064184660225942, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9893565406437804, - "robust_overall_pass": false, - "robust_pass_ratio": 0.6666666666666666, - "robust_max_dpd": 0.11599234857293383, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8810703143159722, - "distance_from_fallback": 0.3 - }, - { - "threshold": 0.060000000000000005, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.1237250991956449, - "max_eo_gap": 0.14081716308101877, - "min_dir": 0.8727226124638846, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.01935904815967404, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9806295374995029, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.1237250991956449, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8727226124638846, - "distance_from_fallback": 0.29 - }, - { - "threshold": 0.07, - "overall_pass": false, - "pass_ratio": 0.3333333333333333, - "max_dpd": 0.14854215851361985, - "max_eo_gap": 0.16781138359217107, - "min_dir": 0.8445113919152774, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.02838253866291185, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9715686365159658, - "robust_overall_pass": false, - "robust_pass_ratio": 0.3333333333333333, - "robust_max_dpd": 0.14854215851361985, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8445113919152774, - "distance_from_fallback": 0.27999999999999997 - }, - { - "threshold": 0.9400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0, - "max_eo_gap_secondary": 0.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0, - "robust_max_eo_gap": 0.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5900000000000002 - }, - { - "threshold": 0.9500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0, - "max_eo_gap_secondary": 0.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0, - "robust_max_eo_gap": 0.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.6000000000000002 - }, - { - "threshold": 0.9300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 5.01898850651632e-05, - "max_eo_gap_secondary": 0.00021159542953872197, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 5.01898850651632e-05, - "robust_max_eo_gap": 0.00021159542953872197, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5800000000000002 - }, - { - "threshold": 0.9200000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.00023421946363742828, - "max_eo_gap_secondary": 0.000711490572749911, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.00023421946363742828, - "robust_max_eo_gap": 0.000711490572749911, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5700000000000003 - }, - { - "threshold": 0.9100000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0, - "max_eo_gap": 0.0, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0010491043380628667, - "max_eo_gap_secondary": 0.0027515068267510373, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0010491043380628667, - "robust_max_eo_gap": 0.0027515068267510373, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5600000000000003 - }, - { - "threshold": 0.9000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00017301536953199343, - "max_eo_gap": 0.0007458975634012929, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0024976095552908875, - "max_eo_gap_secondary": 0.006333931449391483, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0024976095552908875, - "robust_max_eo_gap": 0.006333931449391483, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5500000000000003 - }, - { - "threshold": 0.8900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.004381055791832544, - "max_eo_gap_secondary": 0.01034532571305053, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.004381055791832544, - "robust_max_eo_gap": 0.01034532571305053, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5400000000000003 - }, - { - "threshold": 0.8800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.007053935115793775, - "max_eo_gap_secondary": 0.01654179711285209, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.007053935115793775, - "robust_max_eo_gap": 0.01654179711285209, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5300000000000002 - }, - { - "threshold": 0.8700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.011175597535644762, - "max_eo_gap_secondary": 0.024981129168733143, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.011175597535644762, - "robust_max_eo_gap": 0.024981129168733143, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5200000000000002 - }, - { - "threshold": 0.8600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.015467368597198794, - "max_eo_gap_secondary": 0.03294879019478193, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.015467368597198794, - "robust_max_eo_gap": 0.03294879019478193, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5100000000000002 - }, - { - "threshold": 0.8500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00020185126445399233, - "max_eo_gap": 0.000870213823968175, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.020359855835502108, - "max_eo_gap_secondary": 0.043237890915092, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.020359855835502108, - "robust_max_eo_gap": 0.043237890915092, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.5000000000000002 - }, - { - "threshold": 0.8400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.025619612572834495, - "max_eo_gap_secondary": 0.055862694941826885, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.025619612572834495, - "robust_max_eo_gap": 0.055862694941826885, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4900000000000002 - }, - { - "threshold": 0.8300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.032507178789284705, - "max_eo_gap_secondary": 0.0692713920290453, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.032507178789284705, - "robust_max_eo_gap": 0.0692713920290453, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4800000000000002 - }, - { - "threshold": 0.8200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.03945118466693076, - "max_eo_gap_secondary": 0.08490799570921694, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.03945118466693076, - "robust_max_eo_gap": 0.08490799570921694, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4700000000000002 - }, - { - "threshold": 0.8100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.046921789508703314, - "max_eo_gap_secondary": 0.1014522650383695, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.046921789508703314, - "robust_max_eo_gap": 0.1014522650383695, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4600000000000002 - }, - { - "threshold": 0.8000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0548686454979141, - "max_eo_gap_secondary": 0.1171713837775394, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0548686454979141, - "robust_max_eo_gap": 0.1171713837775394, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.4500000000000002 - }, - { - "threshold": 0.7900000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.06186094608114822, - "max_eo_gap_secondary": 0.1342520009901807, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.06186094608114822, - "robust_max_eo_gap": 0.1342520009901807, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.44000000000000017 - }, - { - "threshold": 0.7800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.00031719484414198794, - "max_eo_gap": 0.0012431626056688214, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0697766773769321, - "max_eo_gap_secondary": 0.15112633055532634, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0697766773769321, - "robust_max_eo_gap": 0.15112633055532634, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.43000000000000016 - }, - { - "threshold": 0.7700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0010961819304327648, - "max_eo_gap": 0.0032816229116945108, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.0788936830945915, - "max_eo_gap_secondary": 0.16915587094644774, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.0788936830945915, - "robust_max_eo_gap": 0.16915587094644774, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.42000000000000026 - }, - { - "threshold": 0.7600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011250178253547636, - "max_eo_gap": 0.0033932622536211885, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.08641695411080393, - "max_eo_gap_secondary": 0.18735044145556565, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.08641695411080393, - "robust_max_eo_gap": 0.18735044145556565, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.41000000000000025 - }, - { - "threshold": 0.7500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011250178253547636, - "max_eo_gap": 0.0033932622536211885, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.09568842715764136, - "max_eo_gap_secondary": 0.20554501196468355, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.09568842715764136, - "robust_max_eo_gap": 0.20554501196468355, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.40000000000000024 - }, - { - "threshold": 0.7400000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011826896151987615, - "max_eo_gap": 0.003641894774754953, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.10498365272821508, - "max_eo_gap_secondary": 0.22605000412575296, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.10498365272821508, - "robust_max_eo_gap": 0.22605000412575296, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.39000000000000024 - }, - { - "threshold": 0.7300000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0011826896151987615, - "max_eo_gap": 0.003641894774754953, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11612737928752148, - "max_eo_gap_secondary": 0.24535852793134746, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.11612737928752148, - "robust_max_eo_gap": 0.24535852793134746, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3800000000000002 - }, - { - "threshold": 0.7200000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012119434335175667, - "max_eo_gap": 0.0036793953858393, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.12719780513061393, - "max_eo_gap_secondary": 0.26425447644195066, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.12719780513061393, - "robust_max_eo_gap": 0.26425447644195066, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3700000000000002 - }, - { - "threshold": 0.7100000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012119434335175667, - "max_eo_gap": 0.0036793953858393, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1393037481451432, - "max_eo_gap_secondary": 0.2850895288390131, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1393037481451432, - "robust_max_eo_gap": 0.2850895288390131, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3600000000000002 - }, - { - "threshold": 0.7000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1519968400713119, - "max_eo_gap_secondary": 0.3056357785295816, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1519968400713119, - "robust_max_eo_gap": 0.3056357785295816, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3500000000000002 - }, - { - "threshold": 0.6900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.16554396677804703, - "max_eo_gap_secondary": 0.32787358692961466, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.16554396677804703, - "robust_max_eo_gap": 0.32787358692961466, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3400000000000002 - }, - { - "threshold": 0.6800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1791231197890534, - "max_eo_gap_secondary": 0.3489149269741728, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.1791231197890534, - "robust_max_eo_gap": 0.3489149269741728, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3300000000000002 - }, - { - "threshold": 0.6700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0012984511182835634, - "max_eo_gap": 0.003778838504375497, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19308658845131468, - "max_eo_gap_secondary": 0.37057512996121794, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.19308658845131468, - "robust_max_eo_gap": 0.37057512996121794, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.3200000000000002 - }, - { - "threshold": 0.6600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.003810517363671919, - "max_eo_gap": 0.009844868735083532, - "min_dir": 0.0, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.20726356580871755, - "max_eo_gap_secondary": 0.3908325769452925, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.20726356580871755, - "robust_max_eo_gap": 0.3908325769452925, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.31000000000000016 - }, - { - "threshold": 0.6500000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22218782359911607, - "max_eo_gap_secondary": 0.4103473883983827, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.22218782359911607, - "robust_max_eo_gap": 0.4103473883983827, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.30000000000000016 - }, - { - "threshold": 0.6400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23735761638892744, - "max_eo_gap_secondary": 0.43151250103143823, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23735761638892744, - "robust_max_eo_gap": 0.43151250103143823, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.29000000000000015 - }, - { - "threshold": 0.6300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.01303466117099476, - "max_eo_gap": 0.033577681187963976, - "min_dir": 0.13404403304803017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.25342414569833355, - "max_eo_gap_secondary": 0.45243006848749895, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.25342414569833355, - "robust_max_eo_gap": 0.45243006848749895, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.28000000000000014 - }, - { - "threshold": 0.6200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013294184225292747, - "max_eo_gap": 0.033975453662108766, - "min_dir": 0.13177210028450426, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.26933054348638347, - "max_eo_gap_secondary": 0.4743378166515389, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26933054348638347, - "robust_max_eo_gap": 0.4743378166515389, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.27000000000000013 - }, - { - "threshold": 0.6100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013294184225292747, - "max_eo_gap": 0.033975453662108766, - "min_dir": 0.13177210028450426, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28583476562082993, - "max_eo_gap_secondary": 0.49624556481557885, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28583476562082993, - "robust_max_eo_gap": 0.49624556481557885, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2600000000000001 - }, - { - "threshold": 0.08000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.15809517248092775, - "max_eo_gap": 0.17768726765109522, - "min_dir": 0.8335097310976739, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.038032644055798204, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9618395032461408, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.15809517248092775, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8335097310976739, - "distance_from_fallback": 0.26999999999999996 - }, - { - "threshold": 0.09000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.17822307820545347, - "max_eo_gap": 0.19842899991154916, - "min_dir": 0.8083789577331744, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.048055230404599025, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.951691008308314, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.17822307820545347, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.8083789577331744, - "distance_from_fallback": 0.25999999999999995 - }, - { - "threshold": 0.1, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2042612352913714, - "max_eo_gap": 0.21937034125278632, - "min_dir": 0.7743940576256616, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.05696641127357405, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.942562648084245, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2042612352913714, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7743940576256616, - "distance_from_fallback": 0.24999999999999997 - }, - { - "threshold": 0.11000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.23930768904618782, - "max_eo_gap": 0.2502639734208494, - "min_dir": 0.728783576311881, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.06567555833555871, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9335706300381653, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.23930768904618782, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.728783576311881, - "distance_from_fallback": 0.23999999999999996 - }, - { - "threshold": 0.12000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2515219805200316, - "max_eo_gap": 0.26250067384950704, - "min_dir": 0.7122606464849844, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.07507937839900314, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9237654752566465, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2515219805200316, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.7122606464849844, - "distance_from_fallback": 0.22999999999999998 - }, - { - "threshold": 0.26000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2636190895512414, - "max_eo_gap": 0.23624550091862243, - "min_dir": 0.4338939620690133, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2536145617667357, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7463854382332642, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2636190895512414, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4338939620690133, - "distance_from_fallback": 0.08999999999999991 - }, - { - "threshold": 0.13, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2658596768553525, - "max_eo_gap": 0.2763477190371312, - "min_dir": 0.6906747724059996, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.08392722129581132, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9144127938313642, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2658596768553525, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6906747724059996, - "distance_from_fallback": 0.21999999999999997 - }, - { - "threshold": 0.5700000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.030159423083006127, - "max_eo_gap": 0.0648470654460438, - "min_dir": 0.15856908037363568, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2686213316291236, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4030004515535375, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2686213316291236, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.15856908037363568, - "distance_from_fallback": 0.2200000000000002 - }, - { - "threshold": 0.27, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2511577971553087, - "max_eo_gap": 0.2292484311493398, - "min_dir": 0.4255819265614355, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.26916839199447895, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7308316080055209, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26916839199447895, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4255819265614355, - "distance_from_fallback": 0.07999999999999996 - }, - { - "threshold": 0.25000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.26956096051550626, - "max_eo_gap": 0.2427376643337971, - "min_dir": 0.44247005727218675, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.23872498274672183, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.761275017253278, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.26956096051550626, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.44247005727218675, - "distance_from_fallback": 0.09999999999999992 - }, - { - "threshold": 0.32000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.201926959395188, - "max_eo_gap": 0.21361606312557896, - "min_dir": 0.38557295561413285, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2697091161904569, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6806583800080782, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2697091161904569, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38557295561413285, - "distance_from_fallback": 0.029999999999999916 - }, - { - "threshold": 0.5600000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.031642591425932266, - "max_eo_gap": 0.06722593441096221, - "min_dir": 0.1711983171752973, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2752640463465945, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.413803696655444, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2752640463465945, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.1711983171752973, - "distance_from_fallback": 0.2100000000000002 - }, - { - "threshold": 0.33, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.19656097294094244, - "max_eo_gap": 0.21158785611115977, - "min_dir": 0.38037105893837414, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.27759151645861113, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6673120721562222, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.27759151645861113, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38037105893837414, - "distance_from_fallback": 0.019999999999999962 - }, - { - "threshold": 0.14, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2795550429578503, - "max_eo_gap": 0.28873971751859007, - "min_dir": 0.6686369301454252, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.6666666666666666, - "max_dpd_secondary": 0.09191548746205047, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.9058039909366099, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2795550429578503, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6686369301454252, - "distance_from_fallback": 0.20999999999999996 - }, - { - "threshold": 0.5500000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.03512295913961884, - "max_eo_gap": 0.0724216375520566, - "min_dir": 0.18362660857718285, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28246822446397823, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.42383383903612554, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28246822446397823, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.18362660857718285, - "distance_from_fallback": 0.20000000000000018 - }, - { - "threshold": 0.34, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16349914397938847, - "max_eo_gap": 0.194086581862903, - "min_dir": 0.3554635882617693, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28442742549731403, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6547373021617968, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28442742549731403, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3554635882617693, - "distance_from_fallback": 0.009999999999999953 - }, - { - "threshold": 0.28, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.24345610157346592, - "max_eo_gap": 0.22376766205028947, - "min_dir": 0.4197378593494001, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.285024154589372, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7149758454106279, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.285024154589372, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4197378593494001, - "distance_from_fallback": 0.06999999999999995 - }, - { - "threshold": 0.5400000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.035209884747781646, - "max_eo_gap": 0.0723340445098051, - "min_dir": 0.18379438959336827, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.28900820730917065, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.43362881297363814, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.28900820730917065, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.18379438959336827, - "distance_from_fallback": 0.19000000000000017 - }, - { - "threshold": 0.24000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.29029064114694414, - "max_eo_gap": 0.2677009068058277, - "min_dir": 0.4636660019001129, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.22358523119392681, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.776414768806073, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.29029064114694414, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4636660019001129, - "distance_from_fallback": 0.10999999999999993 - }, - { - "threshold": 0.35000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.16344147218954447, - "max_eo_gap": 0.194086581862903, - "min_dir": 0.3555444213688215, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2905400316372354, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6425921113181214, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2905400316372354, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3555444213688215, - "distance_from_fallback": 5.551115123125783e-17 - }, - { - "threshold": 0.23000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.2905539255118134, - "max_eo_gap": 0.2680437922395825, - "min_dir": 0.46369386934084617, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2089803312629399, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.7910196687370599, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2905539255118134, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.46369386934084617, - "distance_from_fallback": 0.11999999999999994 - }, - { - "threshold": 0.5300000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0403146748152267, - "max_eo_gap": 0.08114334921441291, - "min_dir": 0.19789293865906576, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.295013762097875, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.44370934092948394, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.295013762097875, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.19789293865906576, - "distance_from_fallback": 0.18000000000000016 - }, - { - "threshold": 0.36000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.14868292125743604, - "max_eo_gap": 0.1916209892002126, - "min_dir": 0.3406425797587434, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.2960028372367042, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6302871616902891, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.2960028372367042, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3406425797587434, - "distance_from_fallback": 0.010000000000000064 - }, - { - "threshold": 0.29000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.21437153485524849, - "max_eo_gap": 0.21858784420377475, - "min_dir": 0.397699873835764, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3003450655624569, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.699654934437543, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3003450655624569, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.397699873835764, - "distance_from_fallback": 0.05999999999999994 - }, - { - "threshold": 0.5200000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.041731812900372714, - "max_eo_gap": 0.08390088355957892, - "min_dir": 0.19688260845059552, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30087657632374554, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.45233279747246347, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30087657632374554, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.19688260845059552, - "distance_from_fallback": 0.17000000000000015 - }, - { - "threshold": 0.37000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.13517643440493854, - "max_eo_gap": 0.17736763861486973, - "min_dir": 0.3288785155721022, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3020048992841837, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.617580314276046, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3020048992841837, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3288785155721022, - "distance_from_fallback": 0.020000000000000073 - }, - { - "threshold": 0.6000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.013554125202987545, - "max_eo_gap": 0.03467155549186215, - "min_dir": 0.13115802603621796, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3033424786224419, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3033424786224419, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2500000000000002 - }, - { - "threshold": 0.5100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05926571198436584, - "max_eo_gap": 0.11093322918059595, - "min_dir": 0.23481920107750384, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3037722196734903, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4646368414157465, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3037722196734903, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23481920107750384, - "distance_from_fallback": 0.16000000000000014 - }, - { - "threshold": 0.15000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30462552756201483, - "max_eo_gap": 0.3094632067079493, - "min_dir": 0.6205829590804469, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.3333333333333333, - "max_dpd_secondary": 0.10364044168391995, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8963595583160798, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30462552756201483, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.6205829590804469, - "distance_from_fallback": 0.19999999999999996 - }, - { - "threshold": 0.38000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.13514759851001654, - "max_eo_gap": 0.17724332235430287, - "min_dir": 0.3289256058521096, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3066923337925269, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6054087376171872, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3066923337925269, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3289256058521096, - "distance_from_fallback": 0.030000000000000082 - }, - { - "threshold": 0.21000000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30725954057676075, - "max_eo_gap": 0.2912496167144003, - "min_dir": 0.49312845553888834, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.18075396825396828, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8192460317460315, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30725954057676075, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.49312845553888834, - "distance_from_fallback": 0.13999999999999996 - }, - { - "threshold": 0.22000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.30728377931431783, - "max_eo_gap": 0.2912774517047823, - "min_dir": 0.4928472214524447, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.19492753623188408, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8050724637681157, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30728377931431783, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.4928472214524447, - "distance_from_fallback": 0.12999999999999995 - }, - { - "threshold": 0.5000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.059294965802684646, - "max_eo_gap": 0.1111818617017297, - "min_dir": 0.23501111641692612, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.30795201446763537, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.47449994973238624, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.30795201446763537, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23501111641692612, - "distance_from_fallback": 0.15000000000000013 - }, - { - "threshold": 0.39000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.11439494192524596, - "max_eo_gap": 0.16190271160661288, - "min_dir": 0.3091079430467421, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3099330214438822, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5938075230054995, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3099330214438822, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3091079430467421, - "distance_from_fallback": 0.04000000000000009 - }, - { - "threshold": 0.49000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0597906090205171, - "max_eo_gap": 0.11147564744825267, - "min_dir": 0.23713078358259218, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.311376924746901, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4845910408768376, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.311376924746901, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.23713078358259218, - "distance_from_fallback": 0.14000000000000007 - }, - { - "threshold": 0.4000000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10845641762598612, - "max_eo_gap": 0.15556287209919506, - "min_dir": 0.3040044213820179, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31326673356787554, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5819683328744976, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31326673356787554, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3040044213820179, - "distance_from_fallback": 0.0500000000000001 - }, - { - "threshold": 0.4800000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05987753462867991, - "max_eo_gap": 0.11151237066656802, - "min_dir": 0.2371443705407816, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31471630691044583, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.4942863107436398, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31471630691044583, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2371443705407816, - "distance_from_fallback": 0.13000000000000012 - }, - { - "threshold": 0.4700000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.05987753462867991, - "max_eo_gap": 0.11151237066656802, - "min_dir": 0.2371443705407816, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3161739632131047, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.50468703176158, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3161739632131047, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2371443705407816, - "distance_from_fallback": 0.1200000000000001 - }, - { - "threshold": 0.30000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.21419517609854205, - "max_eo_gap": 0.218514397767144, - "min_dir": 0.3975120032507631, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31633022774327124, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6836697722567286, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31633022774327124, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.3975120032507631, - "distance_from_fallback": 0.04999999999999993 - }, - { - "threshold": 0.4100000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10102803820945128, - "max_eo_gap": 0.15089301699310287, - "min_dir": 0.29491822558551767, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31638687073610067, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5696087295286993, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31638687073610067, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.29491822558551767, - "distance_from_fallback": 0.06000000000000011 - }, - { - "threshold": 0.42000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.10094069467789164, - "max_eo_gap": 0.15085629377478754, - "min_dir": 0.29481822104459976, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3174257522117674, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.55915589379507, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3174257522117674, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.29481822104459976, - "distance_from_fallback": 0.07000000000000006 - }, - { - "threshold": 0.16000000000000003, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3174942526580365, - "max_eo_gap": 0.3179596496142747, - "min_dir": 0.5808442520204031, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.11575224292615593, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8842477570738438, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3174942526580365, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5808442520204031, - "distance_from_fallback": 0.18999999999999995 - }, - { - "threshold": 0.4600000000000001, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.07810433185705226, - "max_eo_gap": 0.13166759554807111, - "min_dir": 0.2679513177106169, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3179965369126446, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5150996963983899, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3179965369126446, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2679513177106169, - "distance_from_fallback": 0.1100000000000001 - }, - { - "threshold": 0.2, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.3183726637204274, - "max_eo_gap": 0.3091432706631215, - "min_dir": 0.5213367898569017, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1672532781228433, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8327467218771565, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3183726637204274, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5213367898569017, - "distance_from_fallback": 0.14999999999999997 - }, - { - "threshold": 0.44000000000000006, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08336458533082108, - "max_eo_gap": 0.13890594209069865, - "min_dir": 0.273435422295163, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3188279089878116, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5372644569753188, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3188279089878116, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.273435422295163, - "distance_from_fallback": 0.09000000000000008 - }, - { - "threshold": 0.19, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.31895565210873483, - "max_eo_gap": 0.3099594741515642, - "min_dir": 0.5239891957017335, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.1538388543823327, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8461611456176671, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31895565210873483, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5239891957017335, - "distance_from_fallback": 0.15999999999999998 - }, - { - "threshold": 0.17000000000000004, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.31906768589901163, - "max_eo_gap": 0.31915227852930955, - "min_dir": 0.5747685223745502, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.12853692201518285, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8714630779848169, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31906768589901163, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5747685223745502, - "distance_from_fallback": 0.17999999999999994 - }, - { - "threshold": 0.45000000000000007, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08254715011148175, - "max_eo_gap": 0.1375300332173891, - "min_dir": 0.2710332012436776, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31913239252484515, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5259214032261954, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31913239252484515, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2710332012436776, - "distance_from_fallback": 0.10000000000000009 - }, - { - "threshold": 0.43000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.08347992891050908, - "max_eo_gap": 0.13903025835126553, - "min_dir": 0.2731608198123157, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.31916464098515046, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.5473812407030914, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.31916464098515046, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.2731608198123157, - "distance_from_fallback": 0.08000000000000007 - }, - { - "threshold": 0.5900000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.0198019029823924, - "max_eo_gap": 0.04539471874809337, - "min_dir": 0.1448191861439761, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.320732761841726, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.320732761841726, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2400000000000002 - }, - { - "threshold": 0.18000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.32298016264072443, - "max_eo_gap": 0.32001828065149646, - "min_dir": 0.5564282974845477, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.14108868184955137, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.8589113181504484, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.32298016264072443, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.5564282974845477, - "distance_from_fallback": 0.16999999999999993 - }, - { - "threshold": 0.31000000000000005, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.20381591988400619, - "max_eo_gap": 0.21522645791440148, - "min_dir": 0.38521942370553597, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.3324534161490683, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.6675465838509316, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.3324534161490683, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.38521942370553597, - "distance_from_fallback": 0.039999999999999925 - }, - { - "threshold": 0.5800000000000002, - "overall_pass": false, - "pass_ratio": 0.0, - "max_dpd": 0.022402566529530786, - "max_eo_gap": 0.051319459438441406, - "min_dir": 0.1424960213271532, - "overall_pass_secondary": false, - "pass_ratio_secondary": 0.0, - "max_dpd_secondary": 0.33828317658236634, - "max_eo_gap_secondary": 1.0, - "min_dir_secondary": 0.0, - "robust_overall_pass": false, - "robust_pass_ratio": 0.0, - "robust_max_dpd": 0.33828317658236634, - "robust_max_eo_gap": 1.0, - "robust_min_dir": 0.0, - "distance_from_fallback": 0.2300000000000002 - } - ], - "source": "validation_fairness_search", - "fairness_policy_path": "configs/crpto_fairness_policy.yaml", - "validation_rows": 119782, - "secondary_validation_rows": 237584, - "calibration_method": "venn_abers", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-06-07T22:28:25.252712+00:00", - "run_tag": "ijds-rebaseline-2026-06-07" -} diff --git a/models/encoding_stability_status.json b/models/encoding_stability_status.json deleted file mode 100644 index 59cf31c..0000000 --- a/models/encoding_stability_status.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "diagnostic_only": true, - "overall_pass": true, - "summary": { - "n_woe_features": 13, - "n_bucket_features": 3, - "woe_failures": 0, - "bucket_failures": 0, - "max_woe_psi": 0.10608693157944711, - "max_bucket_category_psi": 0.1003346994585424 - }, - "top_woe_instabilities": [ - { - "feature": "fico_score_woe", - "base_feature": "fico_score", - "psi": 0.10608693157944711, - "train_spearman_default": -0.11916273687858307, - "test_spearman_default": -0.13158339068843566, - "abs_spearman_delta": 0.012420653809852591, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "sub_grade_woe", - "base_feature": "sub_grade", - "psi": 0.09223400400726883, - "train_spearman_default": -0.2534810176791446, - "test_spearman_default": -0.2575391044121199, - "abs_spearman_delta": 0.004058086732975308, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "verification_status_woe", - "base_feature": "verification_status", - "psi": 0.07954765868429915, - "train_spearman_default": -0.08087367643109547, - "test_spearman_default": -0.09378849797139888, - "abs_spearman_delta": 0.012914821540303406, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "grade_woe", - "base_feature": "grade", - "psi": 0.053333294678026354, - "train_spearman_default": -0.24820072545334526, - "test_spearman_default": -0.25179197164003003, - "abs_spearman_delta": 0.003591246186684771, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "int_rate_woe", - "base_feature": "int_rate", - "psi": 0.030483774874434568, - "train_spearman_default": -0.24022941633935493, - "test_spearman_default": -0.23804071369188304, - "abs_spearman_delta": 0.0021887026474718962, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "loan_amnt_woe", - "base_feature": "loan_amnt", - "psi": 0.025017605027199387, - "train_spearman_default": -0.06631071159542272, - "test_spearman_default": -0.1049143965732922, - "abs_spearman_delta": 0.03860368497786948, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "installment_burden_woe", - "base_feature": "installment_burden", - "psi": 0.02166627974158798, - "train_spearman_default": -0.11182089075187897, - "test_spearman_default": -0.15987422451986308, - "abs_spearman_delta": 0.04805333376798411, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "dti_woe", - "base_feature": "dti", - "psi": 0.012755564928854449, - "train_spearman_default": -0.09546189823576824, - "test_spearman_default": -0.05988408209837229, - "abs_spearman_delta": 0.035577816137395954, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "annual_inc_woe", - "base_feature": "annual_inc", - "psi": 0.006785792547199207, - "train_spearman_default": -0.0611972187021634, - "test_spearman_default": -0.06339551620010127, - "abs_spearman_delta": 0.0021982974979378664, - "sign_consistent": true, - "overall_pass": true - }, - { - "feature": "purpose_woe", - "base_feature": "purpose", - "psi": 0.004405278836604954, - "train_spearman_default": -0.04958647665452675, - "test_spearman_default": -0.0388688125469419, - "abs_spearman_delta": 0.010717664107584852, - "sign_consistent": true, - "overall_pass": true - } - ], - "top_bucket_instabilities": [ - { - "feature": "fico_bucket", - "category_psi": 0.1003346994585424, - "max_share_shift": 0.08743569248778411, - "rank_corr_train_vs_test": 1.0, - "overall_pass": true - }, - { - "feature": "dti_bucket", - "category_psi": 0.08188180225112404, - "max_share_shift": 0.0387876192005146, - "rank_corr_train_vs_test": 0.8857142857142858, - "overall_pass": true - }, - { - "feature": "int_rate_bucket", - "category_psi": 0.06309509084061672, - "max_share_shift": 0.09619374082153495, - "rank_corr_train_vs_test": 0.9999999999999999, - "overall_pass": true - } - ], - "artifacts": { - "woe_report_path": "data/processed/woe_encoding_stability.parquet", - "bucket_report_path": "data/processed/bucket_binning_stability.parquet" - }, - "config_path": "configs/pd_model.champion.yaml", - "schema_version": "2026-03-30.1", - "generated_at_utc": "2026-04-07T17:43:28.456544+00:00", - "run_tag": "paper1-e2e-all-champions-2026-04-07" -} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json deleted file mode 100644 index d5f5bcb..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json +++ /dev/null @@ -1,291 +0,0 @@ -{ - "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", - "calibration_audit": { - "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", - "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_holdout_audit.csv", - "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_audit_grid.parquet", - "outcome_free_selected_candidate_id": "linear-005", - "period": "2017-12", - "policy_evaluations": [ - { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "endpoint_budget": 0.2620820323280323, - "expected_objective": 103567.0099590662, - "gamma": 0.5, - "gamma_cp": 0.1841640646560647, - "gamma_internalized": 0.09208203232803233, - "gamma_residual": 0.09208203232803232, - "markov_loss_threshold": 0.5783097983448702, - "min_budget_utilization": 0.0, - "n_funded": 193, - "n_panel": 20695, - "objective_risk_mode": "point_pd_plus_aversion", - "pd_cap_slack_penalty": 0.0, - "period": "2017-12", - "policy_mode": "blended_uncertainty", - "realized_return": 53313.04791145165, - "realized_risk_tolerance_excess": 0.0, - "risk_tolerance": 0.17, - "role": "calibration_selected", - "screen_V_leq_sqrt_alpha": true, - "screen_risk_excess_leq_alpha": true, - "solver_status": "Optimal", - "tail_focus_quantile": 1.0, - "total_allocated": 999999.9999999999, - "uncertainty_aversion": 0.0, - "weighted_miscoverage": 0.12492500000000001, - "weighted_outcome": 0.14565, - "weighted_pd_effective": 0.16999999999999998, - "weighted_pd_point": 0.07791796767196764 - }, - { - "candidate_id": "linear-006", - "delta_cap_quantile": 1.0, - "endpoint_budget": 0.20350382300175537, - "expected_objective": 97897.74942687502, - "gamma": 0.75, - "gamma_cp": 0.13401529200702159, - "gamma_internalized": 0.10051146900526618, - "gamma_residual": 0.033503823001755396, - "markov_loss_threshold": 0.5197315890185933, - "min_budget_utilization": 0.0, - "n_funded": 191, - "n_panel": 20695, - "objective_risk_mode": "point_pd_plus_aversion", - "pd_cap_slack_penalty": 0.0, - "period": "2017-12", - "policy_mode": "blended_uncertainty", - "realized_return": 38379.50087450525, - "realized_risk_tolerance_excess": 0.0, - "risk_tolerance": 0.17, - "role": "incumbent_linear", - "screen_V_leq_sqrt_alpha": true, - "screen_risk_excess_leq_alpha": true, - "solver_status": "Optimal", - "tail_focus_quantile": 1.0, - "total_allocated": 1000000.0, - "uncertainty_aversion": 0.0, - "weighted_miscoverage": 0.134525, - "weighted_outcome": 0.15525, - "weighted_pd_effective": 0.17000000000000004, - "weighted_pd_point": 0.06948853099473384 - }, - { - "candidate_id": "point-pd", - "delta_cap_quantile": 1.0, - "endpoint_budget": 0.8880705762941006, - "expected_objective": 141162.2113059011, - "gamma": 0.0, - "gamma_cp": 0.7180705762941006, - "gamma_internalized": 0.0, - "gamma_residual": 0.7180705762941006, - "markov_loss_threshold": 1.2042983423109386, - "min_budget_utilization": 0.0, - "n_funded": 169, - "n_panel": 20695, - "objective_risk_mode": "point_pd_plus_aversion", - "pd_cap_slack_penalty": 0.0, - "period": "2017-12", - "policy_mode": "point_estimate", - "realized_return": 89732.35130590109, - "realized_risk_tolerance_excess": 0.015650000000000025, - "risk_tolerance": 0.17, - "role": "point_pd_matched_tau", - "screen_V_leq_sqrt_alpha": true, - "screen_risk_excess_leq_alpha": true, - "solver_status": "Optimal", - "tail_focus_quantile": 1.0, - "total_allocated": 1000000.0, - "uncertainty_aversion": 0.0, - "weighted_miscoverage": 0.058300000000000005, - "weighted_outcome": 0.18565000000000004, - "weighted_pd_effective": 0.16999999999999996, - "weighted_pd_point": 0.16999999999999996 - } - ], - "same_policy_selected": true - }, - "calibration_metadata": { - "audit_period": "2017-12", - "audit_rows": 20695, - "calibration_fit_rows": 142550, - "calibration_holdout_end": "2017-12-01", - "calibration_holdout_rows": 35638, - "calibration_holdout_start": "2017-11-01", - "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", - "outcomes_isolated_until_post_selection_audit": true, - "partition": "score_decile_mondrian", - "selection_period": "2017-11", - "selection_rows": 14943, - "target_alpha": 0.1, - "used_alpha": 0.095 - }, - "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", - "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml", - "config_sha256": "9df5445defe22044b235dcc0f21bc8f5b734d35c73f916a09e1fe60060f84cc0", - "contrasts": { - "2020+": { - "default_delta_vs_incumbent": -0.010000000000000009, - "default_delta_vs_point": 0.06687499999999999, - "return_cost_vs_point": 118939.59578289572, - "return_delta_vs_incumbent": 16526.81992213076, - "selected_markov_threshold": 0.580689837874186, - "selected_realized_return": 99689.53961659266, - "selected_weighted_outcome": 0.08377499999999999, - "threshold_delta_vs_point": -0.5969627812854879 - }, - "full_oot": { - "default_delta_vs_incumbent": 0.003500000000000003, - "default_delta_vs_point": -0.07902500000000001, - "return_cost_vs_point": 17041.554867740255, - "return_delta_vs_incumbent": 6388.080277992645, - "selected_markov_threshold": 0.5742788554403055, - "selected_realized_return": 179327.5851322598, - "selected_weighted_outcome": 0.03937500000000001, - "threshold_delta_vs_point": -0.663266349105067 - } - }, - "design": { - "alpha": 0.1, - "audit_period": "2017-12", - "budget": 1000000.0, - "combine_years_from": 2020, - "endpoint_budget_cap": 0.28, - "lgd": 0.45, - "max_concentration": 0.25, - "period_order": [ - "2018H1", - "2018H2", - "2019H1", - "2019H2", - "2020+" - ], - "selection_min_budget_utilization": 0.999, - "selection_period": "2017-11", - "selection_rule": "maximize expected point-PD objective on November 2017 under a deterministic endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; reserve December 2017 for an outcome-free selector stability replay and a post-selection decision audit" - }, - "endpoint_cap_stability": { - "cap_lower_inclusive": 0.25903604939435104, - "cap_upper_exclusive": 0.29049078888716334, - "declared_endpoint_budget_cap": 0.28, - "margin_to_lower_boundary": 0.020963950605648984, - "margin_to_upper_boundary": 0.010490788887163316, - "selected_candidate_id": "linear-005", - "selected_endpoint_budget": 0.25903604939435104 - }, - "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_oot_evaluation.csv", - "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", - "grid_size": 9, - "incumbent_policy": { - "candidate_id": "linear-006", - "delta_cap_quantile": 1.0, - "gamma": 0.75, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "oot_alignment": { - "alignment_mode": "id", - "candidate_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\test_fe.parquet", - "exact_alpha_grid_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet" - }, - "recipe": { - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "reference_target_alpha": 0.1, - "reference_used_alpha": 0.095 - }, - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", - "schema_version": "2026-07-09.7", - "selected_calibration_metrics": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "effective_pd_cap_slack": 0.0, - "endpoint_budget": 0.25903604939435104, - "expected_objective": 99387.12330098984, - "gamma": 0.5, - "gamma_cp": 0.17807209878870203, - "gamma_internalized": 0.08903604939435102, - "gamma_residual": 0.089036049394351, - "markov_loss_threshold": 0.575263815411189, - "min_budget_utilization": 0.0, - "n_funded": 162, - "n_panel": 14943, - "objective_risk_mode": "point_pd_plus_aversion", - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "solver_status": "Optimal", - "tail_focus_quantile": 1.0, - "total_allocated": 1000000.0, - "uncertainty_aversion": 0.0, - "weighted_pd_effective": 0.17, - "weighted_pd_point": 0.08096395060564901 - }, - "selected_policy": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "gamma": 0.5, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "selection_audit": { - "endpoint_budget_cap": 0.28, - "min_budget_utilization": 0.999, - "n_eligible": 5, - "n_total": 9, - "outcome_columns_used": 0, - "selected_candidate_id": "linear-005", - "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", - "statistical_assumption_columns_used": 0 - }, - "selector_columns": [ - "candidate_id", - "risk_tolerance", - "gamma", - "uncertainty_aversion", - "policy_mode", - "delta_cap_quantile", - "tail_focus_quantile", - "min_budget_utilization", - "pd_cap_slack_penalty", - "solver_status", - "objective_risk_mode", - "expected_objective", - "n_panel", - "n_funded", - "total_allocated", - "weighted_pd_point", - "weighted_pd_effective", - "gamma_cp", - "gamma_internalized", - "gamma_residual", - "endpoint_budget", - "markov_loss_threshold", - "effective_pd_cap_slack" - ], - "selector_forbidden_columns_present": [], - "selector_input_columns": [ - "id", - "loan_amnt", - "purpose", - "grade", - "term", - "verification_status", - "issue_d", - "_pd_point", - "_pd_low", - "_pd_high", - "_loan_amount", - "_int_rate" - ], - "source_commit": "e0daf55685988408bc68f5b65df0298174f1a516" -} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json deleted file mode 100644 index 08f6f0b..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "bootstrap_return_interval": { - "bootstrap_unit": "origination_month", - "n_draws": 5000, - "n_units": 31, - "p025": 163421.1394838545, - "p975": 193551.6460089551 - }, - "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", - "exact_alpha_reference_replay": { - "high_max_abs": 6.661338147750939e-16, - "low_max_abs": 3.3306690738754696e-16, - "pass": true, - "point_max_abs": 4.440892098500626e-16, - "tolerance": 1e-12 - }, - "full_oot": { - "Gamma_CP": 0.1761021788469351, - "Gamma_internalized": 0.0880510894234675, - "Gamma_residual": 0.0880510894234675, - "endpoint_budget": 0.2580510894234676, - "expected_objective": 168271.56287282018, - "markov_loss_threshold": 0.5742788554403055, - "markov_tail_probability_bound": 0.31622776601683794, - "n_candidates": 276869, - "n_funded": 308, - "observed_accounting_bound": 0.2949260894234676, - "realized_return": 179327.5851322598, - "total_allocated": 1000000.0, - "weighted_default_rate": 0.039375, - "weighted_miscoverage": 0.036875, - "weighted_pd_effective": 0.17, - "weighted_pd_point": 0.0819489105765324 - }, - "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", - "paper_tables": { - "alpha": [ - "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", - "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" - ], - "baseline": [ - "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", - "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" - ], - "bootstrap": [ - "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", - "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" - ], - "grade": [ - "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", - "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" - ], - "selector": [ - "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", - "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" - ], - "temporal": [ - "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", - "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" - ] - }, - "point_pd_contrast": { - "endpoint_budget": 0.9213174385285344, - "markov_loss_threshold": 1.2375452045453723, - "realized_return": 196369.14000000004, - "selected_default_reduction": 0.07902500000000001, - "selected_return_cost": 17041.554867740255, - "selected_return_cost_pct": 0.0867832637436832, - "selected_threshold_reduction": 0.6632663491050668, - "weighted_default_rate": 0.1184, - "weighted_miscoverage": 0.0419 - }, - "retired_active_claims": [ - "alpha01 intervals obtained by cross-family average-width scaling", - "8/8 approximate alpha-grid pass as a headline certificate", - "50,010-policy frontier as the active selector", - "0.345084 Markov threshold", - "capped_blended_uncertainty with delta_cap_quantile=0.975", - "OOT-outcome-selected portfolio hyperparameters", - "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" - ], - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", - "schema_version": "2026-07-09.7", - "selected_policy": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "gamma": 0.5, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "selection_protocol": { - "calibration_audit": { - "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", - "matched_point_pd": { - "endpoint_budget": 0.8880705762941006, - "n_funded": 169, - "observed_accounting_bound": 0.9463705762941006, - "realized_return": 89732.35130590109, - "weighted_default_rate": 0.18565000000000004, - "weighted_miscoverage": 0.058300000000000005 - }, - "more_conservative_policy": { - "endpoint_budget": 0.20350382300175537, - "n_funded": 191, - "observed_accounting_bound": 0.3380288230017554, - "realized_return": 38379.50087450525, - "weighted_default_rate": 0.15525, - "weighted_miscoverage": 0.134525 - }, - "n_candidates": 20695, - "outcome_free_selected_candidate_id": "linear-005", - "period": "2017-12", - "same_policy_selected": true, - "selected_policy": { - "endpoint_budget": 0.2620820323280323, - "n_funded": 193, - "observed_accounting_bound": 0.3870070323280323, - "realized_return": 53313.04791145165, - "weighted_default_rate": 0.14565, - "weighted_miscoverage": 0.12492500000000001 - } - }, - "calibration_metadata": { - "audit_period": "2017-12", - "audit_rows": 20695, - "calibration_fit_rows": 142550, - "calibration_holdout_end": "2017-12-01", - "calibration_holdout_rows": 35638, - "calibration_holdout_start": "2017-11-01", - "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", - "outcomes_isolated_until_post_selection_audit": true, - "partition": "score_decile_mondrian", - "selection_period": "2017-11", - "selection_rows": 14943, - "target_alpha": 0.1, - "used_alpha": 0.095 - }, - "endpoint_budget_cap": 0.28, - "endpoint_cap_stability": { - "cap_lower_inclusive": 0.25903604939435104, - "cap_upper_exclusive": 0.29049078888716334, - "declared_endpoint_budget_cap": 0.28, - "margin_to_lower_boundary": 0.020963950605648984, - "margin_to_upper_boundary": 0.010490788887163316, - "selected_candidate_id": "linear-005", - "selected_endpoint_budget": 0.25903604939435104 - }, - "min_budget_utilization": 0.999, - "n_eligible": 5, - "n_total": 9, - "outcome_columns_used": 0, - "selected_candidate_id": "linear-005", - "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", - "selector_forbidden_columns_present": [], - "selector_input_columns": [ - "id", - "loan_amnt", - "purpose", - "grade", - "term", - "verification_status", - "issue_d", - "_pd_point", - "_pd_low", - "_pd_high", - "_loan_amount", - "_int_rate" - ], - "statistical_assumption_columns_used": 0 - }, - "status": "active_ijds_policy" -} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json deleted file mode 100644 index f388671..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", - "calibration_metadata": { - "calibration_fit_rows": 142550, - "calibration_selection_end": "2017-12-01", - "calibration_selection_rows": 35638, - "calibration_selection_start": "2017-11-01", - "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", - "partition": "score_decile_mondrian", - "target_alpha": 0.1, - "used_alpha": 0.095 - }, - "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", - "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_simple90_v6.yaml", - "config_sha256": "cddf7daf67c91b7715e676b8a9e8dcba08e8d3a9499b8fe7cdffe51cfe690646", - "contrasts": { - "2020+": { - "default_delta_vs_incumbent": -0.010000000000000009, - "default_delta_vs_point": 0.06687499999999999, - "return_cost_vs_point": 118939.59578289572, - "return_delta_vs_incumbent": 16526.81992213076, - "selected_markov_threshold": 0.580689837874186, - "selected_realized_return": 99689.53961659266, - "selected_weighted_outcome": 0.08377499999999999, - "threshold_delta_vs_point": -0.5969627812854879 - }, - "full_oot": { - "default_delta_vs_incumbent": 0.003500000000000003, - "default_delta_vs_point": -0.07902500000000001, - "return_cost_vs_point": 17041.554867740255, - "return_delta_vs_incumbent": 6388.080277992645, - "selected_markov_threshold": 0.5742788554403055, - "selected_realized_return": 179327.5851322598, - "selected_weighted_outcome": 0.03937500000000001, - "threshold_delta_vs_point": -0.663266349105067 - } - }, - "design": { - "alpha": 0.1, - "budget": 1000000.0, - "combine_years_from": 2020, - "lgd": 0.45, - "markov_threshold_cap": 0.6, - "max_concentration": 0.25, - "period_order": [ - "2018H1", - "2018H2", - "2019H1", - "2019H2", - "2020+" - ], - "selection_min_budget_utilization": 0.999, - "selection_rule": "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" - }, - "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_oot_evaluation.csv", - "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", - "grid_size": 9, - "incumbent_policy": { - "candidate_id": "linear-006", - "delta_cap_quantile": 1.0, - "gamma": 0.75, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "recipe": { - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "reference_target_alpha": 0.1, - "reference_used_alpha": 0.095 - }, - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", - "schema_version": "2026-07-09.6", - "selected_calibration_metrics": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "effective_pd_cap_slack": 0.0, - "endpoint_budget": 0.26104699571114653, - "expected_objective": 110346.16233639097, - "gamma": 0.5, - "gamma_cp": 0.18209399142229304, - "gamma_internalized": 0.09104699571114652, - "gamma_residual": 0.09104699571114651, - "markov_loss_threshold": 0.5772747617279845, - "min_budget_utilization": 0.0, - "n_funded": 211, - "n_panel": 35638, - "objective_risk_mode": "point_pd_plus_aversion", - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "solver_status": "Optimal", - "tail_focus_quantile": 1.0, - "total_allocated": 1000000.0, - "uncertainty_aversion": 0.0, - "weighted_pd_effective": 0.17, - "weighted_pd_point": 0.07895300428885349 - }, - "selected_policy": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "gamma": 0.5, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "selection_audit": { - "markov_threshold_cap": 0.6, - "min_budget_utilization": 0.999, - "n_eligible": 5, - "n_total": 9, - "outcome_columns_used": 0, - "selected_candidate_id": "linear-005", - "selection_rule": "max_expected_objective_under_ex_ante_screen" - }, - "selector_columns": [ - "candidate_id", - "risk_tolerance", - "gamma", - "uncertainty_aversion", - "policy_mode", - "delta_cap_quantile", - "tail_focus_quantile", - "min_budget_utilization", - "pd_cap_slack_penalty", - "solver_status", - "objective_risk_mode", - "expected_objective", - "n_panel", - "n_funded", - "total_allocated", - "weighted_pd_point", - "weighted_pd_effective", - "gamma_cp", - "gamma_internalized", - "gamma_residual", - "endpoint_budget", - "markov_loss_threshold", - "effective_pd_cap_slack" - ], - "selector_forbidden_columns_present": [], - "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" -} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json deleted file mode 100644 index 9d3491b..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "bootstrap_return_interval": { - "n_draws": 5000, - "p025": 162706.17200230644, - "p975": 193924.73990027257 - }, - "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", - "exact_alpha_reference_replay": { - "high_max_abs": 6.661338147750939e-16, - "low_max_abs": 3.3306690738754696e-16, - "pass": true, - "point_max_abs": 4.440892098500626e-16, - "tolerance": 1e-12 - }, - "full_oot": { - "Gamma_CP": 0.1761021788469351, - "Gamma_internalized": 0.0880510894234675, - "Gamma_residual": 0.0880510894234675, - "endpoint_budget": 0.2580510894234676, - "expected_objective": 168271.56287282018, - "markov_loss_threshold": 0.5742788554403055, - "markov_tail_probability_bound": 0.31622776601683794, - "n_candidates": 276869, - "n_funded": 308, - "observed_accounting_bound": 0.2949260894234676, - "realized_return": 179327.5851322598, - "total_allocated": 1000000.0, - "weighted_default_rate": 0.039375, - "weighted_miscoverage": 0.036875, - "weighted_pd_effective": 0.17, - "weighted_pd_point": 0.0819489105765324 - }, - "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", - "paper_tables": { - "alpha": [ - "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", - "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" - ], - "baseline": [ - "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", - "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" - ], - "bootstrap": [ - "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", - "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" - ], - "grade": [ - "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", - "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" - ], - "selector": [ - "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", - "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" - ], - "temporal": [ - "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", - "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" - ] - }, - "point_pd_contrast": { - "endpoint_budget": 0.9213174385285344, - "markov_loss_threshold": 1.2375452045453723, - "realized_return": 196369.14000000004, - "selected_default_reduction": 0.07902500000000001, - "selected_return_cost": 17041.554867740255, - "selected_return_cost_pct": 0.0867832637436832, - "selected_threshold_reduction": 0.6632663491050668, - "weighted_default_rate": 0.1184, - "weighted_miscoverage": 0.0419 - }, - "retired_active_claims": [ - "alpha01 intervals obtained by cross-family average-width scaling", - "8/8 approximate alpha-grid pass as a headline certificate", - "50,010-policy frontier as the active selector", - "0.345084 Markov threshold", - "capped_blended_uncertainty with delta_cap_quantile=0.975", - "OOT-outcome-selected portfolio hyperparameters", - "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" - ], - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", - "schema_version": "2026-07-09.6", - "selected_policy": { - "candidate_id": "linear-005", - "delta_cap_quantile": 1.0, - "gamma": 0.5, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "risk_tolerance": 0.17, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0 - }, - "selection_protocol": { - "calibration_metadata": { - "calibration_fit_rows": 142550, - "calibration_selection_end": "2017-12-01", - "calibration_selection_rows": 35638, - "calibration_selection_start": "2017-11-01", - "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", - "partition": "score_decile_mondrian", - "target_alpha": 0.1, - "used_alpha": 0.095 - }, - "markov_threshold_cap": 0.6, - "min_budget_utilization": 0.999, - "n_eligible": 5, - "n_total": 9, - "outcome_columns_used": 0, - "selected_candidate_id": "linear-005", - "selection_rule": "max_expected_objective_under_ex_ante_screen", - "selector_forbidden_columns_present": [] - }, - "status": "active_ijds_policy" -} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json deleted file mode 100644 index 1f9ba3c..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "generated_at_utc": "2026-07-02T06:01:34.539877+00:00", - "source_run_tags": [ - "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", - "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", - "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" - ], - "selection_rule": { - "eligible": "all-alpha pass and nonnegative return_floor_surplus", - "dedupe_key": "semantic_policy_key", - "dedupe_semantics": "duplicate semantic policies across refinement runs have identical metrics; one representative row is retained for the consolidated table", - "body_selection": "highest realized return among eligible finite-grid policies with Markov_cap <= 0.35; falls back to the legacy balanced normalized return/bound/V score only if no eligible policy exists under that declared cap", - "caps": [ - 0.32, - 0.33, - 0.335, - 0.34, - 0.345, - 0.35, - 0.36, - 0.45, - 0.5 - ], - "role_semantics": "finite-grid frontier roles, not continuous optima" - }, - "counts": { - "raw_rows": 51678, - "deduped_semantic_policies": 50010, - "duplicate_rows_removed": 1668, - "eligible_all_alpha_return_floor_policies": 27508, - "nonpass_or_below_floor_policies": 22502 - }, - "by_run": [ - { - "run_label": "expanded", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", - "rows": 3258, - "all_alpha_passers": 3071, - "all_alpha_pass_rate": 0.942602824, - "best_return": 223346.5525, - "min_markov_cap": 0.335784675 - }, - { - "run_label": "micro", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", - "rows": 2949, - "all_alpha_passers": 2949, - "all_alpha_pass_rate": 1.0, - "best_return": 223369.907874, - "min_markov_cap": 0.3287988 - }, - { - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "rows": 4407, - "all_alpha_passers": 4406, - "all_alpha_pass_rate": 0.999773088, - "best_return": 223458.135875, - "min_markov_cap": 0.3167127 - }, - { - "run_label": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", - "rows": 1653, - "all_alpha_passers": 1653, - "all_alpha_pass_rate": 1.0, - "best_return": 213593.775, - "min_markov_cap": 0.29836925 - }, - { - "run_label": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", - "rows": 2343, - "all_alpha_passers": 2343, - "all_alpha_pass_rate": 1.0, - "best_return": 213593.775, - "min_markov_cap": 0.28483896 - }, - { - "run_label": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "rows": 37068, - "all_alpha_passers": 37068, - "all_alpha_pass_rate": 1.0, - "best_return": 213593.775, - "min_markov_cap": 0.26767181 - } - ], - "rows": [ - { - "role": "minimum Markov-cap endpoint", - "run_label": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "local_candidate_id": 10661, - "family": "claim_bound_terminal_ultra_low_cap", - "anchor_rank": 219, - "source_reason": "terminal_cap_threshold_search", - "risk_tolerance": 0.16825, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.95, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.6, - "return": 170467.268819, - "return_floor_surplus": 2.728819, - "Gamma_CP": 0.095719, - "V": 0.031875, - "endpoint_budget_upper": 0.17303595, - "Markov_cap": 0.27303595, - "alpha_pass": "8/8", - "n_funded_mean": 311.125, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.95,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" - }, - { - "role": "body/default balanced return-bound point", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 131, - "family": "claim_micro_ext_body_cap345", - "anchor_rank": 219, - "source_reason": "candidate37_205_body_cap345_extension", - "risk_tolerance": 0.1715, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5475, - "delta_cap_quantile": 0.975, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05, - "return": 184832.475845, - "return_floor_surplus": 14367.935845, - "Gamma_CP": 0.162616, - "V": 0.03535, - "endpoint_budget_upper": 0.24508374, - "Markov_cap": 0.34508374, - "alpha_pass": "8/8", - "n_funded_mean": 320.5, - "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" - }, - { - "role": "lowest realized V return-bound point", - "run_label": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", - "local_candidate_id": 5504, - "family": "claim_bound_terminal_ultra_low_cap", - "anchor_rank": 219, - "source_reason": "terminal_cap_threshold_search", - "risk_tolerance": 0.1675, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.925, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.6, - "return": 171006.195983, - "return_floor_surplus": 541.655983, - "Gamma_CP": 0.09719, - "V": 0.031875, - "endpoint_budget_upper": 0.17478925, - "Markov_cap": 0.27478925, - "alpha_pass": "8/8", - "n_funded_mean": 311.125, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.925,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1675,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" - }, - { - "role": "max-return economic endpoint", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 4041, - "family": "claim_micro_ext_economic_endpoint", - "anchor_rank": 96, - "source_reason": "candidate2122_economic_endpoint_extension", - "risk_tolerance": 0.156875, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.445, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 0.925, - "uncertainty_aversion": 0.1375, - "return": 223458.135875, - "return_floor_surplus": 52993.595875, - "Gamma_CP": 0.457438, - "V": 0.069575, - "endpoint_budget_upper": 0.41075309, - "Markov_cap": 0.51075309, - "alpha_pass": "8/8", - "n_funded_mean": 239.625, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.445,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.156875,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.925,\"uncertainty_aversion\":0.1375}" - }, - { - "role": "highest return under cap<=0.32", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 2021, - "family": "claim_micro_ext_bound_tight", - "anchor_rank": 219, - "source_reason": "candidate949_bound_tight_extension", - "risk_tolerance": 0.1705, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.645, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.2375, - "return": 179436.119445, - "return_floor_surplus": 8971.579445, - "Gamma_CP": 0.139182, - "V": 0.035875, - "endpoint_budget_upper": 0.21990961, - "Markov_cap": 0.31990961, - "alpha_pass": "8/8", - "n_funded_mean": 310.125, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.645,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1705,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.2375}" - }, - { - "role": "highest return under cap<=0.33", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 1715, - "family": "claim_micro_ext_bound_tight", - "anchor_rank": 219, - "source_reason": "candidate949_bound_tight_extension", - "risk_tolerance": 0.17025, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.615, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.2375, - "return": 180905.961232, - "return_floor_surplus": 10441.421232, - "Gamma_CP": 0.144407, - "V": 0.035875, - "endpoint_budget_upper": 0.225846695, - "Markov_cap": 0.325846695, - "alpha_pass": "8/8", - "n_funded_mean": 311.5, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.615,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17025,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.2375}" - }, - { - "role": "highest return under cap<=0.335", - "run_label": "micro", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", - "local_candidate_id": 860, - "family": "claim_micro_bound_tight", - "anchor_rank": 219, - "source_reason": "candidate1206_tight_cap_micro", - "risk_tolerance": 0.17, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.575, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.225, - "return": 181137.903294, - "return_floor_surplus": 10673.363294, - "Gamma_CP": 0.15242, - "V": 0.038375, - "endpoint_budget_upper": 0.2347785, - "Markov_cap": 0.3347785, - "alpha_pass": "8/8", - "n_funded_mean": 314.375, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.575,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.225}" - }, - { - "role": "highest return under cap<=0.34", - "run_label": "expanded", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", - "local_candidate_id": 2037, - "family": "bound_claim_refined_local", - "anchor_rank": 219, - "source_reason": "candidate462_466_return_bound_ridge", - "risk_tolerance": 0.173, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.575, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.175, - "return": 182209.77909, - "return_floor_surplus": 11745.23909, - "Gamma_CP": 0.156892, - "V": 0.038375, - "endpoint_budget_upper": 0.2396791, - "Markov_cap": 0.3396791, - "alpha_pass": "8/8", - "n_funded_mean": 316.0, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.575,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.173,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.175}" - }, - { - "role": "highest return under cap<=0.345", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 512, - "family": "claim_micro_ext_body_cap345", - "anchor_rank": 219, - "source_reason": "candidate37_205_body_cap345_extension", - "risk_tolerance": 0.17225, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5525, - "delta_cap_quantile": 0.975, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0375, - "return": 184800.413581, - "return_floor_surplus": 14335.873581, - "Gamma_CP": 0.162562, - "V": 0.03535, - "endpoint_budget_upper": 0.244996495, - "Markov_cap": 0.344996495, - "alpha_pass": "8/8", - "n_funded_mean": 321.0, - "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17225,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0375}" - }, - { - "role": "highest return under cap<=0.35", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 131, - "family": "claim_micro_ext_body_cap345", - "anchor_rank": 219, - "source_reason": "candidate37_205_body_cap345_extension", - "risk_tolerance": 0.1715, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5475, - "delta_cap_quantile": 0.975, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05, - "return": 184832.475845, - "return_floor_surplus": 14367.935845, - "Gamma_CP": 0.162616, - "V": 0.03535, - "endpoint_budget_upper": 0.24508374, - "Markov_cap": 0.34508374, - "alpha_pass": "8/8", - "n_funded_mean": 320.5, - "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" - }, - { - "role": "highest return under cap<=0.36", - "run_label": "micro_ext", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "local_candidate_id": 3212, - "family": "claim_micro_ext_cap036_return", - "anchor_rank": 219, - "source_reason": "candidate1975_cap036_return_extension", - "risk_tolerance": 0.17575, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.525, - "delta_cap_quantile": 0.95, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.075, - "return": 186050.727749, - "return_floor_surplus": 15586.187749, - "Gamma_CP": 0.1746, - "V": 0.03775, - "endpoint_budget_upper": 0.258685, - "Markov_cap": 0.358685, - "alpha_pass": "8/8", - "n_funded_mean": 318.75, - "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17575,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.075}" - }, - { - "role": "highest return under cap<=0.45", - "run_label": "expanded", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", - "local_candidate_id": 979, - "family": "bound_efficient_local", - "anchor_rank": 223, - "source_reason": "rank219_rank223_bound_frontier", - "risk_tolerance": 0.185, - "policy_mode": "blended_uncertainty", - "gamma": 0.35, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.15, - "return": 198693.277519, - "return_floor_surplus": 28228.737519, - "Gamma_CP": 0.252323, - "V": 0.0456, - "endpoint_budget_upper": 0.34900995, - "Markov_cap": 0.44900995, - "alpha_pass": "8/8", - "n_funded_mean": 310.875, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.35,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.185,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.15}" - }, - { - "role": "highest return under cap<=0.5", - "run_label": "micro", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", - "local_candidate_id": 2840, - "family": "claim_micro_economic_endpoint", - "anchor_rank": 96, - "source_reason": "candidate2777_2857_economic_endpoint_micro", - "risk_tolerance": 0.15825, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 0.95, - "uncertainty_aversion": 0.125, - "return": 222558.7025, - "return_floor_surplus": 52094.1625, - "Gamma_CP": 0.459089, - "V": 0.071075, - "endpoint_budget_upper": 0.3877945, - "Markov_cap": 0.4877945, - "alpha_pass": "8/8", - "n_funded_mean": 238.375, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.5,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.15825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.95,\"uncertainty_aversion\":0.125}" - } - ] -} \ No newline at end of file diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json deleted file mode 100644 index ef2eb7a..0000000 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json +++ /dev/null @@ -1,244 +0,0 @@ -{ - "alpha_mapping": "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe", - "alpha_summaries": [ - { - "avg_width": 0.9882149910656284, - "coverage_gap": 0.006720470691915725, - "empirical_coverage": 0.9967204706919157, - "group_quantiles": { - "score_q00": 4.620892417510161, - "score_q01": 2.9770143672127003, - "score_q02": 2.2280905155147157, - "score_q03": 1.7679793270133157, - "score_q04": 1.379561550348839 - }, - "high_endpoint_at_one_rate": 0.9354243342519386, - "high_endpoint_mean": 0.9883685381679577, - "high_endpoint_min": 0.28857911767290634, - "high_endpoint_p01": 0.6715297528015821, - "high_endpoint_p10": 1.0, - "median_width": 1.0, - "min_grade_coverage": 0.9722703639514731, - "min_partition_coverage": 0.989628743645835, - "partition_count": 5, - "target_alpha": 0.01, - "target_coverage": 0.99, - "used_alpha": 0.0095 - }, - { - "avg_width": 0.9697789837688854, - "coverage_gap": 0.01847830562468178, - "empirical_coverage": 0.9884783056246818, - "group_quantiles": { - "score_q00": 3.642519971959776, - "score_q01": 2.8072631501367455, - "score_q02": 2.1737923639215784, - "score_q03": 1.7387907144323151, - "score_q04": 1.3580168082764283 - }, - "high_endpoint_at_one_rate": 0.8323141991338864, - "high_endpoint_mean": 0.9699657056860221, - "high_endpoint_min": 0.22825533153683755, - "high_endpoint_p01": 0.5334804414885259, - "high_endpoint_p10": 0.9018799702618802, - "median_width": 1.0, - "min_grade_coverage": 0.9664933564413634, - "min_partition_coverage": 0.9697990273996318, - "partition_count": 5, - "target_alpha": 0.03, - "target_coverage": 0.97, - "used_alpha": 0.028499999999999998 - }, - { - "avg_width": 0.9535799768723978, - "coverage_gap": 0.022824693266490725, - "empirical_coverage": 0.9728246932664907, - "group_quantiles": { - "score_q00": 3.233734520299044, - "score_q01": 2.6752015014256125, - "score_q02": 2.1212263259384927, - "score_q03": 1.7077473640552945, - "score_q04": 1.3388102247632694 - }, - "high_endpoint_at_one_rate": 0.7181555175913519, - "high_endpoint_mean": 0.953801858012357, - "high_endpoint_min": 0.20305073151598757, - "high_endpoint_p01": 0.47580041346523627, - "high_endpoint_p10": 0.806835962194862, - "median_width": 1.0, - "min_grade_coverage": 0.9525750124329999, - "min_partition_coverage": 0.9505830880848586, - "partition_count": 5, - "target_alpha": 0.05, - "target_coverage": 0.95, - "used_alpha": 0.0475 - }, - { - "avg_width": 0.795666199875876, - "coverage_gap": 0.025271265472118665, - "empirical_coverage": 0.9552712654721186, - "group_quantiles": { - "score_q00": 0.32438420480886127, - "score_q01": 2.567436984095836, - "score_q02": 2.0727963588460026, - "score_q03": 1.6764580225891925, - "score_q04": 1.3186561259152434 - }, - "high_endpoint_at_one_rate": 0.6135500904760013, - "high_endpoint_mean": 0.7959333620592705, - "high_endpoint_min": 0.023668097607277484, - "high_endpoint_p01": 0.06528823147517372, - "high_endpoint_p10": 0.13040211717455735, - "median_width": 1.0, - "min_grade_coverage": 0.938442835829143, - "min_partition_coverage": 0.9347822665680426, - "partition_count": 5, - "target_alpha": 0.07, - "target_coverage": 0.9299999999999999, - "used_alpha": 0.0665 - }, - { - "avg_width": 0.7888790790793794, - "coverage_gap": 0.03483560817570763, - "empirical_coverage": 0.9348356081757077, - "group_quantiles": { - "score_q00": 0.3201476773467629, - "score_q01": 2.4331722444504513, - "score_q02": 2.007709882293012, - "score_q03": 1.633544476383572, - "score_q04": 1.2899294761760491 - }, - "high_endpoint_at_one_rate": 0.5178730735474177, - "high_endpoint_mean": 0.7892193805747533, - "high_endpoint_min": 0.023406884837288408, - "high_endpoint_p01": 0.06469045329656556, - "high_endpoint_p10": 0.12941711012832322, - "median_width": 1.0, - "min_grade_coverage": 0.9267968822615554, - "min_partition_coverage": 0.9263099219620958, - "partition_count": 5, - "target_alpha": 0.1, - "target_coverage": 0.9, - "used_alpha": 0.095 - }, - { - "avg_width": 0.7831737841625059, - "coverage_gap": 0.038264594447193434, - "empirical_coverage": 0.9182645944471934, - "group_quantiles": { - "score_q00": 0.31739006163700956, - "score_q01": 2.3534507550187485, - "score_q02": 1.9655367760210574, - "score_q03": 1.6066047959404717, - "score_q04": 1.269964951004726 - }, - "high_endpoint_at_one_rate": 0.4545109781160043, - "high_endpoint_mean": 0.7835760369425744, - "high_endpoint_min": 0.023236857746810564, - "high_endpoint_p01": 0.06430135101621326, - "high_endpoint_p10": 0.12877595509528877, - "median_width": 0.9867966540367542, - "min_grade_coverage": 0.9057375416605808, - "min_partition_coverage": 0.906874767744333, - "partition_count": 5, - "target_alpha": 0.12, - "target_coverage": 0.88, - "used_alpha": 0.11399999999999999 - }, - { - "avg_width": 0.6461762449744832, - "coverage_gap": 0.03609775742318566, - "empirical_coverage": 0.8860977574231856, - "group_quantiles": { - "score_q00": 0.31337107987851737, - "score_q01": 0.4420899636645897, - "score_q02": 1.9079870902279796, - "score_q03": 1.5675398154360425, - "score_q04": 1.2412696239135141 - }, - "high_endpoint_at_one_rate": 0.33485872380078663, - "high_endpoint_mean": 0.6466929852254412, - "high_endpoint_min": 0.02298905825369661, - "high_endpoint_p01": 0.06373426874967675, - "high_endpoint_p10": 0.1278415281651632, - "median_width": 0.94425532567159, - "min_grade_coverage": 0.8631010078209027, - "min_partition_coverage": 0.8635623982976592, - "partition_count": 5, - "target_alpha": 0.15, - "target_coverage": 0.85, - "used_alpha": 0.1425 - }, - { - "avg_width": 0.6365853175970958, - "coverage_gap": 0.049380031711748096, - "empirical_coverage": 0.8493800317117481, - "group_quantiles": { - "score_q00": 0.30656504576058813, - "score_q01": 0.43573433937656825, - "score_q02": 1.8175729021078393, - "score_q03": 1.5052283270500615, - "score_q04": 1.194771966347316 - }, - "high_endpoint_at_one_rate": 0.24466805601204902, - "high_endpoint_mean": 0.6373641323567814, - "high_endpoint_min": 0.022569416686283303, - "high_endpoint_p01": 0.06277393066265638, - "high_endpoint_p10": 0.1262591020808178, - "median_width": 0.9128005695874017, - "min_grade_coverage": 0.828006532910199, - "min_partition_coverage": 0.8293756967670011, - "partition_count": 5, - "target_alpha": 0.2, - "target_coverage": 0.8, - "used_alpha": 0.19 - } - ], - "claim_boundary": "Recomputes finite-sample Mondrian quantiles for every declared alpha while freezing the selected partition, calibration fit/holdout split, score scale, and holdout-learned widening factors. It is a retrospective OOT audit, not a post-selection or live-deployment guarantee.", - "config_path": "configs\\experiments\\champion_reopen_ijds_exact_alpha_grid_v1.yaml", - "config_sha256": "c2d2af0d1c0e0f83ef8612888f9b12d423063a601f04c8c51c9680e92327ef3f", - "generated_at_utc": "2026-07-10T02:32:16.941448+00:00", - "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet", - "grid_rows": 276869, - "recipe": { - "calibration_fraction": 0.75, - "fallback_mode": "score_only", - "global_rebalance_factor": 1.0, - "group_multipliers": { - "score_q00": 1.02, - "score_q01": 1.05, - "score_q02": 1.05, - "score_q03": 1.02, - "score_q04": 1.02 - }, - "min_group_size": 100, - "n_score_bins": 5, - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "reference_target_alpha": 0.1, - "reference_used_alpha": 0.095, - "score_scale_family": "bernoulli_sqrt", - "temporal_segment_freq": "Q", - "temporal_segment_multipliers": {}, - "tuning_holdout_ratio": 0.2, - "tuning_random_state": 42 - }, - "reference_replay": { - "high_max_abs": 6.661338147750939e-16, - "low_max_abs": 3.3306690738754696e-16, - "pass": true, - "point_max_abs": 4.440892098500626e-16, - "tolerance": 1e-12 - }, - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1", - "schema_version": "2026-07-09.1", - "source": { - "conformal_intervals_path": "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "conformal_intervals_sha256": "3784cc68a5b72cf290c9678e6cd5d30f7b794dac4c1eedaf393fe9f72cc0c3a1", - "conformal_namespace": "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1", - "conformal_results_path": "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl", - "conformal_results_sha256": "cb8db92908739c37a11f373b9802f6b74831d8d9c26c4824dae586423d3cb07b", - "upstream_canonical_run_tag": "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" - }, - "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" -} diff --git a/models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc b/models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc new file mode 100644 index 0000000..4f287e3 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: f684151ac1ba5c967dbddb164501b6e1.dir + size: 6903 + nfiles: 2 + hash: md5 + path: ijds-allocation-granularity-sensitivity-2026-07-16-v3 diff --git a/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc new file mode 100644 index 0000000..c9dc3d9 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 367789f42b6e31a286e95ad2d019ced7.dir + size: 936375 + nfiles: 6 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-12-v1 diff --git a/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc new file mode 100644 index 0000000..311594c --- /dev/null +++ b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 437a94ba8a8376d609cae08b7161a839.dir + size: 40858 + nfiles: 3 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-14-v3 diff --git a/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc new file mode 100644 index 0000000..70af4a0 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 3e4af46a58d37e9687bee904b4f4ead7.dir + size: 82440 + nfiles: 3 + hash: md5 + path: ijds-binary-geometry-frontier-v4-2026-07-15-v5 diff --git a/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc new file mode 100644 index 0000000..ef22a37 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 68eb29c042c46be7d1ada5cdb74d004f.dir + size: 2024800 + nfiles: 12 + hash: md5 + path: ijds-credit-risk-controls-2026-07-13-v1b diff --git a/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc new file mode 100644 index 0000000..f7c28da --- /dev/null +++ b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 89b70c6492dc9f280e94e2a3507bdaae.dir + size: 13299 + nfiles: 2 + hash: md5 + path: ijds-credit-risk-controls-2026-07-14-v3 diff --git a/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc new file mode 100644 index 0000000..9bc47b3 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 8c29c1f564c64b6487ce21a34b2369be.dir + size: 24185 + nfiles: 2 + hash: md5 + path: ijds-credit-risk-controls-2026-07-15-v5 diff --git a/models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc b/models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc new file mode 100644 index 0000000..6d319d7 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1.dvc @@ -0,0 +1,6 @@ +outs: +- md5: d2de3251b35cfb12892e234986922768.dir + size: 10692 + nfiles: 1 + hash: md5 + path: ijds-endpoint-availability-sensitivity-2026-07-14-v1 diff --git a/models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc b/models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc new file mode 100644 index 0000000..4996717 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 826fc6d03e351c87ac16de36a34ef952.dir + size: 52356 + nfiles: 3 + hash: md5 + path: ijds-fit-label-completion-sensitivity-2026-07-16-v2 diff --git a/models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc b/models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc new file mode 100644 index 0000000..90fa869 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 5ecb3b0e22f928f534f582a4dc7f80ed.dir + size: 1755375 + nfiles: 8 + hash: md5 + path: ijds-missingness-sensitivity-2026-07-15-v3 diff --git a/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc new file mode 100644 index 0000000..68db820 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 10261f986821821538d918def80aaa02.dir + size: 18792 + nfiles: 3 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-13-v1c diff --git a/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc new file mode 100644 index 0000000..d90eff4 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-14-v3.dvc @@ -0,0 +1,6 @@ +outs: +- md5: ca8d64d0aad66a68f7f2617031231aab.dir + size: 20386 + nfiles: 3 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-14-v3 diff --git a/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc new file mode 100644 index 0000000..dbd6178 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 10744b717e5b3c3b4baa3d2f8118b317.dir + size: 32042 + nfiles: 3 + hash: md5 + path: ijds-normalized-objective-frontier-2026-07-15-v5 diff --git a/models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc b/models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc new file mode 100644 index 0000000..f3bc6e9 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6.dvc @@ -0,0 +1,6 @@ +outs: +- md5: fcdac67d8c92182b0c392ae94d9d7bfe.dir + size: 220552 + nfiles: 3 + hash: md5 + path: ijds-portfolio-structure-sensitivity-2026-07-15-v6 diff --git a/models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc b/models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc new file mode 100644 index 0000000..4e36ce3 --- /dev/null +++ b/models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 6d9a95ab8cefc9e52a6898c0833a944e.dir + size: 45770 + nfiles: 3 + hash: md5 + path: ijds-rolling-origin-2017-2026-07-15-v4 diff --git a/models/fairlearn_fairness_status.json b/models/fairlearn_fairness_status.json deleted file mode 100644 index 63e3d88..0000000 --- a/models/fairlearn_fairness_status.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "primary_status_path": "models/fairness_audit_status.json", - "group_metrics_path": "data/processed/fairlearn_group_metrics.parquet", - "n_attributes": 6, - "attributes": [ - { - "attribute": "home_ownership", - "demographic_parity_difference": 0.008742234380824332, - "equalized_odds_difference": 0.022086796811760956, - "dpd_ci_low": 0.00953831531946169, - "dpd_ci_high": 0.01338160527857427, - "eo_ci_low": 0.01675560254403783, - "eo_ci_high": 0.029423131504257353 - }, - { - "attribute": "annual_inc_quartile", - "demographic_parity_difference": 0.008400704557627536, - "equalized_odds_difference": 0.022249134036549156, - "dpd_ci_low": 0.006981762248958292, - "dpd_ci_high": 0.01345159921475334, - "eo_ci_low": 0.019293967031339274, - "eo_ci_high": 0.04380429053341763 - }, - { - "attribute": "verification_status", - "demographic_parity_difference": 0.01152366673334193, - "equalized_odds_difference": 0.024771462501377073, - "dpd_ci_low": 0.008401353349679794, - "dpd_ci_high": 0.013176169941118563, - "eo_ci_low": 0.011089250099697393, - "eo_ci_high": 0.03205558241323281 - }, - { - "attribute": "home_ownership__x__annual_inc_quartile", - "demographic_parity_difference": 0.014363846487668086, - "equalized_odds_difference": 0.03637090327737813, - "dpd_ci_low": 0.011677299308878281, - "dpd_ci_high": 0.017165245649591144, - "eo_ci_low": 0.0187982195845697, - "eo_ci_high": 0.04479807513049959 - }, - { - "attribute": "home_ownership__x__verification_status", - "demographic_parity_difference": 0.0343875474383778, - "equalized_odds_difference": 0.0636573811298401, - "dpd_ci_low": 0.02823498181736677, - "dpd_ci_high": 0.045446982772927906, - "eo_ci_low": 0.03538279343935319, - "eo_ci_high": 0.08757241988805607 - }, - { - "attribute": "annual_inc_quartile__x__verification_status", - "demographic_parity_difference": 0.02054232397484057, - "equalized_odds_difference": 0.04371873973202878, - "dpd_ci_low": 0.015066526101497746, - "dpd_ci_high": 0.027925765159232813, - "eo_ci_low": 0.02860218728639783, - "eo_ci_high": 0.07252232082281743 - } - ], - "bootstrap_samples": 10, - "bootstrap_rows_used": 10000, - "bootstrap_max_rows": 10000, - "prediction_threshold": 0.35, - "outcome_mode": "approval", - "schema_version": "2026-03-06.1-fairlearn", - "generated_at_utc": "2026-04-07T17:42:54.658367+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" -} diff --git a/models/fairness_audit_status.json b/models/fairness_audit_status.json deleted file mode 100644 index ad8e849..0000000 --- a/models/fairness_audit_status.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "overall_pass": true, - "n_attributes": 6, - "n_base_attributes": 3, - "n_intersectional_attributes": 3, - "n_passed": 6, - "attributes": [ - { - "attribute": "home_ownership", - "dpd": 0.008742234380824332, - "eo_gap": 0.022086796811760956, - "dir": 0.9912481257421258, - "tpr_gap": 0.003461823899529315, - "fpr_gap": 0.022086796811760956, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "annual_inc_quartile", - "dpd": 0.008400704557627536, - "eo_gap": 0.022249134036549156, - "dir": 0.9915953879107448, - "tpr_gap": 0.003462939364403339, - "fpr_gap": 0.022249134036549156, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "verification_status", - "dpd": 0.01152366673334193, - "eo_gap": 0.024771462501377073, - "dir": 0.9884514222176747, - "tpr_gap": 0.00523918531670553, - "fpr_gap": 0.024771462501377073, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "home_ownership__x__annual_inc_quartile", - "dpd": 0.014363846487668086, - "eo_gap": 0.03637090327737813, - "dir": 0.9856351875292462, - "tpr_gap": 0.005516043679281046, - "fpr_gap": 0.03637090327737813, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "home_ownership__x__verification_status", - "dpd": 0.0343875474383778, - "eo_gap": 0.0636573811298401, - "dir": 0.9655997718523373, - "tpr_gap": 0.01650250650050855, - "fpr_gap": 0.0636573811298401, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "annual_inc_quartile__x__verification_status", - "dpd": 0.02054232397484057, - "eo_gap": 0.04371873973202878, - "dir": 0.9794546321542239, - "tpr_gap": 0.009328152306949944, - "fpr_gap": 0.04371873973202878, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - } - ], - "prediction_threshold": 0.35, - "primary_threshold": 0.35, - "prediction_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval", - "thresholds": { - "dpd": 0.1, - "eo_gap": 0.11, - "dir": 0.8 - }, - "threshold_frontier": { - "path": "data/processed/fairness_threshold_frontier.parquet", - "thresholds": [ - 0.35, - 0.4, - 0.45, - 0.5 - ], - "worst_primary_attribute": "home_ownership__x__verification_status", - "selected_threshold": 0.35, - "all_primary_pass": true - }, - "decision_policy": { - "path": "models/fairness_decision_policy.json", - "global_threshold": 0.35, - "n_overrides": 0 - }, - "policy_config": "configs/fairness_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:42:20.905091+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" -} diff --git a/models/fairness_decision_policy.json b/models/fairness_decision_policy.json deleted file mode 100644 index beb307d..0000000 --- a/models/fairness_decision_policy.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "global_threshold": 0.35, - "overrides": [], - "selection": { - "source": "fairness_frontier_auto_select", - "n_passed": 6, - "worst_eo_gap": 0.0636573811298401, - "approval_rate": 0.9955430185394537 - }, - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:42:18.523782+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" -} diff --git a/models/governance_status.json b/models/governance_status.json deleted file mode 100644 index 83cad5f..0000000 --- a/models/governance_status.json +++ /dev/null @@ -1,301 +0,0 @@ -{ - "overall_pass": true, - "checks": { - "pass_psi": true, - "pass_breach_ratio": true, - "pass_score_psi": true, - "pass_auc_delta": true, - "pass_brier_increase": true, - "pass_calibration_gap_delta": true, - "pass_predictive_drift": true, - "pass_fairness": true, - "pass_c2st": false, - "pass_explainability": true, - "pass_reason_code_stability": true, - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "thresholds": { - "psi_threshold": 0.25, - "ks_pvalue_min": 0.01, - "cvm_pvalue_min": 0.01, - "c2st_auc_max": 0.6, - "max_feature_breach_ratio": 0.15, - "score_psi_max": 0.15, - "auc_delta_max": 0.05, - "brier_increase_max": 0.02, - "calibration_gap_delta_max": 0.02, - "explanation_rank_overlap_top10_min": 0.6, - "explanation_shap_psi_max": 0.25, - "reason_code_stability_min": 0.55 - }, - "summary": { - "n_features": 33, - "max_psi": 0.13944722694322031, - "mean_psi": 0.024499996173148176, - "min_ks_pvalue": 0.0, - "min_cvm_pvalue": 8.80828621152574e-09, - "c2st_auc": 0.9853501044444442, - "psi_breaches": 0, - "ks_breaches": 32, - "cvm_breaches": 33, - "feature_breach_ratio": 0.0, - "distribution_warning_ratio": 0.9848484848484849, - "c2st_rows_used": 100000, - "c2st_materiality": "severe", - "c2st_effective_driver_count": 1, - "score_psi": 0.01430573726529544, - "auc_train_reference": 0.7464579629585804, - "auc_test_oot": 0.7118119107831851, - "auc_delta_train_to_test": 0.03464605217539529, - "brier_train_reference": 0.13796511693423574, - "brier_test_oot": 0.1550631955367288, - "brier_increase_train_to_test": 0.017098078602493044, - "calibration_gap_train_reference": 0.0303899887875225, - "calibration_gap_test_oot": 0.0034155097827143766, - "calibration_gap_delta": 0.0, - "train_eval_rows": 100000, - "test_eval_rows": 100000, - "n_explanation_segments": 1, - "min_rank_overlap_top10": 0.9, - "max_explanation_shap_psi": 0.06104327913317865, - "min_reason_code_stability": 1.0, - "fairness_overall_pass": true, - "fairness_primary_threshold": 0.35, - "challenger_promotable": false, - "model_shift_type": "mixed_shift", - "governance_posture": "candidate_gate" - }, - "warnings": { - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "c2st": { - "auc": 0.9853501044444442, - "materiality": "severe", - "effective_driver_count": 1, - "top_drivers": [ - { - "feature": "int_rate", - "importance_auc": 0.4170757822222219 - }, - { - "feature": "loan_to_income", - "importance_auc": 0.00718282296296265 - }, - { - "feature": "pub_rec", - "importance_auc": 0.004748391851851608 - }, - { - "feature": "has_bankruptcy", - "importance_auc": 0.003179000740740475 - }, - { - "feature": "loan_amnt", - "importance_auc": 0.0025205829629626844 - }, - { - "feature": "fico_score", - "importance_auc": 0.0018507688888886875 - }, - { - "feature": "inq_last_6mths", - "importance_auc": 0.0005079955555552887 - }, - { - "feature": "term", - "importance_auc": 0.000483162962962654 - }, - { - "feature": "credit_age_years", - "importance_auc": 0.000386952592592323 - }, - { - "feature": "revol_bal_to_income", - "importance_auc": 0.000345855555555256 - } - ] - }, - "model_shift": { - "shift_type": "mixed_shift", - "structural_shift_level": "severe", - "predictive_degradation_level": "moderate", - "governance_posture": "candidate_gate", - "c2st_materiality": "severe", - "pvalue_interpretation": "Both structural shift and predictive degradation are present. This combination deserves the strongest governance posture." - }, - "artifacts": { - "drift_monitoring_path": "data/processed/drift_monitoring.parquet", - "explanation_drift_path": "data/processed/explanation_drift.parquet", - "fairness_status_path": "models/fairness_audit_status.json", - "fairness_frontier_path": "data/processed/fairness_threshold_frontier.parquet", - "challenger_promotion_report_path": "models/challenger_promotion_report.json", - "model_shift_status_path": "models/model_shift_status.json" - }, - "top_drift_features": [ - { - "feature": "fico_score", - "train_n": 798550, - "test_n": 276869, - "psi": 0.13944722694322031, - "ks_statistic": 0.15457282396927496, - "ks_pvalue": 0.0, - "cvm_statistic": 3230.128652537591, - "cvm_pvalue": 4.865294239708717e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "int_rate", - "train_n": 798550, - "test_n": 276869, - "psi": 0.10118060476101726, - "ks_statistic": 0.09481768715722516, - "ks_pvalue": 0.0, - "cvm_statistic": 398.7076133590017, - "cvm_pvalue": 1.135480649505638e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.05347961324875766, - "ks_statistic": 0.09122311383287474, - "ks_pvalue": 0.0, - "cvm_statistic": 909.8105939357774, - "cvm_pvalue": 2.1943404671098676e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_severity", - "train_n": 798550, - "test_n": 276869, - "psi": 0.05178332227166686, - "ks_statistic": 0.0916698979201469, - "ks_pvalue": 0.0, - "cvm_statistic": 44606.0478113642, - "cvm_pvalue": 2.134860395242022e-06, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "log_revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670529338066, - "cvm_pvalue": 4.4769498996899415e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income_sq", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670714583946, - "cvm_pvalue": 4.476950465903684e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_recency", - "train_n": 798550, - "test_n": 276869, - "psi": 0.038966862981116755, - "ks_statistic": 0.08135488943770064, - "ks_pvalue": 0.0, - "cvm_statistic": 11755.072386931657, - "cvm_pvalue": 9.791550346083255e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "credit_age_years", - "train_n": 798550, - "test_n": 276869, - "psi": 0.029775277868375027, - "ks_statistic": 0.054973967117968536, - "ks_pvalue": 0.0, - "cvm_statistic": 290.1511620187666, - "cvm_pvalue": 1.0223037416601244e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - } - ], - "top_explanation_breaches": [ - { - "segment_type": "overall", - "segment": "all", - "reference_period": "2018Q1|2018Q2|2018Q3|2018Q4|2019Q1|2019Q2|2019Q3", - "comparison_period": "2019Q4|2020Q1|2020Q2|2020Q3", - "reference_n": 4763, - "comparison_n": 237, - "rank_overlap_top10": 0.9, - "avg_shap_psi_top5": 0.044564149041987314, - "max_shap_psi_top5": 0.06104327913317865, - "reason_code_match_rate": 1.0, - "pass_rank_overlap": true, - "pass_distribution_shift": true, - "pass_reason_code_stability": true, - "passed_all": true, - "feature_psi_details": "[{\"feature\": \"int_rate\", \"psi\": 0.0571152296148128}, {\"feature\": \"term\", \"psi\": 0.05398847136247015}, {\"feature\": \"fico_score\", \"psi\": 0.06104327913317865}, {\"feature\": \"home_ownership\", \"psi\": 0.03675845025796413}, {\"feature\": \"many_recent_opens\", \"psi\": 0.013915314841510859}]", - "reason_code_details": "[{\"band\": \"low\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}, {\"band\": \"near_threshold\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}]" - } - ], - "primary_threshold": 0.35, - "explainability_pass": true, - "explanation_drift_pass": true, - "reason_code_stability_pass": true, - "challenger_promotable": false, - "policy_config": "configs/mrm_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:43:53.829393+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" -} diff --git a/models/monotonicity_audit_status.json b/models/monotonicity_audit_status.json deleted file mode 100644 index 44db254..0000000 --- a/models/monotonicity_audit_status.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "diagnostic_only": true, - "overall_pass": true, - "summary": { - "n_bands": 10, - "n_pairs": 9, - "n_disruptions": 0, - "disruption_rate": 0.0, - "max_negative_step": 0.0, - "overall_pass": true, - "n_constrained_features": 4, - "mean_feature_violation_rate": 0.00537109375, - "max_feature_violation_rate": 0.01953125, - "mean_predicted_pd": 0.2171240052055989, - "observed_default_rate": 0.21977902907151034 - }, - "top_feature_violations": [ - { - "feature": "annual_inc", - "direction": -1, - "violation_rate": 0.01953125 - }, - { - "feature": "loan_to_income", - "direction": 1, - "violation_rate": 0.001953125 - }, - { - "feature": "installment", - "direction": 1, - "violation_rate": 0.0 - }, - { - "feature": "dti", - "direction": 1, - "violation_rate": 0.0 - } - ], - "artifacts": { - "band_summary_path": "data/processed/monotonicity_band_summary.parquet", - "pair_report_path": "data/processed/monotonicity_pair_report.parquet", - "feature_report_path": "data/processed/monotonicity_feature_report.parquet" - }, - "config_path": "configs/pd_model.champion.yaml", - "schema_version": "2026-03-30.1", - "generated_at_utc": "2026-04-07T17:43:02.044285+00:00", - "run_tag": "paper1-e2e-all-champions-2026-04-07" -} diff --git a/models/mrm_report_status.json b/models/mrm_report_status.json deleted file mode 100644 index 90ad33e..0000000 --- a/models/mrm_report_status.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "schema_version": "2026-03-14.1", - "generated_at_utc": "2026-05-11T03:06:05.214271+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "overall_pass": true, - "compliance_summary": { - "overall_pass": true, - "subsystems": { - "pipeline": true, - "conformal": true, - "governance": true, - "fairness": true - }, - "n_subsystems": 4, - "n_passing": 4 - }, - "report_path": "reports/mrm/mrm_validation_report.json" -} diff --git a/models/paper_figures_status.json b/models/paper_figures_status.json deleted file mode 100644 index 96b0b74..0000000 --- a/models/paper_figures_status.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "schema_version": "2026-03-21.1", - "generated_at_utc": "2026-06-11T08:39:00.657856+00:00", - "run_tag": "untracked", - "output_dir": "reports/crpto/figures", - "figures_attempted": 14, - "figures_succeeded": 14, - "errors": {}, - "files": [ - "crpto_fig10_cqr_per_grade.pdf", - "crpto_fig11_crpto_stability.pdf", - "crpto_fig12_crpto_conceptual_pipeline.pdf", - "crpto_fig13_alpha_gamma_funded_set.pdf", - "crpto_fig14_robust_region_heatmap.pdf", - "crpto_fig15_regret_auditability_frontier.pdf", - "crpto_fig16_tail_risk_frontier.pdf", - "crpto_fig17_tail_risk_lgd.pdf", - "crpto_fig18_tail_constrained_frontier.pdf", - "crpto_fig19_online_coverage_aci.pdf", - "crpto_fig1_journal_pipeline.pdf", - "crpto_fig20_bound_claim_layers.pdf", - "crpto_fig21_end_to_end_arc.pdf", - "crpto_fig22_external_replication.pdf", - "crpto_fig23_external_candidate_sensitivity.pdf", - "crpto_fig24_freddie_all_candidate_certificate.pdf", - "crpto_fig25_price_of_robustness_scaling.pdf", - "crpto_fig7_uncertainty_baselines.pdf", - "crpto_fig8_alpha_pareto.pdf", - "crpto_fig9_spo_regret.pdf", - "crpto_fig_alpha_gamma_bound.pdf" - ] -} diff --git a/models/pd_conformal_width_attribution_status.json b/models/pd_conformal_width_attribution_status.json deleted file mode 100644 index 3380c96..0000000 --- a/models/pd_conformal_width_attribution_status.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "artifact_namespace": "conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1", - "selected_partition": "score_decile_mondrian", - "selected_partition_probability_source": "raw", - "selected_n_score_bins": 5, - "selected_fallback_mode": "score_only", - "selected_alpha_used_90": 0.1, - "selected_alpha_used_95": 0.05, - "selected_min_group_size": 100, - "selected_scaled_scores": true, - "selected_score_scale_family": "bernoulli_sqrt", - "group_factors_after_shrinkback": { - "score_q01": 1.05, - "score_q04": 1.02 - }, - "temporal_factors_after_shrinkback": {}, - "width_attribution_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/pd_conformal_width_attribution.parquet", - "shrinkback_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_shrinkback_report.parquet", - "evaluation_scope": "test", - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-06-07T22:31:23.845840+00:00", - "run_tag": "ijds-rebaseline-2026-06-07" -} diff --git a/models/pd_training_checkpoints/calibration_selection.json b/models/pd_training_checkpoints/calibration_selection.json deleted file mode 100644 index d15459a..0000000 --- a/models/pd_training_checkpoints/calibration_selection.json +++ /dev/null @@ -1,608 +0,0 @@ -{ - "selected_method": "venn_abers", - "selection_report": { - "selected_method": "venn_abers", - "selection_reason": "feasible_multi_metric", - "auc_drop_limit": 0.0015, - "candidates": [ - { - "method": "platt", - "folds_used": 4, - "mean_brier": 0.15958611449060534, - "mean_log_loss": 0.4890764869921422, - "mean_ece": 0.02196692357486136, - "mean_auc_drop": 0.0, - "brier_variance": 2.2113248247945004e-05, - "ece_variance": 2.1460384671970526e-06, - "stability": 2.4259286715142055e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157842795903832, - "auc_drop": 0.0, - "brier": 0.15215240621868587, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4713293711835714, - "ece": 0.02368482016945161 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7136784422022739, - "auc_drop": 0.0, - "brier": 0.159555317773546, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.488901170962, - "ece": 0.021711479165915166 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7200414067628481, - "auc_drop": 0.0, - "brier": 0.1649512491787557, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5015066366393816, - "ece": 0.022735422476058913 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7131337480584359, - "auc_drop": 0.0, - "brier": 0.1616854847914337, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.49456876918361575, - "ece": 0.01973597248801976 - } - ] - }, - { - "method": "isotonic", - "folds_used": 4, - "mean_brier": 0.1595429088972856, - "mean_log_loss": 0.4892541234412081, - "mean_ece": 0.02289631334243558, - "mean_auc_drop": 0.0001586991096110546, - "brier_variance": 2.171736566771213e-05, - "ece_variance": 5.206349571603733e-06, - "stability": 2.6923715239315863e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157301425597784, - "auc_drop": 5.413703060475061e-05, - "brier": 0.1521639826043805, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.47247222274514195, - "ece": 0.025867214283258718 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.713445008770997, - "auc_drop": 0.0002334334312769082, - "brier": 0.15957601661983706, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48870621632901473, - "ece": 0.02327066859711214 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198826403172529, - "auc_drop": 0.00015876644559520425, - "brier": 0.16486495251282393, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5010233468342666, - "ece": 0.022993379359887947 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129452885274685, - "auc_drop": 0.00018845953096735535, - "brier": 0.16156668385210082, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4948147078564092, - "ece": 0.01945399112948353 - } - ] - }, - { - "method": "venn_abers", - "folds_used": 4, - "mean_brier": 0.1595283851639654, - "mean_log_loss": 0.48867730149603966, - "mean_ece": 0.02263933898865999, - "mean_auc_drop": 0.00014739322381726594, - "brier_variance": 2.1761744780224087e-05, - "ece_variance": 4.918644449333954e-06, - "stability": 2.668038922955804e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157573398663852, - "auc_drop": 2.6939723997920773e-05, - "brier": 0.15214071268666568, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4710116811145923, - "ece": 0.025535908230122004 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7134440923750336, - "auc_drop": 0.00023434982724035436, - "brier": 0.15956014452131484, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48862787613086206, - "ece": 0.022902816320554322 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198870849274979, - "auc_drop": 0.00015432183535024624, - "brier": 0.16485156768382223, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5009423497212537, - "ece": 0.02282313215352082 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129597865497553, - "auc_drop": 0.0001739615086805424, - "brier": 0.1615611157640588, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4941272990174505, - "ece": 0.01929549925044282 - } - ] - } - ], - "feasible_candidates": [ - { - "method": "platt", - "folds_used": 4, - "mean_brier": 0.15958611449060534, - "mean_log_loss": 0.4890764869921422, - "mean_ece": 0.02196692357486136, - "mean_auc_drop": 0.0, - "brier_variance": 2.2113248247945004e-05, - "ece_variance": 2.1460384671970526e-06, - "stability": 2.4259286715142055e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157842795903832, - "auc_drop": 0.0, - "brier": 0.15215240621868587, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4713293711835714, - "ece": 0.02368482016945161 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7136784422022739, - "auc_drop": 0.0, - "brier": 0.159555317773546, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.488901170962, - "ece": 0.021711479165915166 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7200414067628481, - "auc_drop": 0.0, - "brier": 0.1649512491787557, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5015066366393816, - "ece": 0.022735422476058913 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7131337480584359, - "auc_drop": 0.0, - "brier": 0.1616854847914337, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.49456876918361575, - "ece": 0.01973597248801976 - } - ] - }, - { - "method": "isotonic", - "folds_used": 4, - "mean_brier": 0.1595429088972856, - "mean_log_loss": 0.4892541234412081, - "mean_ece": 0.02289631334243558, - "mean_auc_drop": 0.0001586991096110546, - "brier_variance": 2.171736566771213e-05, - "ece_variance": 5.206349571603733e-06, - "stability": 2.6923715239315863e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157301425597784, - "auc_drop": 5.413703060475061e-05, - "brier": 0.1521639826043805, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.47247222274514195, - "ece": 0.025867214283258718 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.713445008770997, - "auc_drop": 0.0002334334312769082, - "brier": 0.15957601661983706, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48870621632901473, - "ece": 0.02327066859711214 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198826403172529, - "auc_drop": 0.00015876644559520425, - "brier": 0.16486495251282393, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5010233468342666, - "ece": 0.022993379359887947 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129452885274685, - "auc_drop": 0.00018845953096735535, - "brier": 0.16156668385210082, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4948147078564092, - "ece": 0.01945399112948353 - } - ] - }, - { - "method": "venn_abers", - "folds_used": 4, - "mean_brier": 0.1595283851639654, - "mean_log_loss": 0.48867730149603966, - "mean_ece": 0.02263933898865999, - "mean_auc_drop": 0.00014739322381726594, - "brier_variance": 2.1761744780224087e-05, - "ece_variance": 4.918644449333954e-06, - "stability": 2.668038922955804e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157573398663852, - "auc_drop": 2.6939723997920773e-05, - "brier": 0.15214071268666568, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4710116811145923, - "ece": 0.025535908230122004 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7134440923750336, - "auc_drop": 0.00023434982724035436, - "brier": 0.15956014452131484, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48862787613086206, - "ece": 0.022902816320554322 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198870849274979, - "auc_drop": 0.00015432183535024624, - "brier": 0.16485156768382223, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5009423497212537, - "ece": 0.02282313215352082 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129597865497553, - "auc_drop": 0.0001739615086805424, - "brier": 0.1615611157640588, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4941272990174505, - "ece": 0.01929549925044282 - } - ] - } - ] - }, - "candidate_reports": [ - { - "method": "platt", - "folds_used": 4, - "mean_brier": 0.15958611449060534, - "mean_log_loss": 0.4890764869921422, - "mean_ece": 0.02196692357486136, - "mean_auc_drop": 0.0, - "brier_variance": 2.2113248247945004e-05, - "ece_variance": 2.1460384671970526e-06, - "stability": 2.4259286715142055e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157842795903832, - "auc_drop": 0.0, - "brier": 0.15215240621868587, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4713293711835714, - "ece": 0.02368482016945161 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7136784422022739, - "auc_drop": 0.0, - "brier": 0.159555317773546, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.488901170962, - "ece": 0.021711479165915166 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7200414067628481, - "auc_drop": 0.0, - "brier": 0.1649512491787557, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5015066366393816, - "ece": 0.022735422476058913 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7131337480584359, - "auc_drop": 0.0, - "brier": 0.1616854847914337, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.49456876918361575, - "ece": 0.01973597248801976 - } - ] - }, - { - "method": "isotonic", - "folds_used": 4, - "mean_brier": 0.1595429088972856, - "mean_log_loss": 0.4892541234412081, - "mean_ece": 0.02289631334243558, - "mean_auc_drop": 0.0001586991096110546, - "brier_variance": 2.171736566771213e-05, - "ece_variance": 5.206349571603733e-06, - "stability": 2.6923715239315863e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157301425597784, - "auc_drop": 5.413703060475061e-05, - "brier": 0.1521639826043805, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.47247222274514195, - "ece": 0.025867214283258718 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.713445008770997, - "auc_drop": 0.0002334334312769082, - "brier": 0.15957601661983706, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48870621632901473, - "ece": 0.02327066859711214 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198826403172529, - "auc_drop": 0.00015876644559520425, - "brier": 0.16486495251282393, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5010233468342666, - "ece": 0.022993379359887947 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129452885274685, - "auc_drop": 0.00018845953096735535, - "brier": 0.16156668385210082, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4948147078564092, - "ece": 0.01945399112948353 - } - ] - }, - { - "method": "venn_abers", - "folds_used": 4, - "mean_brier": 0.1595283851639654, - "mean_log_loss": 0.48867730149603966, - "mean_ece": 0.02263933898865999, - "mean_auc_drop": 0.00014739322381726594, - "brier_variance": 2.1761744780224087e-05, - "ece_variance": 4.918644449333954e-06, - "stability": 2.668038922955804e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7157842795903832, - "cal_auc": 0.7157573398663852, - "auc_drop": 2.6939723997920773e-05, - "brier": 0.15214071268666568, - "brier_raw": 0.20959009479678367, - "brier_degraded": false, - "log_loss": 0.4710116811145923, - "ece": 0.025535908230122004 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7136784422022739, - "cal_auc": 0.7134440923750336, - "auc_drop": 0.00023434982724035436, - "brier": 0.15956014452131484, - "brier_raw": 0.21312139315174614, - "brier_degraded": false, - "log_loss": 0.48862787613086206, - "ece": 0.022902816320554322 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7200414067628481, - "cal_auc": 0.7198870849274979, - "auc_drop": 0.00015432183535024624, - "brier": 0.16485156768382223, - "brier_raw": 0.21468577663120803, - "brier_degraded": false, - "log_loss": 0.5009423497212537, - "ece": 0.02282313215352082 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131337480584359, - "cal_auc": 0.7129597865497553, - "auc_drop": 0.0001739615086805424, - "brier": 0.1615611157640588, - "brier_raw": 0.2092836349994178, - "brier_degraded": false, - "log_loss": 0.4941272990174505, - "ece": 0.01929549925044282 - } - ] - } - ] -} diff --git a/models/pd_training_checkpoints/config_validation.json b/models/pd_training_checkpoints/config_validation.json deleted file mode 100644 index 3295399..0000000 --- a/models/pd_training_checkpoints/config_validation.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "config_path": "configs/crpto_pd_model.yaml", - "output": { - "model_path": "models/pd_canonical.cbm", - "conformal_path": "models/pd_canonical_calibrator.pkl", - "default_model_path": "models/pd_catboost_default.cbm", - "tuned_model_path": "models/pd_canonical.cbm", - "status_path": "models/pd_training_status.json", - "checkpoint_dir": "models/pd_training_checkpoints", - "brier_decomposition_path": "data/processed/brier_score_decomposition.json", - "murphy_diagram_path": "reports/figures/calibration/murphy_diagram.png", - "canonical_model_path": "models\\pd_canonical.cbm", - "canonical_calibrator_path": "models\\pd_canonical_calibrator.pkl", - "contract_path": "models\\pd_model_contract.json", - "logreg_model_path": "models/pd_logreg_baseline.pkl", - "training_record_path": "models/pd_training_record.pkl", - "seed_replay_status_path": "models/pd_hpo_seed_replay_status.json", - "test_predictions_path": "data/processed/test_predictions.parquet", - "shap_dir": "reports/figures/shap", - "threshold_semantics_path": "models/threshold_semantics.json" - }, - "feature_source": { - "mode": "feature_config", - "feature_config_path": "data/processed/feature_config.pkl" - }, - "data": { - "train_path": "data/processed/train_fe.parquet", - "test_path": "data/processed/test_fe.parquet", - "calibration_path": "data/processed/calibration_fe.parquet", - "cutoff_date": "2018-01-01" - }, - "hpo_enabled": false, - "validation": { - "scheme": "temporal_train_val_cal_test", - "val_from_tail_fraction_of_train": 0.15, - "oot_test_cutoff": "2018-01-01", - "walk_forward": { - "enabled": false, - "n_windows": 3, - "min_train_rows": 200000, - "window_rows": 80000, - "date_col": "issue_d", - "max_rows": 0 - }, - "seed_replay": { - "enabled": false, - "top_k_trials": 3, - "seeds": [ - 42, - 52, - 62 - ], - "prioritize_gate_pass": true - } - } -} diff --git a/models/pd_training_checkpoints/feature_resolution.json b/models/pd_training_checkpoints/feature_resolution.json deleted file mode 100644 index f1c760b..0000000 --- a/models/pd_training_checkpoints/feature_resolution.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "feature_source": "feature_config", - "catboost_features": [ - "loan_amnt", - "int_rate", - "installment", - "annual_inc", - "dti", - "loan_to_income", - "installment_burden", - "revol_bal_to_income", - "open_acc_ratio", - "fico_score", - "credit_age_years", - "emp_length_num", - "open_acc", - "total_acc", - "revol_bal", - "pub_rec", - "inq_last_6mths", - "mort_acc", - "delinq_severity", - "delinq_recency", - "il_ratio", - "log_annual_inc", - "log_revol_bal", - "loan_to_income_sq", - "fico_x_dti", - "has_delinq_2yrs", - "has_pub_rec", - "has_bankruptcy", - "has_recent_inq", - "has_mortgage", - "many_recent_opens", - "recent_chargeoff", - "grade", - "sub_grade", - "home_ownership", - "purpose", - "verification_status", - "term", - "int_rate_bucket", - "dti_bucket", - "fico_bucket", - "int_rate_bucket__grade" - ], - "logreg_features": [ - "loan_amnt", - "int_rate", - "installment", - "annual_inc", - "dti", - "loan_to_income", - "installment_burden", - "revol_bal_to_income", - "open_acc_ratio", - "fico_score", - "credit_age_years", - "emp_length_num", - "open_acc", - "total_acc", - "revol_bal", - "pub_rec", - "inq_last_6mths", - "mort_acc", - "delinq_severity", - "delinq_recency", - "il_ratio", - "log_annual_inc", - "log_revol_bal", - "loan_to_income_sq", - "fico_x_dti", - "has_delinq_2yrs", - "has_pub_rec", - "has_bankruptcy", - "has_recent_inq", - "has_mortgage", - "many_recent_opens", - "recent_chargeoff" - ], - "categorical_features": [ - "grade", - "sub_grade", - "home_ownership", - "purpose", - "verification_status", - "term", - "int_rate_bucket", - "dti_bucket", - "fico_bucket" - ] -} diff --git a/models/pd_training_checkpoints/hpo_summary.json b/models/pd_training_checkpoints/hpo_summary.json deleted file mode 100644 index 8c620cf..0000000 --- a/models/pd_training_checkpoints/hpo_summary.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "best_params": { - "iterations": 4200, - "learning_rate": 0.057321202729872456, - "depth": 8, - "l2_leaf_reg": 119.37272987133554, - "min_data_in_leaf": 135, - "rsm": 0.5716653769355704, - "random_strength": 1.3208942645900998e-07, - "border_count": 254, - "bootstrap_type": "MVS", - "subsample": 0.678878683066026, - "loss_function": "Logloss", - "auto_class_weights": "Balanced", - "eval_metric": "AUC", - "has_time": true, - "allow_writing_files": false, - "random_seed": 42, - "early_stopping_rounds": 150, - "grow_policy": "SymmetricTree", - "leaf_estimation_iterations": 5, - "monotone_constraints": "installment:1,annual_inc:-1,dti:1,loan_to_income:1", - "feature_weights": { - "loan_to_income": 1.3, - "annual_inc": 1.2, - "dti": 1.2, - "installment": 1.0 - }, - "penalties_coefficient": 0.75 - }, - "hpo_trials_executed": 0, - "hpo_best_validation_auc": 0.7220014049345107, - "seed_replay_report": { - "enabled": false, - "reason": "hpo_disabled", - "rows": [], - "selected_trial": null, - "selected_params": null - } -} diff --git a/models/pd_training_status.json b/models/pd_training_status.json deleted file mode 100644 index 474b443..0000000 --- a/models/pd_training_status.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "stage_name": "pd_training", - "phase": "artifacts_saved", - "state": "completed", - "config_path": "configs/crpto_pd_model.yaml", - "updated_at_utc": "2026-06-07T22:28:50.182734+00:00", - "model_path": "models\\pd_canonical.cbm", - "calibrator_path": "models\\pd_canonical_calibrator.pkl", - "record_path": "models\\pd_training_record.pkl", - "auc_roc": 0.7126777845551742, - "brier_score": 0.1545907367602431 -} diff --git a/models/pd_validation_interpretation_status.json b/models/pd_validation_interpretation_status.json deleted file mode 100644 index 6bad2d6..0000000 --- a/models/pd_validation_interpretation_status.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "diagnostic_only": true, - "overall_pass": true, - "severity": "warning", - "signal_type": "material_slice_deviation", - "summary": { - "gap": 0.0026550238659114322, - "gap_bp": 26.55023865911432, - "abs_gap_bp": 26.55023865911432, - "materiality_band": "low", - "grade_material_breaches": 1, - "max_grade_gap_bp": 507.225447549276, - "band_material_breaches": 1, - "max_band_gap_bp": 160.26695962362837, - "global_brier": 0.15463051782916892, - "global_ece": 0.006379675621599359, - "worst_protected_group_ece": 0.021540776718198477, - "worst_grade_brier": 0.24104901112917315, - "max_decile_gap_bp": 168.75745454345503, - "persistent_quarter_gaps": 8, - "n_quarters_evaluated": 9, - "exact_binomial_p_value": 0.0007157999395488739, - "hl_p_value": 0.0, - "predicted_pd_inside_jeffreys": false - }, - "quarter_rows": [ - { - "issue_quarter": "2018Q1", - "n_obs": 53924, - "observed_default_rate": 0.2329945849714413, - "mean_predicted_pd": 0.2154554186518523, - "rate_gap": 0.01753916631958899, - "abs_gap_bp": 175.3916631958899 - }, - { - "issue_quarter": "2018Q2", - "n_obs": 56915, - "observed_default_rate": 0.2556268119125011, - "mean_predicted_pd": 0.22076424063290606, - "rate_gap": 0.03486257127959502, - "abs_gap_bp": 348.6257127959502 - }, - { - "issue_quarter": "2018Q3", - "n_obs": 47168, - "observed_default_rate": 0.23289094301221167, - "mean_predicted_pd": 0.22108089847018017, - "rate_gap": 0.011810044542031495, - "abs_gap_bp": 118.10044542031494 - }, - { - "issue_quarter": "2018Q4", - "n_obs": 39171, - "observed_default_rate": 0.23305506624798958, - "mean_predicted_pd": 0.220510756155411, - "rate_gap": 0.012544310092578587, - "abs_gap_bp": 125.44310092578587 - }, - { - "issue_quarter": "2019Q1", - "n_obs": 27508, - "observed_default_rate": 0.21241093500072705, - "mean_predicted_pd": 0.2166885452353189, - "rate_gap": -0.004277610234591861, - "abs_gap_bp": 42.77610234591861 - }, - { - "issue_quarter": "2019Q2", - "n_obs": 22737, - "observed_default_rate": 0.20191757927606985, - "mean_predicted_pd": 0.21258154254177034, - "rate_gap": -0.010663963265700488, - "abs_gap_bp": 106.63963265700488 - }, - { - "issue_quarter": "2019Q3", - "n_obs": 16603, - "observed_default_rate": 0.1451544901523821, - "mean_predicted_pd": 0.20750430468773834, - "rate_gap": -0.062349814535356235, - "abs_gap_bp": 623.4981453535623 - }, - { - "issue_quarter": "2019Q4", - "n_obs": 8557, - "observed_default_rate": 0.08425850181138249, - "mean_predicted_pd": 0.21131537285529073, - "rate_gap": -0.12705687104390823, - "abs_gap_bp": 1270.5687104390822 - }, - { - "issue_quarter": "2020Q1", - "n_obs": 2977, - "observed_default_rate": 0.0141081625797783, - "mean_predicted_pd": 0.2047846456046201, - "rate_gap": -0.1906764830248418, - "abs_gap_bp": 1906.764830248418 - } - ], - "artifacts": { - "quarter_materiality_path": "data/processed/pd_backtesting_quarter_materiality.parquet" - }, - "schema_version": "2026-03-30.1", - "generated_at_utc": "2026-04-07T17:43:11.322026+00:00", - "run_tag": "paper1-e2e-all-champions-2026-04-07" -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_exact_context.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_exact_context.json deleted file mode 100644 index 3ffefbc..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_exact_context.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "generated_at_utc": "2026-04-05T21:52:01.591127+00:00", - "run_label": "rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c", - "conformal_intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "search_space": { - "risk_grid": [ - 0.15, - 0.155, - 0.16, - 0.165, - 0.17 - ], - "aversion_grid": [ - 0.0, - 0.05, - 0.1 - ], - "gamma_grid": [ - 0.4, - 0.45, - 0.5, - 0.55, - 0.6 - ], - "delta_cap_grid": [ - 0.75, - 1.0 - ], - "tail_focus_grid": [ - 1.0 - ], - "budget_profiles": [ - { - "name": "free_budget", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0 - }, - { - "name": "floored_budget", - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5 - } - ], - "alpha_grid": [ - 0.01, - 0.03, - 0.05, - 0.1 - ], - "max_candidates": 25000, - "random_states": [ - 42, - 2026 - ], - "policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ], - "bucket_return_k": 20, - "bucket_proxy_k": 20, - "bucket_family_k": 10, - "bucket_region_k": 10, - "incumbent_policy_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware/portfolio_bound_aware_selection.json", - "incumbent_risk_neighbors": [ - 0.155, - 0.16, - 0.165, - 0.17 - ], - "incumbent_gamma_neighbors": [ - 0.45, - 0.5, - 0.55 - ], - "incumbent_policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ] - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "price_of_robustness(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)" - ] - }, - "frontier_raw_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_frontier_raw.parquet", - "frontier_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_frontier.parquet", - "shortlist_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_shortlist.parquet", - "bound_eval_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_bound_eval.parquet", - "selection_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_selection.json", - "runtime_status_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_status.json", - "runtime_checkpoint_dir": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints", - "resource_snapshot_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/resource_snapshot.json", - "frontier_solver_backend": "cuopt", - "exact_solver_backend": "highs", - "budget": 1000000.0, - "t_eval": 0.05, - "max_candidates": 25000, - "random_states": [ - 42, - 2026 - ], - "alpha_grid": [ - 0.01, - 0.03, - 0.05, - 0.1 - ] -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json deleted file mode 100644 index 63222f6..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "001_frontier_complete", - "updated_at_utc": "2026-04-05T21:52:01.559814+00:00", - "frontier_total_units": 910, - "frontier_completed_units": 910, - "frontier_pct_complete": 1.0, - "bound_total_checks": 0, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 910, - "global_completed_units": 910, - "global_pct_complete": 1.0, - "elapsed_sec": 163.21474848903017, - "eta_sec": 0.0, - "frontier_policy_count": 450, - "frontier_raw_rows": 900 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json deleted file mode 100644 index 599d576..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "002_shortlist_built", - "updated_at_utc": "2026-04-05T21:52:01.577272+00:00", - "frontier_total_units": 910, - "frontier_completed_units": 910, - "frontier_pct_complete": 1.0, - "bound_total_checks": 480, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 1390, - "global_completed_units": 910, - "global_pct_complete": 0.6546762589928058, - "elapsed_sec": 163.23313951602904, - "eta_sec": 86.10099666779554, - "shortlist_size": 60, - "shortlist_buckets": { - "forced_incumbent_neighbors": 48, - "incumbent_region": 10, - "conservative_proxy": 2 - } -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json deleted file mode 100644 index d7da925..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "003_selection_complete", - "updated_at_utc": "2026-04-05T22:17:22.187084+00:00", - "frontier_total_units": 910, - "frontier_completed_units": 910, - "frontier_pct_complete": 1.0, - "bound_total_checks": 480, - "bound_completed_checks": 480, - "bound_pct_complete": 1.0, - "global_total_units": 1390, - "global_completed_units": 1390, - "global_pct_complete": 1.0, - "elapsed_sec": 1682.9757338070194, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 105520.69190669092 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_status.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_status.json deleted file mode 100644 index 43f5261..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_status.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "phase": "selection_complete", - "state": "completed", - "updated_at_utc": "2026-04-05T22:17:22.186831+00:00", - "run_tag": "rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c", - "frontier_total_units": 910, - "frontier_completed_units": 910, - "frontier_pct_complete": 1.0, - "bound_total_checks": 480, - "bound_completed_checks": 480, - "bound_pct_complete": 1.0, - "global_total_units": 1390, - "global_completed_units": 1390, - "global_pct_complete": 1.0, - "elapsed_sec": 1682.9757338070194, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 105520.69190669092 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_selection.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_selection.json deleted file mode 100644 index 8bbf73b..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_selection.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "generated_at_utc": "2026-04-05T22:17:22.129540+00:00", - "run_label": "rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c", - "conformal_intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "search_space": { - "risk_grid": [ - 0.15, - 0.155, - 0.16, - 0.165, - 0.17 - ], - "aversion_grid": [ - 0.0, - 0.05, - 0.1 - ], - "gamma_grid": [ - 0.4, - 0.45, - 0.5, - 0.55, - 0.6 - ], - "delta_cap_grid": [ - 0.75, - 1.0 - ], - "tail_focus_grid": [ - 1.0 - ], - "budget_profiles": [ - { - "name": "free_budget", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0 - }, - { - "name": "floored_budget", - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5 - } - ], - "alpha_grid": [ - 0.01, - 0.03, - 0.05, - 0.1 - ], - "max_candidates": 25000, - "random_states": [ - 42, - 2026 - ], - "policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ], - "bucket_return_k": 20, - "bucket_proxy_k": 20, - "bucket_family_k": 10, - "bucket_region_k": 10, - "incumbent_policy_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware/portfolio_bound_aware_selection.json", - "incumbent_risk_neighbors": [ - 0.155, - 0.16, - 0.165, - 0.17 - ], - "incumbent_gamma_neighbors": [ - 0.45, - 0.5, - 0.55 - ], - "incumbent_policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ] - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "price_of_robustness(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.17, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs" - }, - "selected_metrics": { - "risk_tolerance": 0.17, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "cuopt", - "seed_count": 2, - "sample_random_states": "42,2026", - "ab_pass_all": true, - "ab_pass_rate": 1.0, - "realized_total_return": 105520.69190669092, - "realized_total_return_max": 109862.28370644282, - "price_of_robustness": -14304.853978044637, - "price_of_robustness_pct": -14.911787945604422, - "ab_diff_total_return": 21252.868235008806, - "objective_value": 110453.47781692201, - "n_funded": 209.5, - "total_allocated": 1000000.0, - "expected_return_net_point": 110453.47781692201, - "worst_case_pd": 0.28031129568192314, - "point_pd": 0.07974530353273998, - "pd_cap_slack": 0.0, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.45,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.17,\"solver_backend\":\"cuopt\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0}", - "return_first_rank": 73, - "bound_proxy_rank": 425, - "shortlist_bucket": "forced_incumbent_neighbors", - "candidate_rank": 47, - "alpha01_exact_pass": true, - "alpha01_pass_rate": 1.0, - "alpha01_gamma_cp": 0.191961, - "alpha01_weighted_miscoverage_V": 0.072525, - "alpha01_violation": 0.0, - "alpha01_weighted_pd_true": 0.072525, - "alpha01_weighted_pd_constraint_used": 0.17, - "alpha01_empirical_coverage_funded": 0.94255, - "alpha03_exact_pass": true, - "alpha03_weighted_miscoverage_V": 0.072525, - "alpha10_exact_pass": true, - "alpha10_weighted_miscoverage_V": 0.0669375 - }, - "selection_reason": "selected_best_alpha01_exact_pass", - "frontier_raw_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_frontier_raw.parquet", - "frontier_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_frontier.parquet", - "shortlist_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_shortlist.parquet", - "bound_eval_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_bound_eval.parquet", - "runtime_status_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_status.json", - "runtime_checkpoint_dir": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/portfolio_bound_aware_runtime_checkpoints", - "resource_snapshot_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/resource_snapshot.json", - "frontier_solver_backend": "cuopt", - "exact_solver_backend": "highs" -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/resource_snapshot.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/resource_snapshot.json deleted file mode 100644 index 37f11b2..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/resource_snapshot.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "run_label": "rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c", - "solver_backend": "cuopt", - "exact_solver_backend": "highs", - "start": { - "captured_at_utc": "2026-04-05T21:49:18.343677+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 57396816, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 4013.0, - "memory_free_mb": 6041.0, - "gpu_util_pct": 12.0, - "memory_util_pct": 1.0 - } - }, - "backend_validation": { - "validated_at_utc": "2026-04-05T21:49:16.731961+00:00", - "python": "legacy-wsl-python-removed-after-windows-revalidation", - "expected_release": "26.02", - "cuopt": "26.02.00", - "cudf": "26.02.01", - "cupy": "14.0.1", - "pyomo": "6.10.0", - "loguru": "0.7.3" - }, - "end": { - "captured_at_utc": "2026-04-05T22:17:22.888966+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 55522084, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 4039.0, - "memory_free_mb": 6015.0, - "gpu_util_pct": 1.0, - "memory_util_pct": 6.0 - } - }, - "gpu_summary": { - "available": true, - "sample_count": 1582, - "csv_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_25k_gpu_2026-04-05-1611c/gpu_samples.csv", - "peak_gpu_util": 49.0, - "avg_gpu_util": 5.133375474083438, - "peak_memory_used_mb": 4207.0, - "avg_memory_used_mb": 4036.311630847029, - "peak_power_draw_w": 107.73, - "avg_power_draw_w": 30.901447534766092 - } -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json deleted file mode 100644 index 9969f6c..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "001_frontier_complete", - "updated_at_utc": "2026-04-05T22:43:15.906705+00:00", - "frontier_total_units": 50, - "frontier_completed_units": 50, - "frontier_pct_complete": 1.0, - "bound_total_checks": 0, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 50, - "global_completed_units": 50, - "global_pct_complete": 1.0, - "elapsed_sec": 482.63243265799247, - "eta_sec": 0.0, - "frontier_policy_count": 45, - "frontier_raw_rows": 45 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json deleted file mode 100644 index 3382f12..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "002_shortlist_built", - "updated_at_utc": "2026-04-05T22:43:15.921662+00:00", - "frontier_total_units": 50, - "frontier_completed_units": 50, - "frontier_pct_complete": 1.0, - "bound_total_checks": 180, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 230, - "global_completed_units": 50, - "global_pct_complete": 0.21739130434782608, - "elapsed_sec": 482.6481845859671, - "eta_sec": 1737.5334645094817, - "shortlist_size": 45, - "shortlist_buckets": { - "incumbent_region": 24, - "forced_incumbent_neighbors": 12, - "conservative_proxy": 9 - } -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json deleted file mode 100644 index 7d7e5a3..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "003_selection_complete", - "updated_at_utc": "2026-06-07T19:41:22.024459+00:00", - "frontier_total_units": 50, - "frontier_completed_units": 50, - "frontier_pct_complete": 1.0, - "bound_total_checks": 180, - "bound_completed_checks": 180, - "bound_pct_complete": 1.0, - "global_total_units": 230, - "global_completed_units": 230, - "global_pct_complete": 1.0, - "elapsed_sec": 0.17199999990407377, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 170464.5429284627 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_status.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_status.json deleted file mode 100644 index 1e7b951..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_status.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "phase": "selection_complete", - "state": "completed", - "updated_at_utc": "2026-06-07T19:41:22.023463+00:00", - "run_tag": "rank1_alpha01_bound_aware_276k_full_2026-04-05-1734", - "frontier_total_units": 50, - "frontier_completed_units": 50, - "frontier_pct_complete": 1.0, - "bound_total_checks": 180, - "bound_completed_checks": 180, - "bound_pct_complete": 1.0, - "global_total_units": 230, - "global_completed_units": 230, - "global_pct_complete": 1.0, - "elapsed_sec": 0.17199999990407377, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 170464.5429284627 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_selection.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_selection.json deleted file mode 100644 index 16cd040..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_selection.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "generated_at_utc": "2026-06-07T19:41:21.942553+00:00", - "run_label": "rank1_alpha01_bound_aware_276k_full_2026-04-05-1734", - "conformal_intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "search_space": { - "risk_grid": [ - 0.155, - 0.16, - 0.165, - 0.17, - 0.175 - ], - "aversion_grid": [ - 0.0, - 0.05, - 0.1 - ], - "gamma_grid": [ - 0.45, - 0.5, - 0.55 - ], - "delta_cap_grid": [ - 1.0 - ], - "tail_focus_grid": [ - 1.0 - ], - "budget_profiles": [ - { - "name": "free_budget", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0 - } - ], - "alpha_grid": [ - 0.01, - 0.03, - 0.05, - 0.1 - ], - "max_candidates": 0, - "random_states": [ - 42 - ], - "policy_modes": [ - "blended_uncertainty" - ], - "bucket_return_k": 45, - "bucket_proxy_k": 45, - "bucket_family_k": 45, - "bucket_region_k": 45, - "incumbent_policy_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware/portfolio_bound_aware_selection.json", - "incumbent_risk_neighbors": [ - 0.155, - 0.16, - 0.165, - 0.17 - ], - "incumbent_gamma_neighbors": [ - 0.45, - 0.5, - 0.55 - ], - "incumbent_policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ] - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "price_of_robustness(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.175, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs" - }, - "selected_metrics": { - "risk_tolerance": 0.175, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "cuopt", - "seed_count": 1, - "sample_random_states": "42", - "ab_pass_all": true, - "ab_pass_rate": 1.0, - "realized_total_return": 170464.5429284627, - "realized_total_return_max": 170464.5429284627, - "price_of_robustness": -14465.688162534818, - "price_of_robustness_pct": -10.557772959426375, - "ab_diff_total_return": 12478.267770164326, - "objective_value": 143020.85670571326, - "n_funded": 340.0, - "total_allocated": 1000000.0, - "expected_return_net_point": 151480.26707061534, - "worst_case_pd": 0.27839279334831907, - "point_pd": 0.09040589635068147, - "pd_cap_slack": 0.0, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.45,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.175,\"solver_backend\":\"cuopt\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.1}", - "return_first_rank": 1, - "bound_proxy_rank": 43, - "shortlist_bucket": "conservative_proxy", - "candidate_rank": 43, - "alpha01_exact_pass": true, - "alpha01_pass_rate": 1.0, - "alpha01_gamma_cp": 0.187987, - "alpha01_weighted_miscoverage_V": 0.028875, - "alpha01_violation": 0.0, - "alpha01_weighted_pd_true": 0.032875, - "alpha01_weighted_pd_constraint_used": 0.175, - "alpha01_empirical_coverage_funded": 0.9501, - "alpha03_exact_pass": true, - "alpha03_weighted_miscoverage_V": 0.028875, - "alpha10_exact_pass": true, - "alpha10_weighted_miscoverage_V": 0.028875 - }, - "selection_reason": "selected_best_alpha01_exact_pass", - "frontier_raw_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier_raw.parquet", - "frontier_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_frontier.parquet", - "shortlist_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_shortlist.parquet", - "shortlist_exact_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_shortlist_exact.parquet", - "bound_eval_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_bound_eval.parquet", - "runtime_status_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_status.json", - "runtime_checkpoint_dir": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/portfolio_bound_aware_runtime_checkpoints", - "resource_snapshot_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/resource_snapshot.json", - "frontier_solver_backend": "cuopt", - "exact_solver_backend": "highs" -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/resource_snapshot.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/resource_snapshot.json deleted file mode 100644 index 3bd0c41..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/resource_snapshot.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "run_label": "rank1_alpha01_bound_aware_276k_full_2026-04-05-1734", - "solver_backend": "cuopt", - "exact_solver_backend": "highs", - "start": { - "captured_at_utc": "2026-04-05T22:35:13.272961+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 57492252, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 4025.0, - "memory_free_mb": 6029.0, - "gpu_util_pct": 13.0, - "memory_util_pct": 10.0 - } - }, - "backend_validation": { - "validated_at_utc": "2026-04-05T22:35:09.173764+00:00", - "python": "legacy-wsl-python-removed-after-windows-revalidation", - "expected_release": "26.02", - "cuopt": "26.02.00", - "cudf": "26.02.01", - "cupy": "14.0.1", - "pyomo": "6.10.0", - "loguru": "0.7.3" - }, - "end": { - "captured_at_utc": "2026-04-06T00:25:44.536090+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 54479012, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 3970.0, - "memory_free_mb": 6084.0, - "gpu_util_pct": 0.0, - "memory_util_pct": 6.0 - } - }, - "gpu_summary": { - "available": true, - "sample_count": 5904, - "csv_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/gpu_samples.csv", - "peak_gpu_util": 31.0, - "avg_gpu_util": 1.290819783197832, - "peak_memory_used_mb": 4098.0, - "avg_memory_used_mb": 3963.4336043360436, - "peak_power_draw_w": 121.85, - "avg_power_draw_w": 25.447101964769672 - }, - "exact_helper_python": ".venv/Scripts/python.exe", - "exact_helper_end": { - "captured_at_utc": "2026-06-07T19:41:21.965258+00:00", - "cpu_count": 24, - "memory_probe_error": "[Errno 2] No such file or directory: '\\\\proc\\\\meminfo'", - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 2529.0, - "memory_free_mb": 7525.0, - "gpu_util_pct": 30.0, - "memory_util_pct": 15.0 - } - } -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json deleted file mode 100644 index e4ebf88..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/001_frontier_complete.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "001_frontier_complete", - "updated_at_utc": "2026-04-05T21:00:41.284538+00:00", - "frontier_total_units": 844, - "frontier_completed_units": 844, - "frontier_pct_complete": 1.0, - "bound_total_checks": 0, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 844, - "global_completed_units": 844, - "global_pct_complete": 1.0, - "elapsed_sec": 550.1633127930108, - "eta_sec": 0.0, - "frontier_policy_count": 840, - "frontier_raw_rows": 840 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json deleted file mode 100644 index 6701978..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/002_shortlist_built.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "002_shortlist_built", - "updated_at_utc": "2026-04-05T21:00:41.310996+00:00", - "frontier_total_units": 844, - "frontier_completed_units": 844, - "frontier_pct_complete": 1.0, - "bound_total_checks": 640, - "bound_completed_checks": 0, - "bound_pct_complete": 0.0, - "global_total_units": 1484, - "global_completed_units": 844, - "global_pct_complete": 0.568733153638814, - "elapsed_sec": 550.1904231450171, - "eta_sec": 417.20600807205085, - "shortlist_size": 160, - "shortlist_buckets": { - "forced_incumbent_neighbors": 48, - "conservative_proxy": 40, - "incumbent_region": 20, - "family::blended_uncertainty": 20, - "family::capped_blended_uncertainty": 20, - "family::tail_blended_uncertainty": 12 - } -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json deleted file mode 100644 index b9d7ed8..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints/003_selection_complete.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "checkpoint_name": "003_selection_complete", - "updated_at_utc": "2026-04-05T21:07:48.861192+00:00", - "frontier_total_units": 844, - "frontier_completed_units": 844, - "frontier_pct_complete": 1.0, - "bound_total_checks": 640, - "bound_completed_checks": 640, - "bound_pct_complete": 1.0, - "global_total_units": 1484, - "global_completed_units": 1484, - "global_pct_complete": 1.0, - "elapsed_sec": 977.7407562879962, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 90529.78249984664 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_status.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_status.json deleted file mode 100644 index 9be76d8..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_status.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "stage_name": "portfolio_bound_aware", - "phase": "selection_complete", - "state": "completed", - "updated_at_utc": "2026-04-05T21:07:48.860947+00:00", - "run_tag": "rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548", - "frontier_total_units": 844, - "frontier_completed_units": 844, - "frontier_pct_complete": 1.0, - "bound_total_checks": 640, - "bound_completed_checks": 640, - "bound_pct_complete": 1.0, - "global_total_units": 1484, - "global_completed_units": 1484, - "global_pct_complete": 1.0, - "elapsed_sec": 977.7407562879962, - "eta_sec": 0.0, - "selection_reason": "selected_best_alpha01_exact_pass", - "selected_alpha01_exact_pass": true, - "selected_realized_total_return": 90529.78249984664 -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_selection.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_selection.json deleted file mode 100644 index a8cb072..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_selection.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "generated_at_utc": "2026-04-05T21:07:48.769488+00:00", - "run_label": "rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548", - "conformal_intervals_path": "data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "search_space": { - "risk_grid": [ - 0.155, - 0.16, - 0.165, - 0.17 - ], - "aversion_grid": [ - 0.0, - 0.05, - 0.1 - ], - "gamma_grid": [ - 0.4, - 0.45, - 0.5, - 0.55, - 0.6 - ], - "delta_cap_grid": [ - 0.75, - 1.0 - ], - "tail_focus_grid": [ - 0.85, - 0.9, - 0.95, - 1.0 - ], - "budget_profiles": [ - { - "name": "free_budget", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0 - }, - { - "name": "floored_budget", - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5 - } - ], - "alpha_grid": [ - 0.01, - 0.03, - 0.05, - 0.1 - ], - "max_candidates": 5000, - "random_states": [ - 42 - ], - "policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty", - "tail_blended_uncertainty" - ], - "bucket_return_k": 40, - "bucket_proxy_k": 40, - "bucket_family_k": 20, - "bucket_region_k": 20, - "incumbent_policy_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware/portfolio_bound_aware_selection.json", - "incumbent_risk_neighbors": [ - 0.155, - 0.16, - 0.165, - 0.17 - ], - "incumbent_gamma_neighbors": [ - 0.45, - 0.5, - 0.55 - ], - "incumbent_policy_modes": [ - "blended_uncertainty", - "capped_blended_uncertainty" - ] - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "price_of_robustness(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.16, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs" - }, - "selected_metrics": { - "risk_tolerance": 0.16, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs", - "seed_count": 1, - "sample_random_states": "42", - "ab_pass_all": true, - "ab_pass_rate": 1.0, - "realized_total_return": 90529.78249984664, - "realized_total_return_max": 90529.78249984664, - "price_of_robustness": -10261.651925194703, - "price_of_robustness_pct": -13.641597411213333, - "ab_diff_total_return": 21255.789106640936, - "objective_value": 85484.8945978106, - "n_funded": 153.0, - "total_allocated": 1000000.0, - "expected_return_net_point": 85484.89459781056, - "worst_case_pd": 0.24749616044577177, - "point_pd": 0.07250383955422825, - "pd_cap_slack": 0.0, - "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.5,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.16,\"solver_backend\":\"highs\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0}", - "return_first_rank": 137, - "bound_proxy_rank": 249, - "shortlist_bucket": "forced_incumbent_neighbors", - "candidate_rank": 17, - "alpha01_exact_pass": true, - "alpha01_pass_rate": 1.0, - "alpha01_gamma_cp": 0.169994, - "alpha01_weighted_miscoverage_V": 0.077725, - "alpha01_violation": 0.0, - "alpha01_weighted_pd_true": 0.077725, - "alpha01_weighted_pd_constraint_used": 0.16, - "alpha01_empirical_coverage_funded": 0.9313, - "alpha03_exact_pass": true, - "alpha03_weighted_miscoverage_V": 0.077725, - "alpha10_exact_pass": true, - "alpha10_weighted_miscoverage_V": 0.097166 - }, - "selection_reason": "selected_best_alpha01_exact_pass", - "frontier_raw_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_frontier_raw.parquet", - "frontier_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_frontier.parquet", - "shortlist_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_shortlist.parquet", - "bound_eval_path": "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_bound_eval.parquet", - "runtime_status_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_status.json", - "runtime_checkpoint_dir": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/portfolio_bound_aware_runtime_checkpoints", - "resource_snapshot_path": "models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/resource_snapshot.json", - "frontier_solver_backend": "highs", - "exact_solver_backend": "highs" -} diff --git a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/resource_snapshot.json b/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/resource_snapshot.json deleted file mode 100644 index 667b70b..0000000 --- a/models/portfolio_bound_aware/rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548/resource_snapshot.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "schema_version": "2026-04-05.2", - "run_label": "rank1_alpha01_bound_aware_5k_corrected_2026-04-05-1548", - "solver_backend": "highs", - "exact_solver_backend": "highs", - "start": { - "captured_at_utc": "2026-04-05T20:51:31.120186+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 57671608, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 3802.0, - "memory_free_mb": 6252.0, - "gpu_util_pct": 19.0, - "memory_util_pct": 12.0 - } - }, - "backend_validation": null, - "end": { - "captured_at_utc": "2026-04-05T21:07:48.789299+00:00", - "cpu_count": 24, - "memory_total_kib": 61714552, - "memory_available_kib": 56895968, - "swap_free_kib": 25165824, - "gpu": { - "name": "NVIDIA GeForce RTX 3080", - "driver_version": "595.71", - "memory_total_mb": 10240.0, - "memory_used_mb": 3749.0, - "memory_free_mb": 6305.0, - "gpu_util_pct": 1.0, - "memory_util_pct": 28.0 - } - } -} diff --git a/models/portfolio_optimization_last_valid_artifact.json b/models/portfolio_optimization_last_valid_artifact.json deleted file mode 100644 index 379b922..0000000 --- a/models/portfolio_optimization_last_valid_artifact.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "stage_name": "portfolio_optimization", - "artifact_key": "portfolio_allocations", - "artifact_path": "data\\processed\\portfolio_allocations.parquet", - "validated_at_utc": "2026-06-07T22:36:27.085451+00:00", - "solver_status": "TerminationCondition.optimal", - "n_candidates_used": 5000, - "objective_value": 66101.82798271991 -} diff --git a/models/portfolio_optimization_runtime_status.json b/models/portfolio_optimization_runtime_status.json deleted file mode 100644 index 043df2e..0000000 --- a/models/portfolio_optimization_runtime_status.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "stage_name": "portfolio_optimization", - "phase": "completed", - "state": "completed", - "updated_at_utc": "2026-06-07T22:36:27.086451+00:00", - "portfolio_results_path": "models\\portfolio_results.pkl", - "portfolio_allocations_path": "data\\processed\\portfolio_allocations.parquet", - "solver_status": "TerminationCondition.optimal" -} diff --git a/models/portfolio_research_policy.json b/models/portfolio_research_policy.json deleted file mode 100644 index 9740e48..0000000 --- a/models/portfolio_research_policy.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "schema_version": "2026-03-08.1", - "generated_at_utc": "2026-04-07T16:09:24.688621+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "selection_stage": "tradeoff_research_only", - "selection_policy": { - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "research_selection_policy": { - "name": "robustness_aware", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return * sqrt(gamma) * (1 + 0.05 * min(lambda, 1))", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.12, - "policy_mode": "segment_tail_blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 94362.84590012347, - "realized_total_return": 146200.01509477518, - "price_of_robustness": -85890.61170733956, - "price_of_robustness_pct": -113.4342685989363, - "n_funded": 284 - }, - "selected_policy_robustness_aware": { - "risk_tolerance": 0.12, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selected_policy_balanced_robustness": { - "risk_tolerance": 0.12, - "policy_mode": "segment_tail_blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "research_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 93586.60884040719, - "realized_total_return": 145423.7780350589, - "price_of_robustness": -86239.44879177192, - "price_of_robustness_pct": -113.89497179741394, - "n_funded": 286, - "robustness_aware_score": 102830.13959435746 - }, - "balanced_selection_policy": { - "name": "balanced_robustness", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)", - "price_of_robustness(asc)" - ] - }, - "balanced_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 94362.84590012347, - "realized_total_return": 146200.01509477518, - "price_of_robustness": -85890.61170733956, - "price_of_robustness_pct": -113.4342685989363, - "n_funded": 284 - }, - "guardrail_selection_policy": { - "name": "guardrail_robustness", - "constraints": { - "gamma_gt_zero": true, - "ab_pass_frontier": true, - "price_of_robustness_pct_gte": -25.0 - }, - "fallback": "balanced_robustness", - "rank_order": [ - "realized_total_return(desc)", - "price_of_robustness_pct(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)" - ] - }, - "selected_policy_guardrail_robustness": { - "risk_tolerance": 0.08, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.5, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "guardrail_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 41808.37311607883, - "realized_total_return": 57744.15169060486, - "price_of_robustness": -15804.500551269535, - "price_of_robustness_pct": -24.63614666364077, - "n_funded": 150 - }, - "frontier_path": "data/processed/portfolio_robustness_frontier.parquet", - "summary_path": "data/processed/portfolio_robustness_summary.parquet", - "candidate_universe_path": "data/processed/champion_candidate_universe.parquet" -} diff --git a/models/portfolio_tradeoff/conformal-finalist-rank2_grade_cal_bins10_mgs100/portfolio_research_policy.json b/models/portfolio_tradeoff/conformal-finalist-rank2_grade_cal_bins10_mgs100/portfolio_research_policy.json deleted file mode 100644 index 3e6bad1..0000000 --- a/models/portfolio_tradeoff/conformal-finalist-rank2_grade_cal_bins10_mgs100/portfolio_research_policy.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "schema_version": "2026-03-08.1", - "generated_at_utc": "2026-04-05T17:35:31.707761+00:00", - "run_tag": "portfolio-finalist-rank2_grade_cal_bins10_mgs100", - "selection_stage": "tradeoff_research_only", - "selection_policy": { - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "research_selection_policy": { - "name": "robustness_aware", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return * sqrt(gamma) * (1 + 0.05 * min(lambda, 1))", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.1, - "policy_mode": "blended_uncertainty", - "gamma": 0.0, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138 - }, - "selected_policy_robustness_aware": { - "risk_tolerance": 0.1, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selected_policy_balanced_robustness": { - "risk_tolerance": 0.1, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "research_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138, - "robustness_aware_score": 82788.33585897704 - }, - "balanced_selection_policy": { - "name": "balanced_robustness", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)", - "price_of_robustness(asc)" - ] - }, - "balanced_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138 - }, - "guardrail_selection_policy": { - "name": "guardrail_robustness", - "constraints": { - "gamma_gt_zero": true, - "ab_pass_frontier": true, - "price_of_robustness_pct_gte": -25.0 - }, - "fallback": "balanced_robustness", - "rank_order": [ - "realized_total_return(desc)", - "price_of_robustness_pct(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)" - ] - }, - "selected_policy_guardrail_robustness": { - "risk_tolerance": 0.12, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 0.5, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "guardrail_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 6337.662764617533, - "realized_total_return": 74383.01170844692, - "price_of_robustness": -10763.261550604082, - "price_of_robustness_pct": -17.388718601417033, - "n_funded": 110 - }, - "frontier_path": "data/processed/portfolio_tradeoff/conformal-finalist-rank2_grade_cal_bins10_mgs100/portfolio_robustness_frontier.parquet", - "summary_path": "data/processed/portfolio_tradeoff/conformal-finalist-rank2_grade_cal_bins10_mgs100/portfolio_robustness_summary.parquet", - "candidate_universe_path": "data/processed/champion_candidate_universe.parquet" -} diff --git a/models/portfolio_tradeoff/conformal-finalist-rank3_grade_cal_bins10_mgs1000/portfolio_research_policy.json b/models/portfolio_tradeoff/conformal-finalist-rank3_grade_cal_bins10_mgs1000/portfolio_research_policy.json deleted file mode 100644 index 352b6d5..0000000 --- a/models/portfolio_tradeoff/conformal-finalist-rank3_grade_cal_bins10_mgs1000/portfolio_research_policy.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "schema_version": "2026-03-08.1", - "generated_at_utc": "2026-04-05T17:41:26.768134+00:00", - "run_tag": "portfolio-finalist-rank3_grade_cal_bins10_mgs1000", - "selection_stage": "tradeoff_research_only", - "selection_policy": { - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "research_selection_policy": { - "name": "robustness_aware", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return * sqrt(gamma) * (1 + 0.05 * min(lambda, 1))", - "realized_total_return(desc)", - "price_of_robustness(asc)" - ] - }, - "selected_policy": { - "risk_tolerance": 0.1, - "policy_mode": "blended_uncertainty", - "gamma": 0.0, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138 - }, - "selected_policy_robustness_aware": { - "risk_tolerance": 0.1, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "selected_policy_balanced_robustness": { - "risk_tolerance": 0.1, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "research_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138, - "robustness_aware_score": 82788.33585897704 - }, - "balanced_selection_policy": { - "name": "balanced_robustness", - "rank_order": [ - "ab_pass(desc)", - "realized_total_return(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)", - "price_of_robustness(asc)" - ] - }, - "balanced_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 57130.50326085591, - "realized_total_return": 117080.38737806416, - "price_of_robustness": -37145.285838212556, - "price_of_robustness_pct": -66.20104077714483, - "n_funded": 138 - }, - "guardrail_selection_policy": { - "name": "guardrail_robustness", - "constraints": { - "gamma_gt_zero": true, - "ab_pass_frontier": true, - "price_of_robustness_pct_gte": -25.0 - }, - "fallback": "balanced_robustness", - "rank_order": [ - "realized_total_return(desc)", - "price_of_robustness_pct(desc)", - "gamma(desc)", - "uncertainty_aversion(asc)" - ] - }, - "selected_policy_guardrail_robustness": { - "risk_tolerance": 0.12, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 0.5, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "solver_backend": "highs" - }, - "guardrail_selection_metrics": { - "ab_pass": true, - "ab_diff_total_return": 6337.662764617533, - "realized_total_return": 74383.01170844692, - "price_of_robustness": -10763.261550604082, - "price_of_robustness_pct": -17.388718601417033, - "n_funded": 110 - }, - "frontier_path": "data/processed/portfolio_tradeoff/conformal-finalist-rank3_grade_cal_bins10_mgs1000/portfolio_robustness_frontier.parquet", - "summary_path": "data/processed/portfolio_tradeoff/conformal-finalist-rank3_grade_cal_bins10_mgs1000/portfolio_robustness_summary.parquet", - "candidate_universe_path": "data/processed/champion_candidate_universe.parquet" -} diff --git a/models/portfolio_tradeoff_last_valid_artifact.json b/models/portfolio_tradeoff_last_valid_artifact.json deleted file mode 100644 index 767db28..0000000 --- a/models/portfolio_tradeoff_last_valid_artifact.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "stage_name": "portfolio_tradeoff", - "artifact_key": "portfolio_robustness_summary", - "artifact_path": "data/processed/portfolio_robustness_summary.parquet", - "validated_at_utc": "2026-04-07T16:09:24.690607+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "frontier_rows": 570, - "summary_rows": 3, - "selected_policy_mode": "segment_tail_blended_uncertainty", - "artifact_namespace": "" -} diff --git a/models/portfolio_tradeoff_runtime_status.json b/models/portfolio_tradeoff_runtime_status.json deleted file mode 100644 index e5c5e4c..0000000 --- a/models/portfolio_tradeoff_runtime_status.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "stage_name": "portfolio_tradeoff", - "phase": "completed", - "state": "completed", - "updated_at_utc": "2026-04-07T16:09:24.690876+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "frontier_path": "data/processed/portfolio_robustness_frontier.parquet", - "summary_path": "data/processed/portfolio_robustness_summary.parquet", - "research_policy_path": "models/portfolio_research_policy.json", - "n_candidates_used": 80000, - "artifact_namespace": "" -} diff --git a/models/spo_comparison_status.json b/models/spo_comparison_status.json deleted file mode 100644 index 58539a8..0000000 --- a/models/spo_comparison_status.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "generated_at_utc": "2026-03-17T13:51:52.567738+00:00", - "run_tag": "spo-comparison-2026-03-17-135148", - "sample_size": 50000, - "prediction_column": "y_prob_final", - "two_stage": { - "approach": "two_stage_point", - "rank_regret": 0.2861425024, - "mae": 0.31006220505671045, - "description": "Point PD estimate, optimize directly on predictions" - }, - "robust_conformal": { - "approach": "robust_conformal", - "rank_regret": 0.2988450904, - "mae_point": 0.31006220505671045, - "mae_high": 0.5729341047503984, - "worst_case_coverage": 0.92328, - "description": "Conformal pd_high as worst-case, optimize conservatively" - }, - "spo_plus": { - "approach": "spo_plus", - "status": "capability_available", - "reason": "pyepo and torch installed; full training requires optmodel setup", - "description": "SPO+ trains neural net with decision-focused loss (Elmachtoub & Grigas 2022)", - "reference_code": "src/optimization/spo_integration.py" - }, - "portfolio_economics": { - "tol_0.05": { - "risk_tolerance": 0.05, - "robust_return": 127965.77482191283, - "nonrobust_return": 1.77860275305654e-07, - "price_of_robustness": -127965.77482173497, - "price_of_robustness_pct": -10864257629244.516 - }, - "tol_0.06": { - "risk_tolerance": 0.06, - "robust_return": 142523.17192359804, - "nonrobust_return": 4.843798021980999e-09, - "price_of_robustness": -142523.1719235932, - "price_of_robustness_pct": -14183614627880.256 - }, - "tol_0.07": { - "risk_tolerance": 0.07, - "robust_return": 153667.5222482712, - "nonrobust_return": 1.0475260551701673e-07, - "price_of_robustness": -153667.52224816647, - "price_of_robustness_pct": -13909677287092.807 - }, - "tol_0.08": { - "risk_tolerance": 0.08, - "robust_return": 164116.9927100543, - "nonrobust_return": 5.8359038880871145e-09, - "price_of_robustness": -164116.99271004845, - "price_of_robustness_pct": -16316477874338.11 - }, - "tol_0.09": { - "risk_tolerance": 0.09, - "robust_return": 174190.41415113155, - "nonrobust_return": 1.1114099521228605e-07, - "price_of_robustness": -174190.4141510204, - "price_of_robustness_pct": -15676715637491.256 - }, - "tol_0.10": { - "risk_tolerance": 0.1, - "robust_return": 183071.3499921966, - "nonrobust_return": 3.88537264920913e-11, - "price_of_robustness": -183071.34999219657, - "price_of_robustness_pct": -18306423726439.14 - }, - "tol_0.12": { - "risk_tolerance": 0.12, - "robust_return": 197319.5040261472, - "nonrobust_return": 7.6950807995654e-09, - "price_of_robustness": -197319.5040261395, - "price_of_robustness_pct": -19581270940568.1 - }, - "tol_0.14": { - "risk_tolerance": 0.14, - "robust_return": 204462.5428333754, - "nonrobust_return": 1.5543262806369112e-07, - "price_of_robustness": -204462.54283321995, - "price_of_robustness_pct": -17695756365809.44 - }, - "tol_0.16": { - "risk_tolerance": 0.16, - "robust_return": 207398.26797579072, - "nonrobust_return": -1.4359604251072927e-08, - "price_of_robustness": -207398.26797580507, - "price_of_robustness_pct": -20446227068449.992 - }, - "tol_0.18": { - "risk_tolerance": 0.18, - "robust_return": 207098.72793462727, - "nonrobust_return": 1.3635021966207329e-08, - "price_of_robustness": -207098.72793461365, - "price_of_robustness_pct": -20431291682571.516 - }, - "tol_0.20": { - "risk_tolerance": 0.2, - "robust_return": 207550.1780223529, - "nonrobust_return": -2.5684107366079056e-07, - "price_of_robustness": -207550.17802260973, - "price_of_robustness_pct": -16513637433735.361 - } - }, - "summary": { - "two_stage_rank_regret": 0.2861425024, - "robust_rank_regret": 0.2988450904, - "robust_vs_two_stage_pct": -4.4392524331261365 - } -} diff --git a/models/threshold_semantics.json b/models/threshold_semantics.json deleted file mode 100644 index 9efc564..0000000 --- a/models/threshold_semantics.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-06-07T22:28:29.733367+00:00", - "run_tag": "ijds-rebaseline-2026-06-07", - "pd_internal_selected_threshold": 0.05, - "pd_internal_fallback_threshold": 0.35, - "fairness_primary_threshold": 0.35, - "decision_policy_global_threshold": 0.35, - "source_artifacts": { - "fairness_status": "models/fairness_audit_status.json", - "fairness_decision_policy": "models/fairness_decision_policy.json", - "fairness_frontier": "data/processed/fairness_threshold_frontier.parquet", - "decision_threshold": "models/decision_threshold.json", - "decision_threshold_v2": "models/decision_threshold_v2.json" - }, - "business_meaning": { - "pd_internal_selected_threshold": "Threshold interno de screening/seleccion PD usado en busqueda y analisis tecnico.", - "pd_internal_fallback_threshold": "Fallback interno tomado de la policy de fairness durante la busqueda de threshold PD.", - "fairness_primary_threshold": "Threshold operativo principal para auditoria de fairness y narrativa de aprobacion.", - "decision_policy_global_threshold": "Threshold operativo global aplicado por la policy de decision/aprobacion." - }, - "pd_internal_threshold_source": "validation_fairness_search", - "calibration_method": "venn_abers" -} diff --git a/monitor_regret_auditability.cmd b/monitor_regret_auditability.cmd deleted file mode 100644 index 7586488..0000000 --- a/monitor_regret_auditability.cmd +++ /dev/null @@ -1,20 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "REPO_ROOT=%~dp0" -set "DEFAULT_ROOT=D:\crpto_experiments\regret_auditability\regret_auditability_20260513_v3_resource_tuned" - -set "ARTIFACT_ROOT=%~1" -if "%ARTIFACT_ROOT%"=="" if not "%CRPTO_SANDBOX_ROOT%"=="" set "ARTIFACT_ROOT=%CRPTO_SANDBOX_ROOT%" -if "%ARTIFACT_ROOT%"=="" set "ARTIFACT_ROOT=%DEFAULT_ROOT%" - -set "INTERVAL=%~2" -if "%INTERVAL%"=="" set "INTERVAL=60" - -set "ONCE_ARG=" -if /I "%~3"=="once" set "ONCE_ARG=--once" - -set "PYTHON_EXE=%REPO_ROOT%.venv\Scripts\python.exe" -if not exist "%PYTHON_EXE%" set "PYTHON_EXE=python" - -"%PYTHON_EXE%" "%REPO_ROOT%scripts\search\monitor_regret_auditability.py" --artifact-root "%ARTIFACT_ROOT%" --interval "%INTERVAL%" %ONCE_ARG% diff --git a/notebooks/10_crpto_cp_robust_opt.ipynb b/notebooks/10_crpto_cp_robust_opt.ipynb deleted file mode 100644 index 7f24c63..0000000 --- a/notebooks/10_crpto_cp_robust_opt.ipynb +++ /dev/null @@ -1,271 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# 10) CRPTO - Conformal Prediction + Robust Optimization\n", - "\n", - "Notebook de soporte para el paper:\n", - "- **Objetivo**: materializar tablas y figuras publicables de la frontera robusta + benchmark conformal.\n", - "- **Salidas**: `reports/crpto/figures/` y `reports/crpto/tables/`.\n", - "\n", - "Estructura alineada a paper (IMRaD+):\n", - "1. Setup y datos\n", - "2. Métricas clave\n", - "3. Figuras principales\n", - "4. Tablas exportables\n", - "5. Notas de validez y reproducibilidad" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "> **Nota de extraccion CRPTO (2026-03-17)** — Este notebook corresponde al **CRPTO original** (CP + Robust Opt, venue EJOR standalone).\n", - "> CRPTO queda consolidado como **CRPTO**,\n", - "> que integra la contribución de CRPTO con el bound teórico α↔Γ (Bertsimas & Sim),\n", - "> la comparación SPO+ regret, y los baselines de uncertainty sets.\n", - "> Este notebook se mantiene como **borrador histórico** de las figuras y tablas originales.\n", - "> Para el material actualizado y publicable, ver `notebooks/12_crpto_predict_optimize.ipynb` (si existe) o la página Streamlit correspondiente." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "\n", - "import json\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import plotly.express as px\n", - "import plotly.graph_objects as go\n", - "import plotly.io as pio\n", - "\n", - "pio.templates.default = \"plotly_white\"\n", - "\n", - "PROJECT_ROOT = (\n", - " Path.cwd().resolve().parent if Path.cwd().name == \"notebooks\" else Path.cwd().resolve()\n", - ")\n", - "DATA_DIR = PROJECT_ROOT / \"data\" / \"processed\"\n", - "MODEL_DIR = PROJECT_ROOT / \"models\"\n", - "\n", - "\n", - "def load_parquet(name: str) -> pd.DataFrame:\n", - " return pd.read_parquet(DATA_DIR / f\"{name}.parquet\")\n", - "\n", - "\n", - "def load_json(name: str, from_models: bool = False) -> dict:\n", - " base = MODEL_DIR if from_models else DATA_DIR\n", - " return json.loads((base / f\"{name}.json\").read_text())\n", - "\n", - "\n", - "def ensure_dirs(base: Path) -> dict[str, Path]:\n", - " dirs = {\n", - " \"base\": base,\n", - " \"fig\": base / \"figures\",\n", - " \"tbl\": base / \"tables\",\n", - " }\n", - " for d in dirs.values():\n", - " d.mkdir(parents=True, exist_ok=True)\n", - " return dirs\n", - "\n", - "\n", - "def export_figure(fig: go.Figure, stem: str, out_fig_dir: Path) -> None:\n", - " html_path = out_fig_dir / f\"{stem}.html\"\n", - " fig.write_html(html_path)\n", - " try:\n", - " png_path = out_fig_dir / f\"{stem}.png\"\n", - " fig.write_image(png_path, width=1400, height=850, scale=2)\n", - " print(f\"Saved: {html_path} and {png_path}\")\n", - " except Exception as exc:\n", - " print(f\"Saved HTML only ({html_path}). PNG skipped: {exc}\")\n", - "\n", - "\n", - "def export_table(df: pd.DataFrame, stem: str, out_tbl_dir: Path, max_rows: int = 2000) -> None:\n", - " csv_path = out_tbl_dir / f\"{stem}.csv\"\n", - " tex_path = out_tbl_dir / f\"{stem}.tex\"\n", - " out_df = df.copy().head(max_rows)\n", - " out_df.to_csv(csv_path, index=False)\n", - " try:\n", - " latex = out_df.to_latex(index=False, escape=False)\n", - " tex_path.write_text(latex, encoding=\"utf-8\")\n", - " print(f\"Saved: {csv_path} and {tex_path}\")\n", - " except Exception as exc:\n", - " print(f\"Saved CSV only ({csv_path}). LaTeX skipped: {exc}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "out = ensure_dirs(PROJECT_ROOT / \"reports\" / \"paper_material\" / \"crpto\")\n", - "pipeline_summary = load_json(\"pipeline_summary\")\n", - "model_comparison = load_json(\"model_comparison\")\n", - "conformal_status = load_json(\"conformal_policy_status\", from_models=True)\n", - "robust_summary = load_parquet(\"portfolio_robustness_summary\")\n", - "robust_frontier = load_parquet(\"portfolio_robustness_frontier\")\n", - "variant_benchmark = load_parquet(\"conformal_variant_benchmark\")\n", - "variant_by_group = load_parquet(\"conformal_variant_benchmark_by_group\")\n", - "print(\"Loaded shapes:\")\n", - "print(\"robust_summary\", robust_summary.shape)\n", - "print(\"robust_frontier\", robust_frontier.shape)\n", - "print(\"variant_benchmark\", variant_benchmark.shape)\n", - "print(\"variant_by_group\", variant_by_group.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "pipeline = pipeline_summary.get(\"pipeline\", {})\n", - "pd_metrics = model_comparison.get(\"final_test_metrics\", {})\n", - "metrics = pd.DataFrame(\n", - " [\n", - " {\"metric\": \"pd_auc\", \"value\": pd_metrics.get(\"auc_roc\", np.nan)},\n", - " {\"metric\": \"coverage_90\", \"value\": conformal_status.get(\"coverage_90\", np.nan)},\n", - " {\"metric\": \"coverage_95\", \"value\": conformal_status.get(\"coverage_95\", np.nan)},\n", - " {\"metric\": \"avg_width_90\", \"value\": conformal_status.get(\"avg_width_90\", np.nan)},\n", - " {\"metric\": \"robust_return\", \"value\": pipeline.get(\"robust_return\", np.nan)},\n", - " {\"metric\": \"nonrobust_return\", \"value\": pipeline.get(\"nonrobust_return\", np.nan)},\n", - " {\"metric\": \"price_of_robustness\", \"value\": pipeline.get(\"price_of_robustness\", np.nan)},\n", - " ]\n", - ")\n", - "metrics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# Figure 1: robust vs non-robust return by risk tolerance\n", - "plot_df = robust_summary.melt(\n", - " id_vars=[\"risk_tolerance\"],\n", - " value_vars=[\"baseline_nonrobust_return\", \"best_robust_return\"],\n", - " var_name=\"policy\",\n", - " value_name=\"return_net\",\n", - ")\n", - "fig1 = px.bar(\n", - " plot_df,\n", - " x=\"risk_tolerance\",\n", - " y=\"return_net\",\n", - " color=\"policy\",\n", - " barmode=\"group\",\n", - " title=\"CRPTO-Fig1: Net Return by Risk Tolerance\",\n", - ")\n", - "fig1\n", - "export_figure(fig1, \"crpto_fig1_return_by_tolerance\", out[\"fig\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Figure 2: price of robustness (%)\n", - "fig2 = px.line(\n", - " robust_summary,\n", - " x=\"risk_tolerance\",\n", - " y=\"price_of_robustness_pct\",\n", - " markers=True,\n", - " title=\"CRPTO-Fig2: Price of Robustness (%)\",\n", - ")\n", - "fig2\n", - "export_figure(fig2, \"crpto_fig2_price_of_robustness_pct\", out[\"fig\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "# Figure 3: conformal benchmark trade-off\n", - "fig3 = px.scatter(\n", - " variant_benchmark,\n", - " x=\"avg_width\",\n", - " y=\"min_group_coverage\",\n", - " color=\"variant\",\n", - " size=\"coverage\",\n", - " hover_data=[\"coverage_gap\", \"std_group_coverage\"],\n", - " title=\"CRPTO-Fig3: Width vs Min Group Coverage\",\n", - ")\n", - "fig3\n", - "export_figure(fig3, \"crpto_fig3_conformal_tradeoff\", out[\"fig\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "# Export core tables for manuscript\n", - "export_table(metrics, \"crpto_table0_key_metrics\", out[\"tbl\"])\n", - "export_table(robust_summary, \"crpto_table1_robustness_summary\", out[\"tbl\"])\n", - "export_table(variant_benchmark, \"crpto_table2_conformal_variant_benchmark\", out[\"tbl\"])\n", - "export_table(variant_by_group, \"crpto_tableA1_benchmark_by_group\", out[\"tbl\"])\n", - "export_table(robust_frontier, \"crpto_tableA2_robustness_frontier\", out[\"tbl\"], max_rows=5000)" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "## Threats to Validity (draft)\n", - "- Sensibilidad a restricciones de optimización y configuración de penalizaciones.\n", - "- Riesgo de extrapolación fuera del dominio Lending Club.\n", - "- Cobertura conformal y utilidad económica no son equivalentes; se deben reportar ambos ejes.\n", - "\n", - "## Reproducibilidad\n", - "```bash\n", - "uv run dvc repro generate_conformal benchmark_conformal_variants optimize_portfolio optimize_portfolio_tradeoff\n", - "uv run pytest -q\n", - "```" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/book/.gitignore b/paper/.gitignore similarity index 100% rename from book/.gitignore rename to paper/.gitignore diff --git a/paper/CRPTO.qmd b/paper/CRPTO.qmd index 29ca30e..249fabc 100644 --- a/paper/CRPTO.qmd +++ b/paper/CRPTO.qmd @@ -1,60 +1,30 @@ --- -title: "CRPTO" -author: "Carlos Alfredo Vergara Rojas" -lang: es -bibliography: ../book/references.bib -format: - html: - toc: true - pdf: - documentclass: article -execute: - freeze: true +title: "CRPTO Manuscript Index" +author: "Anonymous" +lang: en +format: html --- -# Resumen +# Active IJDS Manuscript -CRPTO convierte una PD calibrada en una decision de portafolio auditable. La -version IJDS activa reproduce exactamente el endpoint conformal al 90%, usa la -regla simple `q=(p+u)/2` con `tau=0.17`, y selecciona esa politica dentro de una -grilla `3x3` en noviembre bajo `B_u<=0.28`. Un replay outcome-free en diciembre -elige la misma politica; al abrir outcomes, la miscoverage `0.124925` muestra -que estabilidad de seleccion no equivale a cobertura del funded set. +The single active paper is +`CRPTO_ijds.qmd`, with online supplement `supplement_ijds.qmd` and official +INFORMS source `submission/CRPTO_ijds_submission.tex`. -En 276,869 prestamos OOT, la politica financia 308 prestamos y obtiene -`$179,327.59` sobre `$1M`, con default ponderado `0.039375`, miscoverage -`0.036875` y endpoint `0.258051`. La baseline point-PD emparejada obtiene -`$196,369.14` y default `0.118400`. El costo observado de retorno es `8.678%` y -la reduccion de default es `7.9025` puntos porcentuales. El paper reporta tambien -los periodos donde point-PD domina; no afirma superioridad universal. +The active claim registry is +`../docs/research/active_claims_2026-07-10.md`. The experiment uses a +status-independent 540,121-loan universe, a 2012-selected conformal upper-score +guardrail, and 15 separate monthly 2016--2017 decisions. It finds a genuine +trade-off: lower default, lower standardized payoff, and worse funded-set +miscoverage relative to point PD. -# Fuentes del manuscrito +Compact-v7, A1--A40, and external-dataset experiments are historical +provenance. They do not define or validate the active paper. -- `CRPTO_ijds.qmd`: body anonimo y fuente narrativa activa. -- `supplement_ijds.qmd`: pruebas y evidencia A35--A40, mas diagnosticos - historicos A1--A34. -- `submission/CRPTO_ijds_submission.tex`: handoff compacto bajo `informs4`. -- `../docs/research/active_claims_2026-07-04.md`: contrato numerico vigente. - -# Evidencia activa - -- A35: replay exacto y saturacion por alpha. -- A36: selector de noviembre, estabilidad del cap y auditoria de diciembre. -- A37: evaluacion OOT total y temporal. -- A38: composicion por grado de credito. -- A39: bootstrap por mes de originacion y sensibilidad por prestamo. -- A40: politica seleccionada, blend conservador y baseline point-PD. - -La gobernanza vive en -`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. -Los artefactos historicos protegidos por `EXTRACTION_MANIFEST.json` permanecen -intactos como procedencia, no como claims activos. - -# Comandos +# Build ```powershell just ijds-evidence just paper-submission just paper-submission-official -just submission-check ``` diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index e1dd100..40330fe 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -1,10 +1,46 @@ --- -title: "CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" +title: "CRPTO: An Identification Audit of Binary Conformal Credit Portfolio Optimization" author: "Anonymous" -date: today +date: "July 2026" +abstract: | + How should predictive uncertainty be audited after it becomes a constraint + coefficient in an optimizer? Conformal Risk-Aware Predict-Then-Optimize + (CRPTO) combines a Platt-scaled credit-default score, 90%-target binary + absolute-residual intervals, and monthly loan-allocation linear programs. Our + retrospectively locked design uses all 640,543 eligible 36-month loans in the + declared temporal roles, fixes score strata, reports eight complete residual + windows, constructs status-independent menus, and freezes allocations before + reading evaluation outcomes. Because the archive is not a verified + point-in-time snapshot, we reconstruct availability at September 30, 2020 and + retain 12,076 of 376,890 primary candidates as unresolved. Under the declared + six-month rule, all 40 model-window sharp coverage upper bounds are below 0.90 + across five credit-risk specifications (maximum 0.8977). The CatBoost pattern + recurs under three missing-value encodings and one later retrospective origin. + Four declared fitting-label scenarios for 215 unavailable labels also leave + all 32 scenario-window upper bounds below 0.90. A derived binary threshold + mechanism is consistent with an observed W7--W8 change in interval geometry, + but the all-default scenario removes that crossing; the mechanism is + diagnostic, not universal. + For decisions, two outcome-blind common-frontier rulers and three coordinates + compare the point and full-upper scores. Sharp common-outcome bounds, an exact + width identity, and basis enumeration show that direction depends on the + comparator: all 216 broad point-cap envelopes include zero, and no ruler, + coordinate, or policy is selected. Deterministic USD 25 rounding changes + evaluated rates by at most 0.0013 percentage points. CRPTO contributes a + reproducible audit of the machine-learning--conformal--optimization interface: + coverage transport, interval geometry, unresolved outcomes, and comparator + design must be distinguished before making a portfolio claim. +keywords: + - conformal prediction + - predict-then-optimize + - comparator design + - credit risk + - portfolio selection + - temporal transport + - partial identification lang: en -bibliography: ../book/references.bib -csl: ../book/apa.csl +bibliography: references.bib +csl: apa.csl link-citations: true format: html: @@ -23,521 +59,1081 @@ execute: warning: false --- -# Abstract - -Credit models matter only through the decisions they change. We study how -finite-sample predictive uncertainty can constrain a loan portfolio after a -probability-of-default (PD) model has been frozen. Conformal Robust -Predict-Then-Optimize (CRPTO) recomputes a 90% Mondrian conformal upper endpoint -exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places -$q_i$ in a portfolio-risk constraint while retaining point PD in the economic -objective. Nine round-number policies are ranked on November 2017 without -default, realized-return, miscoverage, or assumption-conditional columns; an -outcome-free December replay selects the same rule before its outcomes are -opened. That audit misses nominal funded-set coverage, making the statistical -boundary observable rather than rhetorical. The policy is then replayed on -276,869 out-of-time Lending Club loans. -It funds 308 loans and earns `$179,327.59` on a `$1M` budget, with weighted -default `0.039375`, weighted miscoverage `0.036875`, and conformal endpoint -budget `0.258051`. A matched point-PD allocation earns `$196,369.14` but has -weighted default `0.118400` and endpoint budget `0.921317`. Thus CRPTO pays -`8.678%` of realized return for a `7.9025` percentage-point default reduction; -the advantage reverses in some temporal slices, so we do not claim universal -dominance. The contribution is a small, auditable prediction-to-decision -guardrail with an exact replay, an inspectable selector, and explicit statistical -boundaries, rather than another credit-scoring leaderboard or a collection of -policy variants. - -**Keywords:** conformal prediction; predict-then-optimize; credit risk; -portfolio optimization; calibration; reproducible data science. - -# Introduction - -Credit allocation is a contextual optimization problem. A lender estimates a -PD and then chooses which loans to fund under capital, concentration, and risk -constraints [@sadana2025contextual]. Credit-scoring research has made the first -step increasingly reliable through discrimination benchmarks, probability -calibration, and cost-aware evaluation [@lessmann2015; @chen2024creditrisk; -@yang2025costaware]. Yet a calibrated probability does not specify how much -model uncertainty a portfolio should bear. A point-PD optimizer can concentrate -capital in loans that look attractive precisely where the predictive model is -least certain. - -Conformal prediction offers finite-sample coverage language under explicit -exchangeability conditions [@vovk2005; @angelopoulos2023], while robust -optimization makes uncertainty operational through feasibility sets -[@bertsimas2004; @goldfarb2003robustportfolio]. Joining the two is attractive, -but an applied paper still has to answer three questions. Which conformal level -is informative rather than nearly vacuous? How is a policy selected without -using the outcomes on which it is later reported? And what economic value is -lost when conformal uncertainty actually changes the funded set? - -CRPTO answers those questions with one deliberately simple policy. A frozen, -calibrated CatBoost model produces $p_i$. An exactly replayed 90% Mondrian -recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, +# Introduction {#sec-introduction} + +Predictive models create value through decisions, yet a property established +for a predictor need not survive the rule that consumes it. A probability of +default can be calibrated in a population while an optimizer concentrates +capital in a systematically different subset. A conformal prediction set can +attain marginal or groupwise coverage before selection while missing outcomes +among the observations that receive positive exposure. Even if the predictive +object were stable, a comparison can still fail when two scores are assigned +the same numerical threshold: the score and its cap jointly define a feasible +decision problem. + +The usual retrospective workflow creates additional hazards. A loan that has +reached an endpoint by an evaluation cutoff has a label, whereas a current or +later-resolved loan may not. Treating a later archive as if it were a verified +historical snapshot, or filtering to resolved outcomes before constructing the +candidate menu, uses information unavailable at the decision date. Pooling +several years of originations into one allocation similarly lets a decision +made today choose from tomorrow's loans. Finally, optimizing one payoff and +evaluating another can make an apparent improvement an accounting artifact. +These are estimand defects rather than presentation details. + +CRPTO studies this handoff after making timing, observability, geometry, and +comparator contracts explicit. A temporally trained CatBoost model and Platt +calibrator produce point score $p_i$. Score strata are fixed from all 2011 +predictions, independently of outcomes. We then report the complete set of +eight consecutive six-month residual windows beginning in 2012 and ending no +later than January 2013. No result selects, weights, or removes a window. Four +coverage-only controls span a separately fitted numeric logistic model, +domain-constrained monotonic CatBoost, a platform-signal WOE/IV scorecard, and +a pricing-excluded application WOE/IV scorecard. Each has its own Platt map and +2011 taxonomy; +none is selected from OOT outcomes or enters portfolio optimization. The +resulting clipped residual interval $[\ell_i,u_i]$ predicts the observed binary outcome. It is +neither a confidence interval for latent individual PD nor the convex hull of +its intersection with $\{0,1\}$. Its upper endpoint enters a guardrail score $$ -q_i = p_i + 0.5(u_i-p_i) = \frac{p_i+u_i}{2}, +q_i(\gamma)=(1-\gamma)p_i+\gamma u_i, +$$ {#eq-score} + +in the portfolio risk constraint. Guardrail and point-score policies maximize the +same model-implied objective over identical monthly menus, budgets, loan +bounds, and purpose constraints. The complete diagnostic score path is +$\gamma\in\{0,0.25,0.50,0.75,1\}$. Its primary empirical contrast is the +full-upper-score endpoint minus the point-score endpoint. No development +outcome selects a score, and no OOT result promotes a gamma. + +The comparator is part of the estimand. We therefore use two rulers constructed +without policy-development or OOT evaluation outcomes on the common attainable +frontier. We call a construction *evaluation-outcome-blind* when earlier model +and conformal fitting may use historical labels but policy construction reads +neither development nor OOT evaluation outcomes. The primary ruler imposes the same +model-implied objective floor for every score and hence matches plug-in +opportunity cost. The secondary ruler uses the same relative relaxation from +each score's minimum-risk allocation to the common plug-in optimum; it is +invariant to positive affine score transformations but does not match +opportunity cost. Both rulers are evaluated at coordinates +$\{0.25,0.50,0.75\}$. A supporting audit checks these finite tracks against +same-cap, development-matched, contemporaneous funded-moment, and exact +point-cap comparators. The exact HiGHS basis frontier avoids fixed-grid +interpolation over the declared point-cap support. + +The empirical message is an identification result rather than a winner. Under +the declared six-month availability rule, all 40 model-by-window all-candidate +OOT coverage upper bounds are below 0.90. The primary CatBoost pattern recurs +under three semantics-preserving missing-value encodings, one later +retrospective origin, and four declared fit-label scenarios. One score +stratum crosses the nominal miscoverage prevalence threshold between W7 and W8, +coinciding with an abrupt change in conformal geometry, but the crossing +disappears when every label unavailable at its fitting cutoff is completed as +default. The empirical coverage failure is therefore more stable than that +particular geometric explanation. In the portfolio audit, no endpoint ordering +survives all rulers and coordinates. +Objective-matched .25 crosses zero for all three metrics in every window; .50 +is adverse in all eight; and .75 leaves payoff and default unidentified in +seven. Normalized .25 and .50 are adverse, while normalized .75 leaves payoff +unidentified in one window and does not equalize plug-in opportunity cost. +Independently, every broad-support exact-cap payoff, default, and miscoverage +envelope crosses zero. + +The paper makes three contributions. + +1. It gives an executable decision-interface audit: freeze the population, + information set, predictive object, and allocations; join outcomes only + afterward; then report complete perturbation families without selecting a + favorable result. +2. It derives binary prevalence-sensitive geometry and sharp common-outcome + contrast bounds, including an exact width decomposition. The completion + sensitivity separates a stable nontransport finding from a nonuniversal + geometric explanation. +3. It defines two common-frontier rulers and an exact basis-endpoint support + audit, showing why score, ruler, coordinate, and comparator support jointly + define the portfolio estimand. The negative conclusion is constructive: + coverage does not transport here and no universal portfolio direction is + identified. + +This audit was retrospectively protocol-locked and hash-verified after the +archive had been inspected; it is not a prospective trial, preregistration, or +causal estimate. The purpose is +to identify which conclusions are properties of the predictive object and +which are artifacts of the decision comparison. + +# Related Work {#sec-related} + +## From predictive quality to decision quality + +Research on decision quality distinguishes estimation quality from the quality +of the action induced by an estimate. Fernandez-Loria and Provost show that a +useful decision ranking and an accurate effect estimate are different objects +[@fernandezloria2022causaldecision], and later make the identifying assumptions +behind observational decision rules explicit [@fernandezloria2025observational]. +Cost-aware calibration similarly evaluates probability errors through their +downstream asymmetry rather than through calibration alone [@yang2025costaware]. +Das et al. link credit-modeling choices to empirical design and +reproducibility [@das2023creditgraph]. +More broadly, the AI--OR interface is valuable when prediction, mathematical +optimization, and operational interpretation form one auditable chain +[@wiberg2025ai_or]. + +Decision-focused learning trains predictors against an optimization loss +[@donti2017; @elmachtoub2022; @mandi2024]. Contextual optimization and +predict-then-optimize methods instead preserve a modular predictor and expose +how forecast errors enter the decision [@bertsimas2020prescriptive; +@sadana2025contextual]. CRPTO takes the modular route because the research +question concerns governance of a frozen score, not a new credit-scoring +leaderboard. The baseline and guardrail therefore share the model, payoff, +budget, concentration limits, solver, and monthly candidate menus. They cannot, +however, share a numeric risk threshold by label alone: once the score changes, +the threshold defines a different feasible set. CRPTO treats comparator +alignment as part of the empirical design rather than an implementation +default. + +## What conformal coverage does and does not transport + +Split conformal prediction supplies finite-sample marginal coverage under +exchangeability [@vovk2005; @angelopoulos2023]. Exact conditional coverage is +generally unattainable without strong restrictions [@barber2021limits], and +departures from exchangeability require an explicit discrepancy, weighting, or +adaptation mechanism [@tibshirani2019covshift; @gibbs2021aci; +@barber2023beyond; @farinhas2024nonexchangeable_crc]. Selecting among valid +conformal objects can itself invalidate them; recent work constructs stable or +otherwise controlled selection procedures precisely because validity is not +closed under arbitrary data-dependent choice +[@hegazy2025valid_selection_conformal_sets]. + +Conformal uncertainty sets have also entered portfolio, robust, and contextual +optimization [@kato2025; @johnstone2021; @patel2024]. Kato studies asset-return +portfolio selection, while Sun et al. calibrate predict-then-optimize contextual +linear programs [@sun2024ptc]. Those designs supply useful decision-level +benchmarks, but their sampling units and guarantees do not transfer to binary +credit allocation with reconstructed endpoints. Recent work instead calibrates +decision loss, operational violations, or the miscoverage--regret +frontier rather than using predictive coverage as a proxy +[@yeh2025training; @zhou2026creme; +@stratigakos2026decision_calibrated_sets]. Wang and Dobriban derive +risk-optimal actions and prediction sets from an explicit decision loss +[@wang2026optimal_decision_prediction_sets], while Zheng and Jin couple +coverage to counterfactual actions when outcomes depend on the induced policy +[@zheng2026counterfactual_decision_sets]. CRPTO does not compete with those +methods by claiming a new selected-set theorem. Its accepted-loan archive +cannot identify a counterfactual funding effect. It asks a complementary +empirical question: when a simple conformal score is attached to a conventional +credit LP, where does its apparent risk effect come from, and where does +candidate coverage fail to follow? + +The closest decision-level methods also clarify what this audit cannot borrow. +CROMS selects conformal models using downstream robust-decision risk +[@bao2025croms]; decision-theoretic conformal prediction begins from the +agent's loss [@kiyani2025]; and inverse conformal risk control calibrates a +robustness choice against decision regret [@zhou2026creme]. Those methods +require labeled decision contexts under their stated sampling conditions. +CRPTO instead has eleven common development menus, fifteen later monthly +decisions, and direct evidence against temporal coverage transport. Treating +hundreds of thousands of loans as independent decision contexts would ignore +that each action is one coupled monthly allocation. We therefore use these +methods to define the correct target, not to import a finite-sample guarantee. + +## Credit maturity and economic evaluation + +Credit scoring and profit scoring are related but distinct. High discrimination +does not determine which loan is profitable [@lessmann2015], and Lending Club +studies have long shown that interest rate, default, recoveries, and portfolio +constraints jointly shape the investment decision +[@serrano2016profitscoring; @lyocsa2022profit]. Recent uncertainty-aware profit +models reinforce the need to evaluate the economic target directly +[@xu2024profit_risk_credit; @xu2025profit_uncertainty_credit]. +Credit-risk auditability also motivates transparent scorecards and economically +signed constraints alongside flexible learners. WOE/IV binning provides a +supervised, inspectable representation, while mathematical-programming binning +can enforce bin size and monotonic structure [@navaspalencia2020]. We use these +devices as model-class controls: they test whether the transport result depends +on CatBoost, incumbent grade/pricing signals, or unconstrained nonlinear shape. +They are not a scorecard-superiority claim. + +Maturity is equally central. Ignoring random censoring can bias empirical risk +[@ausset2022censoring]. In online lending, default and prepayment are competing +events whose timing changes portfolio profitability [@li2023online_loans], and +dynamic portfolio models track state transitions and cash flows rather than a +single binary reward [@djeundje2025dynamic_loan_portfolio_profitability]. Our +status-indexed payoff proxy is deliberately simpler. It is useful for isolating the +decision effect of PD and conformal uncertainty, but it is not an internal rate +of return, a discounted cash-flow estimate, or a substitute for survival +analysis. + +## Closest-work boundary + +CRPTO lies between several mature literatures, so its contribution cannot be +that any one ingredient is new. Classical and data-driven robust optimization +make the price of protection explicit [@bertsimas2004; +@bertsimas2018datadriven; @goldfarb2003robustportfolio]. P2P lending research +already combines credit scores, returns, and portfolio constraints +[@guo2016p2p; @zhao2016p2pportfolio; @chi2019p2p; @babaei2020p2p]. Conformal +robust optimization carries coverage-backed sets into downstream decisions +[@johnstone2021; @patel2024; @hu2026crc], while valid-selection and +decision-calibration methods directly target the inferential break caused by +choosing a set or action [@hegazy2025valid_selection_conformal_sets; +@zhou2026creme; @stratigakos2026decision_calibrated_sets]. + +The active CRPTO role is narrower: it audits what happens when a conventional, +frozen credit score receives a simple conformal upper-score constraint. It does +not retrain through the optimizer, calibrate a selected-set loss, or claim +selected-set validity. Its theoretical additions concern this specific handoff: +a binary-residual threshold discontinuity, comparator-feasibility results, and +an exact identification-width identity for unresolved binary outcomes. Its +empirical additions are a maturity-restricted credit protocol, complete residual-window +reporting, an exact comparator frontier, coherent economic comparison, and sharp +treatment of unresolved outcomes. + +Fernandez-Loria and Provost motivate the distinction between an intermediate +estimate and the action it induces, Yang and Bi make downstream cost central to +calibration, Das et al. link credit-modeling choices to empirical design, and +Wiberg et al. frame the AI--OR interface. The unit of analysis is the combined +predictive and decision system: the discrete geometry of the interval, the +allocation it induces, and the comparator support required to interpret that +allocation. Online Supplement Table S13 preserves the detailed closest-work +matrix while keeping the manuscript focused on the active estimands. + +# Data and Locked Evaluation Design {#sec-data} + +## Decision unit, target, and estimand + + + + +The decision unit is an issue month. For month $t$, the candidate set +$\mathcal I_t$ contains only loans observable in that month, and each policy +maps the same menu into dollar exposures with a fresh USD 1 million budget. This +is different from ranking the full archive once: no April 2016 decision can +fund a May 2016 loan, and no capital is carried across months. Equal monthly +budgets make the pooled exposure-weighted metric equivalent to the average of +the 15 monthly dollar-weighted metrics. + +The endpoint is terminal default reconstructed as observable by September 30, +2020. A Fully Paid status is available at the month-end of `last_pymnt_d`; a +Charged Off status is available at that month-end plus six calendar months. +The latter is a conservative modeling assumption, not the known operational +charge-off date. The distributed archive is not a verified point-in-time snapshot: its +last-payment field extends through December 2020 and its last-credit-pull field +through October 2020. We therefore date endpoint availability from servicing +fields and keep later terminal statuses unresolved. The policy estimands are +historical guardrail-minus-point differences in a status-indexed standardized payoff proxy, +exposure-weighted terminal default, and exposure-weighted interval miscoverage +over the same menus. They are not treatment effects: funding does not cause the +recorded status, rejected-loan outcomes are unavailable, and no behavioral +response to deployment is modeled. + +There are therefore three distinct populations in the analysis: the candidate +rows to which predictive coverage refers, the listed loan amounts that define +available exposure, and the optimizer-selected funded dollars that define the +decision result. Treating those populations as interchangeable would erase the +mechanism the paper is designed to measure. + +## Status-independent loan universe + + + +The source is the Lending Club 2007--2020Q3 public research archive. A full +raw-file audit scans 2,925,493 rows, of which 2,925,492 are valid dated loans, +across all 142 columns. There are 2,060,077 36-month and 865,415 60-month +contracts. The active design retains 640,543 rows: every loan eligible under +the declared 36-month horizon, dates, schema, and observability rules, not a +sample or computational row cap. Its evaluation panel contains 465,117 loans: +376,890 primary candidates and 88,227 extension candidates. Candidate +membership depends on issue month, term, and fields observable at origination. +It never depends on whether the reconstructed endpoint is resolved. + +Adding the excluded rows would change the estimand, not merely increase sample +size. Sixty-month loans have a different outcome horizon; 48 fields have +negligible early support but near-complete later coverage; and the intervening +cohorts are heavily duration-censored at the March 2016 information cutoff. Available labels are +59,910/162,570 for 2014, 28,878/283,173 for 2015, and 1,110/96,120 for 2016Q1, +with no observed bads in the last group. Training a terminal-default classifier +on those resolved subsets would condition on maturity. A survival estimand +could use them, but it would change the target, theory, and paper rather than +make this binary protocol more complete. + +Status strings containing `Charged Off` are positive and strings containing +`Fully Paid` are negative only when their reconstructed availability is no +later than the cutoff. The reason taxonomy partitions all 376,890 candidates: +307,842 are Fully Paid by the cutoff, 56,972 are Charged Off by the cutoff, +11,551 remain nonterminal or unresolved, 47 have reconstructed terminal +availability after the cutoff, and 478 have terminal status but no +reconstructible availability date. Thus 11,551 + 47 + 478 = 12,076 outcomes +remain unrestricted. Exact `Default` and every other nonterminal status are +included in the 11,551-row category. This is a cutoff-specific +terminal classification endpoint, not a lifetime hazard or causal response. +Unresolved loans remain candidates and enter sharp bounds after allocations +are frozen. + +The optimizer uses listed loan amount as available exposure. A full primary +OOT reconciliation finds only USD 18,000 of requested-minus-funded difference +over 376,890 loans, a funded ratio of 0.999996. The distinction is empirically +immaterial here while remaining explicit in the estimand. + +The chronology has shared PD and probability-calibration blocks, a 13-month +residual pool, one common outcome-free development block, and one common OOT +panel. Every consecutive six-month residual window beginning January--August +2012 is reported; all end by January 2013. February--December 2013 supplies the +same eleven development menus for every window. Policy-development outcomes +are neither required nor read. The first primary month is April 2016. Its +window ends in June 2017, at least 39 months before the September 2020 cutoff +for a 36-month contract. Contract age nevertheless does not force +administrative resolution, which is why 12,076 primary candidates remain +unresolved. July--September 2017 is retained as a more heavily censored +extension rather than silently discarded. + +| Block | Issue months | Rows | Labels available/read | Role | +|---|---:|---:|---:|---| +| PD development | 2007-06--2010-12 | 17,433 | 17,392 | train/validate | +| Probability calibration | 2011-01--2011-12 | 14,101 | 14,077 | Platt fit and taxonomy | +| Residual pool | 2012-01--2013-01 | 49,007 | 48,857 | eight overlapping six-month recipes | +| Policy development | 2013-02--2013-12 | 94,885 | not read | common outcome-free frontier construction | +| Primary OOT | 2016-04--2017-06 | 376,890 | post-freeze only | locked evaluation | +| Censored extension | 2017-07--2017-09 | 88,227 | post-freeze only | stress only | + +: Active data blocks. Residual-window rows overlap by construction and are not independent samples. {#tbl-protocol} + +## Information boundary + +The implementation materializes two ID-keyed panels. The decision panel +contains issue date, amount, purpose, contractual rate, point score, conformal +endpoints, and the frozen score stratum. It rejects outcome, status-indexed payoff, +miscoverage, or outcome-derived columns. The outcome panel contains the +reconstructed endpoint and is joined only after the solver returns an allocation. IDs must align +one-to-one; partial joins fail. This physical separation does not create a new +statistical theorem, but it makes the timing claim testable in code. + +Label-dependent fitting is additionally restricted by an information cutoff +of March 31, 2016. A Fully Paid label becomes available at the month-end of its +last payment; a Charged Off label is conservatively dated six calendar months +after that month-end. +The retained shares are 99.765% in PD development, 99.830% in probability +calibration, and 99.694% over the residual pool. Every residual month exceeds +99%; the minimum is 99.296% in January 2013. Missing dates and labels arriving +after the cutoff are excluded from fitting, not from the candidate universe. + +The outcome-free protocol froze two learner score vectors, 64 residual +recipes, 51,117 solve records, and 5,001,617 funded rows before any outcome +join. Its maximum absolute C2 match residual is $8.33\times10^{-17}$. The +active evaluator verifies that freeze by SHA-256, reconstructs +status availability, and injects window-specific endpoints only after the +shared point allocation is loaded. It changes neither predictions, recipes, +comparators, nor allocations. Online Supplement Appendix F records the active +lineage and replay contract. + +The two-ruler diagnostic has a separate outcome-free freeze. It contains 6,240 +solves and 622,455 funded rows over eight windows, 26 development/OOT months, +five gamma values, three interior coordinates, and two rulers. Reversed-ID +reruns and a separate GLOP implementation check the endpoint allocations. +A hash-locked evaluator then joins the same outcome panel without refitting, +resolving, or selecting a track. Its 48 window cells are six specification +tracks observed through eight overlapping residual recipes, not 48 +replications. + +## Identification safeguards + +The reusable audit has five stages. First, define status-independent monthly +menus and dated label availability. Second, fit scores, taxonomies, and every +residual recipe without evaluation outcomes. Third, declare comparator rulers, +solve the complete policy grid, and persist allocations. Fourth, join one keyed +outcome panel and compute sharp candidate and paired-policy bounds. Fifth, +report every declared window, model control, ruler, coordinate, support, and +sensitivity; a claim survives only its named perturbation family. This order +prevents outcome filtering, future-menu look-ahead, payoff mismatch, and +result-dependent comparator choice. It does not restore exchangeability, +identify lifetime cash flows, or turn a finite grid into universal support. +Online Supplement Appendices A and G give the full audit map and boundaries. + +# Method {#sec-method} + +## Platt-scaled default score and credit-risk controls + +A CatBoost classifier uses 29 numeric and 9 categorical origination-time +features. Hyperparameters are fixed in the executable protocol: 500 trees, +depth 6, learning rate 0.04, class balancing, Bernoulli subsampling, and a +time-aware ordering. The last 20% of PD-development months form a temporal +validation tail. A logistic Platt map is then fitted on the 2011 raw margins, +separate from both model training and conformal fitting. + +The prediction model is deliberately not tuned against downstream outcomes. +Canonical seed 42 is inherited from the locked design. Four protocol-locked +coverage-only specifications alter model class or credit-risk information: + +1. numeric-feature logistic regression; +2. CatBoost with 20 domain-signed monotonic constraints, such as increasing + risk in interest rate, debt burden, inquiries, and adverse-history flags and + decreasing risk in income, FICO, and credit age; +3. a 26-field OptBinning WOE/IV logistic scorecard including borrower, + contract, grade, subgrade, interest, and derived pricing signals; and +4. a 19-field pricing-excluded application WOE/IV scorecard excluding grade, + subgrade, interest, installment, and derived platform-pricing signals. + +Each WOE process is fit only on PD development, uses two to eight bins, a 5% +minimum bin share, and automatic monotonic trend [@navaspalencia2020]. Every +learner receives its own 2011 Platt map and fixed score taxonomy. On the Platt +block, AUCs in primary/control order are 0.676327, 0.656880, 0.678091, +0.660533, and 0.662401; Brier scores range from 0.090054 to 0.091000. All five +are frozen before the primary OOT outcome join and use every eligible row. The +four controls never enter portfolio optimization, and OOT outcomes select +neither model nor feature. These diagnostics test specification dependence; +they are not a predictive leaderboard or a claim that WOE/IV is novel. + +The full-file contract identifies two active-input exceptions to the 95% raw +coverage rule. Missing delinquency recency is structurally meaningful and is +mapped to the frozen no-recent-delinquency convention; bankruptcy count has +92.17% minimum early coverage and missing values map to no recorded bankruptcy. +Observed-value indicators are retained by feature engineering for auditability, +but the active portfolio model does not add them as post hoc predictors. A +separately frozen sensitivity preserves the two feature definitions while +comparing the active sentinel convention, explicit missing indicators, and +CatBoost-native nullable values. It selects no encoding and does not identify a +missingness mechanism. + +## Exact binary-outcome Mondrian intervals + +Let $Y_i\in\{0,1\}$ denote terminal default and $p_i$ the Platt-scaled score. +Five groups are defined by quantiles of all 2011 Platt-scaled scores, without +using residual-window labels. The frozen CatBoost edges are +$0.008219$, $0.050265$, $0.080369$, $0.111894$, $0.153262$, and $0.475854$. +Within fixed group $g$, the conformity score on an availability-safe residual +window is $s_i=|Y_i-p_i|$. The complete specification reports all eight +consecutive six-month windows beginning January--August 2012. Every included +month exceeds 99% label retention under the six-month lag. For $n_g$ +observations and target $\alpha=0.10$, +the finite-sample rank is + $$ +k_g=\left\lceil(n_g+1)(1-\alpha)\right\rceil, +$$ {#eq-rank} -inside the risk constraint. The economic objective remains point-PD expected -net return. This separation is important: $p_i$ prices expected loss, while -$q_i$ limits the amount of uncertainty the funded portfolio may carry. It also -removes the capped, tail-focused, and uncertainty-penalty branches that made an -earlier research frontier difficult to explain and easy to misread. - -The empirical design uses a temporal Lending Club panel. The conformal recipe -is fit inside the calibration period, November 2017 ranks a declared -$3\times3$ grid of round-number risk tolerances and conformal weights, and -December replays the outcome-free selector before auditing the already-fixed -decision. The ranking artifact contains no defaults, realized returns, -miscoverage, or assumption-conditional statistics. The fixed rule is then -evaluated on loans originated from January 2018 through September 2020. Earlier -project development did inspect this static OOT corpus, so we describe the -final run as a transparent retrospective lockbox replay, not as a pristine -prospective or preregistered trial. - -The paper makes three contributions. First, it gives an auditable -prediction-to-decision construction in which the economic objective and the -conformal guardrail have separate, inspectable roles. Second, it replaces -approximate cross-alpha scaling with an exact replay, a deterministic endpoint -screen, and a temporally separated selector audit. Third, it reports the price -and limits of that guardrail against matched point-PD and more-conservative -comparators, including a pre-OOT funded-set coverage miss and temporal slices -where CRPTO wins and slices where it does not. The novelty is the closed, -inspectable decision protocol for a frozen credit model, not a claim that -conformal prediction, robust optimization, or credit scoring is individually -new. - -![CRPTO carries a frozen calibrated PD through an exact conformal replay, a simple portfolio guardrail, and a funded-set audit.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="92%" fig-alt="Four-stage CRPTO pipeline from calibrated PD to conformal intervals, portfolio allocation, and funded-set audit."} - -# Related Work - -CRPTO sits at the intersection of conformal prediction, robust optimization, -and decision-focused learning. Split conformal methods provide marginal -coverage without a parametric posterior, while Mondrian variants condition on -declared partitions [@vovk2005; @bostrom2021]. Conditional and weighted -extensions require additional structure, and exact conditional coverage is -generally unavailable without restrictive assumptions [@barber2021limits; -@barber2023beyond; @jonkers2024wcps]. We therefore distinguish population or -partition coverage from coverage after a portfolio has adaptively reweighted -the loans. - -Data-driven robust and contextual optimization translate predictive -uncertainty into decisions [@bertsimas2018datadriven; -@bertsimas2020prescriptive; @sadana2025contextual]. Recent work uses conformal -sets directly in robust optimization [@johnstone2021; @patel2024; -@sun2024ptc; @hu2026crc]. CRPTO is an applied complement: it retains a frozen -credit PD model, exposes the exact funded rows and uncertainty premium, and -compares the resulting allocation with a matched point-PD decision. - -Decision-calibrated prediction sets and inverse conformal risk control go -further by calibrating downstream violation or regret and, after choosing a -robustness level, using a separate split to restore risk-estimation validity -[@zhou2026creme; @stratigakos2026decision_calibrated_sets]. CRPTO does not -import that guarantee into a batch portfolio where the optimizer chooses the -funded weights. Instead, it separates a deterministic endpoint-budget screen -from an independent post-selection audit and reports when selected-set -miscoverage misses its nominal target. - -Decision-focused learning and SPO+ train predictions against downstream regret -[@donti2017; @elmachtoub2022; @mandi2024]. That is a different institutional -choice. CRPTO asks what can be done after a calibrated model already exists and -must remain unchanged for governance. A synthetic SPO+ comparison is retained -in the supplement because it confirms the expected distinction: decision- -focused training improves its own regret objective, whereas CRPTO produces an -auditable uncertainty-constrained funded set. - -Credit-allocation research already covers profit scoring, P2P investment -recommendation, robust loan portfolios, rejection, and multiobjective risk -[@guo2016p2p; @zhao2016p2pportfolio; @serrano2016profitscoring; @chi2019p2p; -@babaei2020p2p; @xu2025profit_uncertainty_credit; -@xu2024profit_risk_credit]. Conformal credit scoring also means that the safe -claim is not "first use of conformal prediction in credit" -[@kawasumi2026ordinal]. The remaining gap is a file-backed protocol that shows -exactly how a conformal endpoint changes a budgeted credit decision and what -that change costs. - -| Literature family | Existing contribution | CRPTO boundary | -|---|---|---| -| Credit scoring and calibration | Accurate, calibrated PD and cost-aware prediction. | Treats PD as an input contract; does not claim AUC leadership. | -| P2P and robust credit portfolios | Economic loan selection under risk and uncertainty. | Uses an exact conformal endpoint as a simple portfolio guardrail. | -| Conformal robust optimization | Coverage-backed uncertainty sets for downstream decisions. | Adds a frozen credit stack, funded-set audit, and matched economic comparator. | -| Decision-focused learning | Training-time reduction of decision regret. | Keeps the predictive model frozen and emphasizes post-hoc auditability. | -| Decision-risk calibration and valid selection | Calibrates decision losses or corrects validity after choosing among sets [@zhou2026creme; @hegazy2025valid_selection_conformal_sets]. | Uses a deterministic selector plus an independent diagnostic audit; formal selected-set validity is not asserted. | - -: Closest-work boundary for the submitted CRPTO claim. - -# Data and Evaluation Design - -The data are Lending Club retail loans originated from 2007 through 2020. The -feature contract contains only information available at origination. The model -and evaluation pipeline use temporal rather than random splits. - -| Split | Period | Loans | Role | -|---|---|---:|---| -| Train | Jun 2007--Mar 2017 | `1,346,311` | Fit the PD model and calibrator. | -| Conformal fit | Mar--Oct 2017 | `142,550` | Estimate the frozen conformal recipe. | -| Policy selection | Nov 2017 | `14,943` | Rank the nine outcome-free policies. | -| Calibration audit | Dec 2017 | `20,695` | Replay the selector, then open outcomes. | -| OOT evaluation | Jan 2018--Sep 2020 | `276,869` | Freeze-then-evaluate portfolio decisions. | - -: Temporal Lending Club design. - -The frozen conformal recipe uses the most recent 75% of the original -calibration pool (`178,188` rows). Within that subset, `142,550` rows estimate -conformal quantiles, `14,943` November rows select the policy, and `20,695` -December rows form a pre-OOT audit. The conformal recipe uses calibration -labels, as conformal prediction requires. The loader stores holdout outcomes -separately from the 12-column policy frame, which contains identifiers, -origination context, PD endpoints, amounts, and rates. Ranking cannot access -outcomes, realized returns, miscoverage, or the assumption-conditional Markov -quantity. December first reruns that same outcome-free selector and only then -joins outcomes for the decision audit. - -The final OOT panel covers several regimes, including the 2020 disruption. We -report the full panel and five temporal slices (`2018H1`, `2018H2`, `2019H1`, -`2019H2`, and `2020+`). Each slice solves the same fixed policy on a fresh `$1M` -budget, so slice returns are comparable stress evaluations rather than -components that sum to the full-panel return. - -# Method - -## Calibrated PD and Exact Conformal Replay - -Let $p_i\in[0,1]$ be the calibrated PD for loan $i$. The predictive layer is a -frozen CatBoost classifier with AUC `0.7139`, Brier score `0.1544`, and expected -calibration error about `0.0070` on the paper-facing evaluation. These values -are not presented as a leaderboard result; probability quality matters because -the downstream objective consumes PD directly. - -The conformal recipe partitions calibrated scores into five score-quantile -Mondrian cells. For calibration loan $j$, it computes a scaled residual +and $c_g$ is the $k_g$-th ordered residual. A future score assigned to group +$g(i)$ receives $$ -s_j = \frac{|Y_j-p_j|}{\sqrt{p_j(1-p_j)}}, +[\ell_i,u_i]= +\left[\max\{0,p_i-c_{g(i)}\},\min\{1,p_i+c_{g(i)}\}\right]. +$$ {#eq-interval} + +The interval predicts the observed binary outcome. It is not a confidence +interval for latent individual PD. No holdout-learned widening, floor, +taxonomy adaptation, or post-cutoff label enters any recipe. Taxonomies with +1, 2, and 10 groups are closed coverage diagnostics. Each of the four controls +repeats this construction with its own scores and edges. + + + +For binary $Y$, miscoverage has the useful identity + $$ +m_i=\mathbf 1\{Y_i=0,\ell_i>0\}+ + \mathbf 1\{Y_i=1,u_i<1\}. +$$ {#eq-binary-miss} -with numerical clipping near zero. Each cell uses the finite-sample `higher` -quantile at the frozen conservative level `0.095` for the target -$\alpha=0.10$. Recorded group and temporal widening factors are then applied -without narrowing. The upper endpoint is -$u_i=\min\{1,p_i+\widehat q_{g(i)}\sqrt{p_i(1-p_i)}\}$ after those frozen -adjustments. +Thus a default is counted as covered whenever $u_i=1$, even though such an +endpoint offers little discrimination to the optimizer. Conversely, a narrow +low-score interval with $u_i<1$ misses every realized default. This geometry is +central to interpreting funded-set coverage. -The replay implementation reconstructs this recipe from its result payload. At -the reference 90% level, it reproduces the stored point, lower, and upper -vectors with maximum absolute error below `6.67e-16`. This matters because an -earlier exploratory analysis scaled 90% row radii using average widths from a -different conformal family. Those cross-alpha values are retained only as -historical provenance and are not used here. +## Coherent objective and status-indexed payoff proxy -## One Portfolio Policy +For contractual annual rate $r_i$ and fixed loss given default +$\lambda=0.45$, define the per-dollar status-indexed standardized payoff proxy -Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, -$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ the portfolio risk -tolerance. CRPTO solves +$$ +\pi_i(Y_i)=(1-Y_i)r_i-Y_i\lambda, +$$ {#eq-status-payoff} + +with model-implied plug-in objective coefficient + +$$ +\bar\pi_i=(1-p_i)r_i-p_i\lambda. +$$ {#eq-expected-payoff} + +This coefficient equals the conditional expectation of the proxy only if $p_i$ equals +the conditional default probability. We use it as a model-implied plug-in +objective and evaluate the corresponding status-indexed proxy, not as proof of +conditional calibration. The pair intentionally omits payment timing, +principal amortization, prepayment, fees, recoveries, and discounting. We +therefore do not call the endpoint profit, cash-flow return, NPV, welfare, or IRR. + +In month $t$, let $a_{it}$ be dollar exposure to loan $i$, bounded by its listed +amount $A_i$. For a policy $(\tau,\gamma)$, the LP is $$ \begin{aligned} -\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\max_{a_{it}}\quad & \sum_{i\in\mathcal I_t}a_{it}\bar\pi_i \\ \text{s.t.}\quad -& \sum_i x_i a_i \le B, \\ -& \sum_i x_i a_i q_i \le \tau\sum_i x_i a_i, \\ -& 0\le x_i\le \bar x_i, +& \sum_i a_{it}=B, \\ +& \sum_i a_{it}q_i(\gamma)\le \tau B, \\ +& \sum_{i:\,purpose_i=k}a_{it}\le 0.25B \quad\forall k,\\ +& 0\le a_{it}\le A_i, \end{aligned} -$$ +$$ {#eq-lp} + +where $B=\$1$ million in every month. The point-score baseline sets $\gamma=0$. +The objective is identical across policies, so differences arise from the risk +score and the loans made feasible by it. The LP permits fractional dollar +exposure. A separately frozen implementation diagnostic floors each positive +exposure to a USD 25 lot and holds the residual as cash; it does not reoptimize +or define an integer-policy estimand. -with the existing concentration and eligibility constraints. The selected -policy uses $\tau=0.17$ and $q_i=(p_i+u_i)/2$. The objective is intentionally -based on $p_i$, not $q_i$: expected economics and uncertainty feasibility are -separate contracts. The matched point-PD baseline changes only the risk score -to $q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and -risk tolerance fixed. +## Common-frontier rulers and supporting comparator audit -For funded-exposure weights -$w_i=x_ia_i/\sum_jx_ja_j$, define +Let $\mathcal A_t$ contain the budget, loan-bound, and purpose constraints in +@eq-lp, let $v_i=\bar\pi_i$, and let $s_i(\gamma)=q_i(\gamma)$ for +$\gamma\in\{0,0.25,0.50,0.75,1\}$. For every month and score, define the +minimum funded score, the common unconstrained plug-in optimum, and the score +of that optimum as $$ -\begin{aligned} -\Gamma_{\mathrm{CP}} &= \sum_iw_i(u_i-p_i),\\ -\Gamma_{\mathrm{int}} &= \sum_iw_i(q_i-p_i),\\ -\Gamma_{\mathrm{res}} &= \sum_iw_i(u_i-q_i),\\ -B_u &= \sum_iw_i u_i. -\end{aligned} +m_{\gamma t}=B^{-1}\min_{a\in\mathcal A_t}s(\gamma)^\top a, +\quad z_t^*=\max_{a\in\mathcal A_t}v^\top a, +\quad o_{\gamma t}=B^{-1}s(\gamma)^\top a_t^*. +$$ {#eq-ruler-anchors} + +The primary **objective-matched ruler** compares scores at a common plug-in +opportunity cost. Let $z^{\min}_{\gamma t}$ be the objective attained by a +minimum-score portfolio and set $z_t^L=\max_\gamma z^{\min}_{\gamma t}$. At +$\rho\in\{0.25,0.50,0.75\}$, every score solves + $$ +\min_{a\in\mathcal A_t}s(\gamma)^\top a +\quad\text{s.t.}\quad +v^\top a\ge z_t^L+\rho(z_t^*-z_t^L). +$$ {#eq-objective-ruler} -For the midpoint policy, -$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. -This identity is one reason to prefer the midpoint over nonlinear caps or tail -rules: every quantity has a direct interpretation. +The shared floor is a model-implied objective, not true expected return. The +secondary **normalized-score ruler** instead sets + +$$ +c_{\gamma t}(\eta)=m_{\gamma t} + +\eta(o_{\gamma t}-m_{\gamma t}),\qquad +\eta\in\{0.25,0.50,0.75\}, +$$ {#eq-normalized-ruler} + +and maximizes $v^\top a$ subject to +$s(\gamma)^\top a\le Bc_{\gamma t}(\eta)$. Equal $\eta$ is invariant to a +positive affine rescaling of a score, but it is not equal default risk, +objective sacrifice, or operational tolerance. At coordinate one both rulers +reach the verified common plug-in optimum, so the endpoint allocation contrast +is structurally null. The empirical diagnostic is therefore the complete +declared three-coordinate interior grid, not a continuous-frontier claim. + +For each ruler and coordinate, the frozen contrast is +$\gamma=1$ minus $\gamma=0$. Interior gamma values verify the path but cannot +be selected as winners. Every allocation is outcome-free; unresolved outcomes +enter only through the common-outcome bounds after the freeze. + +A supporting audit checks the finite grid against declared point-cap +comparators. C0 copies the numerical cap, +C1 uses the eleven-menu development point-score support, and C2 matches the +frozen guardrail's funded point-score moment on the same menu. C2 is an +outcome-free decomposition, not a deployable policy. HiGHS basis ranging then +enumerates 3,067 exact point caps over development support and [0.05, 0.12] +broad stress. Nine historical $(\tau,\gamma)$ pairs index this audit; they are +not a closed policy family or promotion candidates. Full definitions are in +Online Supplement Appendix C. No OOT outcome chooses any learner, window, +gamma, ruler, coordinate, cap, policy, or comparator. + +# Audit Theory and Estimands {#sec-theory} + +The theory separates four objects that are often conflated: score units, +comparator stringency, binary interval geometry, and unresolved-outcome +identification. None of the statements assumes funded-set conformal validity or +makes one ruler uniquely correct. Appendix D contains the proofs. + +## Comparator non-invariance + +Let $\mathcal F_s(\tau)$ denote the allocations satisfying @eq-lp when the +risk score is $s$. All nonrisk constraints and the objective are held fixed. + +**Proposition 1 (what comparator matching preserves).** Suppose the budget +binds, $\sum_i a_i=B$. + +1. If $s_i=\kappa p_i+b$ with $\kappa>0$, then -## Calibration-Only Final Selector +$$ +\sum_i a_i s_i\le \tau_s B +\quad\Longleftrightarrow\quad +\sum_i a_i p_i\le \frac{\tau_s-b}{\kappa}B. +$$ {#eq-affine-cap} -The candidate set crosses -$\tau\in\{0.15,0.17,0.19\}$ with -$\gamma\in\{0.25,0.50,0.75\}$ in -$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, -at least 99.9% of the budget is allocated, the effective-PD cap holds, and +2. If $u_i\ge p_i$ and the same numerical cap is copied to + $q_i(\gamma)$, then $$ -B_u\le 0.28 +\mathcal F_q(\tau)\subseteq\mathcal F_p(\tau). +$$ {#eq-feasible-nesting} + +3. For $\widetilde s=\kappa s+b$, the normalized ruler transforms as + $c_{\widetilde s}(\eta)=\kappa c_s(\eta)+b$, so + $$ +\widetilde s^\top a\le Bc_{\widetilde s}(\eta) +\quad\Longleftrightarrow\quad +s^\top a\le Bc_s(\eta). +$$ {#eq-normalized-invariance} -on the November block. Among eligible candidates, the rule maximizes expected -point-PD objective. Five of nine candidates pass, and the selected candidate is -$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic; it does not use -the weighted-validity assumption introduced below. The selected row remains -optimal for every cap in $[0.259036,0.290491)$, so the round `0.28` value has -positive margin to both policy-change boundaries. An outcome-free replay on -December independently selects the same candidate. +4. If C2 sets $c=(p^\top a^q)/B$ for a guardrail allocation $a^q$, that + allocation is feasible for the point-score LP and + +$$ +V_p(c)\ge \bar\pi^\top a^q. +$$ {#eq-c2-dominance} -| Candidate | $\tau$ | $\gamma$ | Nov. expected objective | Nov. $B_u$ | Dec. $B_u$ | Status | -|---|---:|---:|---:|---:|---:|---| -| Higher-return, low guardrail | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | Ineligible | -| Selected midpoint | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | Selected twice | -| More-conservative blend | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | Eligible | -| Higher risk tolerance | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | Ineligible | +Parts 1 and 3 identify exact invariances. Parts 2 and 4 show why same-cap and +funded-moment matching are not neutral performance tests: they mechanically +order feasible plug-in objectives but do not order realized payoff, default, or +miscoverage. Equality of one optimizer is also insufficient to establish +affine score equivalence. -: Temporally separated selector examples. A36 reports all nine candidates on -both months. +## Binary interval geometry -## Accounting and Statistical Boundary + -Let $Z_i=\mathbf 1\{Y_i>u_i\}$ and -$V=\sum_iw_iZ_i$. Because $Y_i\le u_i+Z_i$ for every funded loan, +**Proposition 2 (binary threshold geometry).** Fix $0\le p<1/2$ and let +$Y\sim\mathrm{Bernoulli}(\pi)$. Under the lower population +$(1-\alpha)$-quantile convention, $$ -\sum_iw_iY_i\le B_u+V +c_{1-\alpha}= +\begin{cases} +p, & \pi\le\alpha,\\ +1-p, & \pi>\alpha. +\end{cases} +$$ {#eq-binary-phase} + +Thus the clipped interval changes from $[0,2p]$ to $[0,1]$ as prevalence moves +above $\alpha$; coverage jumps from $1-\pi$ to one and the upper decision score +from $2p$ to one. For any varying-score interval, @eq-binary-miss gives the +exact two-case miscoverage decomposition: $u=1$ covers every default, whereas +every default with $u<1$ is missed. Endpoint saturation can therefore improve +coverage while removing ranking information. The constant-score result is a +mechanism, not a proof for an empirical stratum with varying $p_i$. + +## Sharp bounds for unresolved outcomes + + + +For an unresolved loan, $Y_i$ may be either 0 or 1. Default therefore lies in +$[0,1]$, payoff in $[-\lambda,r_i]$, and miscoverage in the two attainable +values implied by @eq-binary-miss. + +**Proposition 3 (sharp common-outcome bounds and width).** Let $U$ index +unrestricted outcomes and write an additive fixed-allocation metric as +$T(Y)=C+\sum_{i\in U}g_i(Y_i)$. Then + $$ +T_L=C+\sum_{i\in U}\min_{y\in\{0,1\}}g_i(y),\qquad +T_U=C+\sum_{i\in U}\max_{y\in\{0,1\}}g_i(y) +$$ {#eq-sharp-bounds} -holds deterministically. It is an exact funded-set accounting statement, not a -coverage theorem. +are sharp because each endpoint is attained by a joint assignment of all +unresolved outcomes. This is partial identification under unrestricted binary +completion, not a sampling confidence interval. -If one additionally assumes weighted funded-set validity, -$\mathbb E[V]\le\alpha$, Markov's inequality gives +For all-candidate coverage, let $m_i(0)$ and $m_i(1)$ be the two attainable +miscoverage values. With $m_i^L=\min_y m_i(y)$ and +$m_i^U=\max_y m_i(y)$, the sharp coverage interval is $$ -\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) -\le\sqrt{\alpha}. +\left[1-\frac{1}{n}\sum_i m_i^U, + 1-\frac{1}{n}\sum_i m_i^L\right]. +$$ {#eq-sharp-coverage} + +For paired policies, subtracting marginal intervals is not sharp. Instead, form +the union of funded IDs, retain each signed exposure difference, and assign the +same unresolved $Y_i$ once to both policies. If $d_i(Y_i)$ is loan $i$'s +contrast contribution, then + + + $$ +T_U-T_L=\sum_{i\in U}\left|d_i(1)-d_i(0)\right|. +$$ {#eq-identification-width} + +For exposure difference $\delta_i=a_i^A-a_i^B$, the dollar payoff-proxy width +is $\sum_{i\in U}|\delta_i|(r_i+\lambda)$. Let $N_A,N_B$ be the declared +positive capital normalizers and define +$\delta_i^w=a_i^A/N_A-a_i^B/N_B$. Payoff-rate width is +$\sum_{i\in U}|\delta_i^w|(r_i+\lambda)$, default-rate width is +$\sum_{i\in U}|\delta_i^w|$, and miscoverage-rate width is +$\sum_{i\in U}|\delta_i^w|\,|m_i(1)-m_i(0)|$. The fully invested policies use +$N_A=N_B=B$; the floor-with-cash diagnostic also uses committed capital $B$ +rather than renormalizing away cash. Hence unresolved outcomes alone do not +determine identification loss: width depends on where the policies disagree in +exposure and, for miscoverage, on prediction-set geometry. These are +finite-archive identification intervals, not confidence intervals. -This assumption does not follow from marginal split conformal because the -optimizer chooses the weights using $p_i$ and $u_i$. We therefore treat the -Markov expression as a secondary, assumption-conditional sensitivity. The -primary evidence is the exact replay, OOT-outcome-column-free policy ranking -conditional on the frozen conformal recipe, and observed funded-set audit. -Full proofs and sharper-assumption diagnostics are in the online supplement. - -# Results - -## Exact 90% Conformal Evidence - -At the active target $\alpha=0.10$, exact OOT coverage is `0.934836`, average -interval width is `0.788879`, minimum score-partition coverage is `0.926310`, -minimum letter-grade coverage is `0.926797`, and `51.7873%` of upper endpoints -equal one. The intervals are conservative and broad, which is expected for a -binary outcome on the probability scale. - -The exact alpha sensitivity explains why the paper does not headline a 99% -interval. At target alpha `0.01`, empirical coverage is `0.996720`, but average -width is `0.988215` and `93.5424%` of upper endpoints equal one. Such endpoints -carry almost no ranking information for a portfolio. The selected 90% level is -the frozen recipe's reference level and preserves materially more decision -resolution. A35 reports the complete sensitivity from alpha `0.01` to `0.20`. - -## Pre-OOT Selector and Decision Audit - -November selects the midpoint policy from five eligible rows. Applying the -same outcome-free rule to December again selects `linear-005`; policy identity -is therefore stable across the two calibration months. Outcomes are opened -only after that replay. - -| December policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | -|---|---:|---:|---:|---:|---:| -| Selected 50/50 CRPTO | `193` | `$53,313.05` | `0.145650` | `0.124925` | `0.262082` | -| More-conservative 75% blend | `191` | `$38,379.50` | `0.155250` | `0.134525` | `0.203504` | -| Point-PD matched-$\tau$ | `169` | `$89,732.35` | `0.185650` | `0.058300` | `0.888071` | - -: Independent December 2017 post-selection decision audit. - -The selected policy remains below its operational tolerance -(`0.145650 < 0.17`) but misses nominal funded-set coverage -(`V=0.124925 > 0.10`). Its deterministic accounting right-hand side is -`B_u+V=0.387007`. Thus an outcome-free, month-stable selector does not create -selected-set conformal validity. This negative audit is part of the result: it -motivates the two-layer reporting of deterministic endpoint exposure and -observed miscoverage instead of relabeling marginal coverage as a portfolio -guarantee. - -## Full OOT Funded-Set Audit - -The fixed midpoint policy allocates the full `$1M` budget across 308 loans. Its -expected point-PD objective is `$168,271.56`; realized return is `$179,327.59`. -The weighted default rate is `0.039375`, and weighted miscoverage is `0.036875`. - -| Quantity | OOT value | -|---|---:| -| Weighted point PD | `0.081949` | -| Weighted midpoint score | `0.170000` | -| $\Gamma_{\mathrm{CP}}$ | `0.176102` | -| $\Gamma_{\mathrm{int}}$ | `0.088051` | -| $\Gamma_{\mathrm{res}}$ | `0.088051` | -| Endpoint budget $B_u$ | `0.258051` | -| Observed accounting bound $B_u+V$ | `0.294926` | -| Conditional Markov threshold | `0.574279` | - -: Exact full-OOT audit of the selected policy. - -The observed weighted outcome `0.039375` is well below the exact accounting -right-hand side `0.294926`. Under the additional weighted-validity assumption, -the `0.574279` event threshold has probability bound -$\sqrt{0.10}=0.316228$. This loose probability statement is not interpreted as -a direct default cap. The operational controls are $\tau=0.17$, the midpoint -score, and the exact funded-set diagnostics. - -The primary fixed-allocation bootstrap resamples 31 origination-month clusters -and gives a 95% return interval of `$163,421.14`--`$193,551.65` from 5,000 -draws. A loan-level sensitivity gives -`$162,706.17`--`$193,924.74`. Neither scheme resamples the model, conformal -recipe, policy selector, or optimizer; both are contribution-level stability -diagnostics rather than full-pipeline confidence intervals. The month-level -scheme is primary because the temporal results reject a naive independence -story across loans. - -## Matched Point-PD and More-Conservative Comparators - -| Policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | Conditional threshold | -|---|---:|---:|---:|---:|---:|---:| -| Selected 50/50 CRPTO | `308` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | -| More-conservative 75% blend | `312` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | -| Point-PD matched-$\tau$ | `225` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | - -: Full-OOT matched decision comparison. - -Relative to point PD, selected CRPTO gives up `$17,041.55`, or `8.678%` of -realized return. Weighted default falls by `7.9025` percentage points, -miscoverage falls by `0.5025` percentage points, and the endpoint-plus-Markov -threshold falls by `66.3266` percentage points. The default contrast is much -larger than the miscoverage contrast: conformal robustness changes which loans -are funded, but it does not create a dramatic selected-set coverage gain. - -The 75% blend reveals the remaining internal trade-off. It lowers default by -`0.35` percentage points and the threshold by `0.057655` relative to the -selected midpoint, but costs another `$6,388.08` in realized return. The -calibration selector chooses the midpoint because it is the highest expected- -objective candidate under the declared screen, not because it dominates every -risk metric. - -## Temporal Heterogeneity - -The full-panel average hides substantial regime dependence. - -| Period | CRPTO return | Point-PD return | CRPTO default | Point-PD default | +## Declared comparator sensitivity envelope + + + +Let comparator $j$ produce a sharp contrast interval $[L_j,U_j]$ for a fixed +guardrail and metric. For a declared finite set $\mathcal J$, define + +$$ +\mathcal I_{\mathcal J}= +\left[\min_{j\in\mathcal J}L_j,\max_{j\in\mathcal J}U_j\right]. +$$ {#eq-multiverse} + +The declared comparator sensitivity envelope $\mathcal I_{\mathcal J}$ +summarizes endpoints over a named finite comparator set. A sign survives only +when the envelope lies strictly on one side of zero; this is deterministic +design sensitivity, not universal quantification over baselines. + +**Proposition 4 (basis-endpoint sufficiency).** Within one optimal LP basis, +the point allocation is affine in cap $c$. Against a fixed guardrail, +observed-outcome contrast contributions are therefore affine. For an unresolved +binary outcome, the sharp lower contribution is the minimum of two affine +functions and the sharp upper contribution is their maximum. The lower bound +is concave and the upper bound convex on that basis range, so their adverse +extrema occur at its endpoints. Evaluating every support endpoint and every +HiGHS basis-ranging endpoint therefore gives the exact sharp sensitivity +envelope over the declared cap interval, up to solver tolerance. + +Together these statements separate exact algebra, finite-archive partial +identification, and empirical sensitivity. Online Supplement Appendix D gives +the proof map and Appendix G states the corresponding claim boundaries. + +# Results {#sec-results} + +## Coverage fails across the complete residual-window specification + + + +Under the active six-month evaluation endpoint, every five-stratum recipe +attains its finite-sample fit rank, but none transports 90% all-candidate +coverage to the primary OOT panel. CatBoost resolved coverage +ranges from 0.8659 to 0.8792. After retaining all 12,076 unresolved candidates, +its sharp coverage bounds range from [0.8425, 0.8696] to [0.8570, 0.8826]. The +separately calibrated logistic control is closer to nominal: resolved +coverage ranges from 0.8732 to 0.8931 and the largest upper bound is 0.8962. +Nevertheless, all eight logistic upper bounds also remain below 0.90. The +finding is therefore not unique to CatBoost, while its magnitude is +learner-dependent. Four additional credit-risk specifications test that +conclusion more broadly below. + +![Resolved coverage (markers) and sharp all-candidate bounds (error bars) for the primary CatBoost and numeric-logistic specifications. The dashed line is nominal 90% coverage.](../reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png){#fig-coverage width=94%} + +Online Supplement Table S6 reports all eight exact window bounds and widths; +the figure retains their complete nonselective pattern in the main text. + +All five all-candidate sharp upper bounds remain below 0.90 in all eight windows, +including the monotonic learner and the pricing-excluded application scorecard. +The latter has the largest across-window upper endpoint, 0.8977, so even assigning every +unresolved candidate to coverage does not reach 0.90. + +| Score | OOT AUC | Brier | Slope | $U_{\max}$ | |---|---:|---:|---:|---:| -| 2018H1 | `$92,530.73` | `$118,101.99` | `0.106703` | `0.190825` | -| 2018H2 | `$156,185.51` | `$95,603.58` | `0.026725` | `0.236728` | -| 2019H1 | `$123,590.69` | `$144,281.46` | `0.077325` | `0.170275` | -| 2019H2 | `$110,251.95` | `$256,966.20` | `0.103250` | `0.023775` | -| 2020+ | `$99,689.54` | `$218,629.14` | `0.083775` | `0.016900` | - -: Fixed-policy temporal stress evaluation; each row uses a fresh `$1M` budget. - -CRPTO is economically and statistically attractive in 2018H2, when the -point-PD portfolio concentrates heavily in realized defaults. In 2019H2 and -2020+, the point-PD policy earns much more and defaults less. The honest result -is therefore a full-period return-risk trade-off with temporal heterogeneity, -not a universal robustness premium. This pattern also cautions against treating -a static conformal guardrail as a substitute for live recalibration. - -## Funded-Set Composition - -The selected allocation is concentrated in grades C and D: they represent -`31.36%` and `58.68%` of funded exposure. Grade F is only `1.30%` of exposure -but has a high realized default rate (`0.269231`), illustrating why aggregate -metrics should be accompanied by composition. A38 reports every funded grade -and reconciles to the full `$1M` allocation. Because Lending Club does not -provide the protected attributes needed for a legal fair-lending audit, this is -a business-risk composition check, not a fairness certification. - -# Robustness, Comparators, and Implications - -The online supplement contains the evidence that helps a reviewer challenge -the one active method without turning the paper into several methods: exact -alpha saturation (A35), all nine selector cells (A36), temporal evaluation -(A37), grade composition (A38), fixed-allocation bootstrap (A39), and matched -comparators (A40). Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, -and Freddie/Mendeley analyses are retained as diagnostics or external context. -They do not select the active policy or create additional claims. - -For a credit committee, the useful control is not the Markov threshold by -itself. The committee first chooses a conventional conformal level, a -transparent blend $\gamma$, and a portfolio tolerance $\tau$. It can then read -the economic cost, endpoint exposure, realized default, and temporal behavior -together. In this application the midpoint rule has a clear interpretation: -half of the conformal premium enters the enforceable risk score and half remains -visible as residual endpoint exposure. - -The matched baseline turns that interpretation into a decision. A committee -that accepts the full-period evidence pays `8.678%` of realized return for a -`7.9025` percentage-point reduction in weighted default. A more conservative -committee can move to the 75% blend and pay an additional `$6.4K`. A committee -that emphasizes the 2019H2 or 2020 regimes should reject the static CRPTO rule -or require a new temporal recalibration protocol. The method exposes that -choice rather than hiding it behind a single score. - -# Reproducibility and Limitations - -The research bundle versions code, manuscript sources, configurations, tables, -figures, and governance files together. Heavy data and model artifacts are -stored separately and checked by hashes. The active evidence builder regenerates -A35--A40 from the frozen exact-alpha and policy outputs. Claim-sync tests verify -that the body, supplement, and official IJDS LaTeX manuscript share the same -policy, selector counts, and numeric anchors. - -Several limitations bound the contribution. First, Lending Club retail -origination ended in 2020; the panel is historical and cannot establish live -post-2020 performance. Second, the final tagged selector is outcome-free with -respect to OOT ranking, but earlier project development inspected the same -static OOT corpus. The evaluation is therefore retrospective, not a new -prospective holdout. Third, the conformal intervals are broad because the -outcome is binary; at 90%, more than half of OOT upper endpoints equal one. -Fourth, marginal or Mondrian coverage does not imply validity under -optimizer-selected funded weights. The independent December audit makes this -visible: the same policy is reselected without outcomes, yet funded-set -miscoverage is `0.124925`. The Markov result is therefore conditional on an -explicit weighted-validity assumption and is absent from policy selection. -Fifth, temporal slices show that the point-PD comparator can dominate both -return and default. Finally, public data do not support a legal fair-lending -certification or a causal interpretation of policy contrasts. - -These limitations suggest a focused next step rather than a larger current -paper: a genuinely prospective or formally selection-valid protocol that -freezes conformal and policy choices before a new evaluation period -[@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets; -@zhou2026creme]. -It is not required to interpret the present retrospective decision audit. - -For double-anonymous review, author-identifying repository URLs are omitted -from the manuscript. The code/data companion and artifact-access instructions -can be disclosed under the journal's policy without embedding credentials or -identity in the anonymous PDF. - -# Conclusion - -CRPTO shows how a frozen credit model can become an auditable portfolio -decision without adding a maze of policy variants. An exact 90% conformal -replay produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a -nine-cell November selector fixes $\tau=0.17$ under the deterministic -$B_u\le0.28$ screen. December independently selects the same rule, then misses -nominal funded-set coverage; this is why CRPTO reports observed miscoverage -rather than claiming that policy stability implies conformal validity. -On the full OOT panel, the policy earns `$179,327.59`, with weighted default -`0.039375`, miscoverage `0.036875`, $\Gamma_{\mathrm{CP}}=0.176102$, -$\Gamma_{\mathrm{res}}=0.088051$, endpoint `0.258051`, observed accounting -bound `0.294926`, and conditional Markov threshold `0.574279`. Against the -matched point-PD allocation, it pays `8.678%` of realized return for a -`7.9025` percentage-point default reduction. Temporal reversals keep the claim -narrow: CRPTO is an inspectable retrospective return-risk guardrail, not a -universal winner, prospective deployment guarantee, or new credit-scoring -leaderboard. +| CatBoost | 0.6406 | 0.1299 | 0.7954 | 0.8826 | +| Logistic | 0.6420 | 0.1288 | 0.5432 | 0.8962 | +| Monotone CB | 0.6520 | 0.1286 | 0.8351 | 0.8865 | +| Platform WOE | 0.6331 | 0.1295 | 0.9187 | 0.8949 | +| Pricing-excluded WOE | 0.6129 | 0.1302 | 0.7098 | 0.8977 | + +: Five-model coverage-specification audit. $U_{\max}$ is the largest all-candidate coverage upper bound across eight windows. Predictive metrics use 364,814 resolved outcomes; coverage retains all 376,890 candidates. Full hulls are in Online Supplement Table S2. No OOT metric selects a model. {#tbl-credit-controls} + +All five scores underpredict later default on average and have calibration +slopes below one. Their AUC, calibration, WOE/IV, and PSI diagnostics are +reported in the Online Supplement as descriptive controls, not as a model +contest or an explanation of the coverage gap. + + + + + +Two complete sensitivities narrow the specification concern. The active +sentinel encoding, explicit missing indicators, and native nullable CatBoost +features each have all eight coverage upper bounds below 0.90, with maxima +0.882597, 0.884332, and 0.880037. At a later retrospective origin, all eight +CatBoost upper bounds are again below 0.90, with maximum 0.874768. No encoding, +model, or origin is selected. These are three semantics-preserving encoding +paths and two retrospective origins, not evidence about the missingness +mechanism, independent replications, or prospective temporal invariance. + +A separate four-scenario sensitivity addresses 215 labels unavailable at their +own fitting cutoffs: 41 in PD development, 24 in Platt calibration, and 150 in +conformal fitting. It refits the complete score-and-recipe pipeline after +retaining only observed labels, completing every unavailable label as +nondefault, completing every one as default, or using the later terminal +archive. Across all four scenarios and eight windows, every overall coverage +upper bound remains below 0.90; the scenario maxima range from 0.8826 to +0.884669. No completion is selected. + +The result is specific to the declared archive and multi-year transport gap; +it does not say that conformal prediction fails under exchangeability. It +occurs before the optimizer selects a funded set and therefore cannot identify +a preferred portfolio rule. + +## A prevalence crossing coincides with a change in binary interval geometry + + + + +The large W8 width change is concentrated in CatBoost stratum 2. Fit prevalence +declines from 0.1167 in W1 to 0.1017 in W7, then to 0.0971 in W8. That final +change crosses $\alpha=0.10$. The fitted residual quantile remains near 0.89 +through W7 but falls from 0.8884 to 0.1118 in W8; mean OOT width in the stratum +falls from 0.9843 to 0.2076. Its discrete intersection simultaneously moves +from 0.96% $\{0,1\}$ and 99.04% $\{0\}$ in W7 to 0% $\{0,1\}$, 99.74% +$\{0\}$, and 0.26% empty in W8. + +![CatBoost stratum-2 fit prevalence and residual quantile across all eight windows. The W7--W8 prevalence crossing coincides with a discontinuous quantile change.](../reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png){#fig-phase width=94%} + +This empirical pattern matches the mechanism in Proposition 2, but scores vary +within the stratum and the proposition is not used as a finite-sample proof. +The retrospectively protocol-locked fit-label sensitivity gives the same threshold crossing at +0, 3, and 6 months; their minimum monthly retention is 99.71%, 99.67%, and +99.30%, respectively. At 8 and 12 months the W7 quantile also lies near 0.112, +so the crossing disappears, but those specifications retain only 98.68% and +97.45% in their weakest month and fail the strict greater-than-99% rule. The evidence thus +supports stability across lags satisfying the strict retention rule, not a +claim that prevalence causally produced the empirical change. +Nor does the narrower W8 interval repair transport: stratum-2 all-candidate +coverage is only [0.8225, 0.8547]. A narrower interval can simply expose more +rare defaults as misses. + +The fit-label scenario sensitivity separates this explanation from the coverage +result. The observed-only, all-nondefault, and hindsight scenarios retain the +W7--W8 crossing. Under the all-default scenario, however, W7 and W8 prevalences +are 0.105974 and 0.100287 and both residual quantiles remain near 0.889, so the +crossing disappears even though all eight overall coverage upper bounds remain +below 0.90. Because model and calibration refits are nonlinear, these four +scenarios are not sharp bounds over all $2^{215}$ label assignments. The phase +mechanism explains one observed path; it is not invariant to every plausible +fit-label completion. + +This is the **fit-label timing** axis: it changes which 2012 labels enter +conformal fitting and therefore refits the residual recipes. The +**evaluation-endpoint availability** axis below instead holds scores, recipes, +supports, and allocations fixed while changing which outcomes are observable at +the September 2020 cutoff. The two one-factor sensitivities were not crossed +factorially, so they do not establish joint lag robustness. + +## Endpoint direction changes with ruler and coordinate + + + + + +The two-ruler freeze contains 6,240 portfolios, and the evaluator reports all +720 monthly and 48 complete-window endpoint contrasts. @tbl-two-ruler reports +$\gamma=1$ minus $\gamma=0$; *adverse* means lower payoff and higher +default/miscoverage, while *crossing* means the sharp interval contains zero. +Each row is one +ruler-coordinate track over the same fifteen OOT menus. The eight windows are +overlapping residual specifications, so the table is a six-track census, not +independent replications or votes. + +| Ruler | .25 | .50 | .75 | +|---|---|---|---| +| Objective matched | All three cross in 8/8; 4 active months | Adverse in 8/8 | Payoff/default cross in 7/8; miscoverage adverse in 8/8 | +| Normalized score | Adverse in 8/8 | Adverse in 8/8 | Payoff adverse in 7/8 and crossing in 1/8; default/miscoverage adverse in 8/8 | + +: Six protocol-locked endpoint tracks. Full sharp hulls are in Online Supplement Table S9. {#tbl-two-ruler} + +Under objective matching, coordinate .25 changes only four months. The +allocation contrast is identical across windows to cents: 44 loan-month +positions change and one-way turnover is USD 155,937.27 across USD 15 million. +Under the reconstructed endpoint, however, its status-indexed payoff-proxy bound is +[-USD 9,134.34, USD 5,603.66], and its default and miscoverage bounds are both +[-0.0068, 0.1265] percentage points. The repeated allocation is therefore one +unidentified contrast, not eight independent confirmations. Proposition 3 +reconciles these hulls: in every window the unresolved exposure disagreement +contributes exactly USD 14,738 of payoff-proxy width and 0.1333 percentage +points of default and miscoverage width. At .50, every +sharp window bound gives lower payoff and higher default and miscoverage. At +.75, payoff and default cross zero in seven windows; only W8 is adverse for +both, while miscoverage is higher in all eight. + +The 12,076 unresolved outcomes have three distinct recorded reasons: 11,551 +nonterminal or otherwise unresolved statuses, 47 terminal outcomes whose +reconstructed availability follows the cutoff, and 478 terminal statuses with +no reconstructible availability date. Treating the latter 525 as one timing +category would overstate what the archive identifies. They remain unrestricted +in the same common-outcome bounds; no missingness mechanism is assumed. + +The normalized ruler gives lower payoff and higher default and miscoverage in +all .25 and .50 cells. At .75, default and miscoverage remain higher in all +eight windows, while payoff is lower in seven and crosses zero in one. It also +gives $\gamma=1$ between USD 28,263 and USD 557,294 more optimized plug-in +objective than $\gamma=0$ across the declared coordinates. The ruler therefore +compares the same relative score relaxation, not the same opportunity cost. +Its status-indexed sign cannot be interpreted as a causal effect or combined with +the objective-matched tracks as evidence for a preferred endpoint. Across all +48 cells, payoff is lower in 32 and crosses zero in 16; default is higher in 33 +and crosses in 15; miscoverage is higher in 40 and crosses in 8. + +The complete retrospective endpoint-availability grid changes the Charged Off +administrative lag to 0, 3, 6, 8, and 12 months without refitting any score, +recipe, support, ruler, or allocation. Coverage upper bounds remain below 0.90 +in 40/40 cells at lags 0, 3, 6, and 8, but in 39/40 at 12 months, where the +maximum is 0.900411. No opposite one-sided portfolio direction appears, and all +216 broad-support envelopes include zero at every lag. The six-month slice +reconciles to the active evaluation in all 120 coverage cells, 48 two-ruler +contrasts, and 648 exact-support envelopes. Full counts are in Online +Supplement Appendix E. These results make the 40/40 statement +endpoint-contract-specific and do not select the six-month endpoint. + +## Structural assumptions do not restore a universal direction + + + + +A separate outcome-free freeze crosses monthly budgets of USD 0.5, 1, and 2 +million, maximum purpose shares of 0.20, 0.25, 0.30, and 1.00, and LGD values +of 0.25, 0.45, and 0.65. The evaluation reports the complete 36-scenario +Cartesian product without selecting a scenario. Each scenario retains both +rulers, all three coordinates, and all eight residual windows, giving 48 +guardrail-minus-point cells per metric and 1,728 cells over the structural +grid. + +Every scenario contains at least 17 cells with higher default and at least 21 +with higher miscoverage. No scenario makes all three metrics favorable in all +48 cells, but no scenario makes all three metrics adverse in all 48 cells +either. At least one favorable cell appears in 26 scenarios for payoff and 20 +each for default and miscoverage. The baseline scenario reconciles exactly to +the active two-ruler result. Thus budget, concentration, and LGD change the +mix of adverse, favorable, and unidentified cells, but do not supply a +scenario-independent ordering. Online Supplement Tables S9D--S9E report the +complete census and binding diagnostics. + +The active portfolios are continuous LP allocations. A deterministic USD 25 +floor-with-cash diagnostic covers all 1,440 baseline portfolios and 143,175 +positive source rows; 2,985 exposures change and monthly residual cash is USD +25--75. Across the 96 ruler-coordinate-window endpoint tracks, the largest +absolute perturbations are 0.000345 percentage points in payoff rate, 0.001284 +in default rate, and 0.001190 in miscoverage rate. This supports numerical +adequacy of the continuous relaxation at this granularity for the observed +archive. It is not an integer reoptimization or a claim about other lot rules. + +## Exact point-cap support checks the finite diagnostic + + + +The supporting C2 exercise reconciles Proposition 1, part 4, in all 1,080 cells: its maximum +funded-score residual is $8.33\times10^{-17}$ and point-minus-guardrail plug-in +objective never falls below numerical tolerance. Named C0/C1/C2 status-indexed +directions nevertheless disagree; their full census is reported in the Online +Supplement rather than treated as another policy contest. + +The exact point-cap frontier evaluates 3,067 distinct caps. Over [0.05, 0.12] +broad stress, all 216 policy-by-window-by-metric envelopes cross zero. The +development-admissible support is narrower: default still crosses zero in all +72 cells; payoff is lower in 6 and crosses in 66; miscoverage is higher in 27 +and crosses in 45. All 27 W8 envelopes cross zero. + +The evaluated-cap numerical audit covers 7,297 point-cap rows. Although 2,941 +bases are primal-degenerate, none has a near-zero nonbasic reduced cost, and +reversing candidate order changes no allocation beyond $1.45\times10^{-14}$. +This supports deterministic stability at the evaluated caps; it is not a claim +of uniqueness over an unenumerated continuous joint frontier. + +The exact frontier is not a seventh track. It checks the finite +two-ruler grid against a continuous point-cap support. Together the analyses +show that status-indexed direction depends on residual recipe, comparator support, +ruler, frontier coordinate, and declared portfolio structure; none identifies +a universally safer or more profitable conformal endpoint. + +# Discussion {#sec-discussion} + +## What the audit establishes + +The guardrail changes a score, feasible region, and allocation; copying a +numeric threshold cannot separate them. Objective matching fixes model-implied +opportunity cost, whereas normalization fixes a scale-invariant relative score +relaxation. Neither is neutral. The objective-matched result changes from +unidentified at .25 to adverse at .50 and mostly unidentified at .75; the +mostly adverse normalized result compares different plug-in objective levels. +Selecting a ruler or coordinate by its sign would recreate the identification +problem. + +C0 proves mechanical same-threshold nesting; C2 removes one funded point-score +moment; and the exact cap frontier shows that neither is a unique +counterfactual. None of the 216 broad-stress envelopes has a fixed sign. Some +payoff and miscoverage signs survive the narrower development support, but +default never does and all three metrics become unidentified in W8. This exact +support result is the safeguard against overinterpreting a finite grid, not a +reason to privilege the broadest support after outcomes. + +The complete structural grid adds a different safeguard. It shows that the +absence of a universally favorable direction is not an artifact of one budget, +purpose cap, or LGD, while favorable cells under many scenarios prevent the +opposite claim of universal harm. Structural sensitivity therefore sharpens +the identification boundary; it does not identify a preferred scenario. + +The upper score is pointwise larger and, at a shared cap, mechanically more +restrictive; neither fact supplies OOT coverage or funded-set validity. +*Robust* must name a protected quantity and perturbation +[@bertsimas2004; @bertsimas2018datadriven]. Here it +describes only the observed coverage failure across complete windows and five +learner specifications, not causal benefit, selected-set validity, or policy +dominance. CRPTO therefore uses *risk-aware* in its expanded name. + +## Implications for decision-focused data science + +Three implications may transfer beyond credit, without establishing external +validity. First, define the decision estimand: name the matched quantity and +binding constraints, and use exact support analysis when LP structure permits. +Second, preserve the decision information set. Status-independent menus, dated +labels, complete eligible windows, and unresolved-outcome bounds prevent later +status from redefining an earlier choice. Third, separate predictive, geometric, +and decision claims: marginal coverage is not funded-set validity, narrower +binary intervals need not improve decisions, controls are not a leaderboard, +and optimized and evaluated payoffs must agree. + +## Ethical and governance implications + +Calling one credit score *safer* can redirect capital because of a copied cap +rather than better predictive validity. Policy claims should disclose the +comparator, calibration window, binding constraints, and unresolved-outcome +treatment. The accepted-loan archive has neither rejected-applicant +counterfactuals nor a disparate-impact design; CRPTO audits model-risk claims +and does not authorize deployment or fair-lending conclusions. + +# Limitations {#sec-limitations} + +One discontinued accepted-loan archive cannot establish live, external, or +fair-lending performance. It is not a verified point-in-time snapshot, and its +terminal labels and status-indexed payoff proxy omit cash-flow timing, +prepayment, recoveries, fees, discounting, and capital costs. Sixty-month loans, +immature cohorts, and 48 late-schema fields are excluded; disclosed mappings +cover delinquency recency and legacy bankruptcy count. The encoding audit does +not identify a missingness mechanism. + +The learners, overlapping windows, and second origin share one chronology, so +they are neither independent nor prospective replications. Temporal separation +does not restore exchangeability, candidate coverage does not imply funded-set +coverage, and the constant-score theorem identifies a mechanism rather than the +varying-score empirical path. The four fit-label scenarios are not exhaustive +nonlinear bounds. Likewise, the finite ruler grid, one-moment C2, and point-cap +support are not a continuous joint frontier; the structural grid is stylized, +and USD 25 flooring is not integer optimization. The inspected archive is +neither a pristine lockbox nor a preregistration. + +# Reproducibility {#sec-reproducibility} + +One registry binds every active lineage and 31 DVC pointers. Publication replay +verifies frozen artifacts and regenerates 18 tables and three figures +transactionally; scientific replay uses each lineage's protocol commit and +`uv.lock`, not the current source tree. Online Supplement Appendix F specifies +both modes. + +# Conclusion {#sec-conclusion} + +CRPTO shows why connecting machine learning, conformal prediction, and +optimization creates an identification problem rather than a self-validating +pipeline. Coverage nontransport persists across the declared learners and +fit-label scenarios, but the observed geometric crossing is not +scenario-invariant. Portfolio direction changes with the outcome-blind ruler +and coordinate, and all 216 broad-support exact envelopes include zero. + +CRPTO therefore selects no model or policy. Its executable audit keeps +unresolved outcomes, coverage, binary geometry, sharp bounds, and comparator +support distinct: portfolio direction is meaningful only after its estimand +and comparator are specified. diff --git a/paper/README.md b/paper/README.md index a494001..c0486bd 100644 --- a/paper/README.md +++ b/paper/README.md @@ -1,70 +1,54 @@ -# paper-crpto Manuscript Workspace +# CRPTO Manuscript Workspace -This folder contains the manuscript extraction layer for the standalone CRPTO -paper. The Quarto book remains the full companion dossier; this folder is where -the submission-shaped versions are written. +This directory contains one active IJDS manuscript and one online supplement. -## Current Venue Decision +## Active Sources -- Primary target: INFORMS Journal on Data Science. -- Secondary pivot: European Journal of Operational Research. -- Config source: `../configs/crpto_publication_targets.yaml`. -- Strategy memo: `../docs/research/crpto_publication_strategy_2026-05-12.md`. +- Claim registry: `../docs/research/active_claims_2026-07-14.md`. +- Evidence source registry: `../configs/ijds_active_evidence_sources.yaml`. +- Paper-facing evidence: `../reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`. +- Canonical body: `CRPTO_ijds.qmd`. +- Canonical supplement: `supplement_ijds.qmd`. +- Generated official TeX: `submission/CRPTO_ijds_submission.tex`. -## Files +The paper is a retrospective identification audit of one integrated +ML--conformal--optimization pipeline. The distributed archive is not a verified +point-in-time snapshot, so the endpoint is reconstructed as observable by the +declared cutoff. All 376,890 primary candidates remain in the menus; 364,814 +are resolved and 12,076 enter sharp binary bounds. No learner, window, gamma, +ruler, coordinate, comparator, or policy is selected. -- `CRPTO.qmd`: generic landing manuscript stub. -- `CRPTO_ijds.qmd`: active IJDS-style anonymous body source and current - manuscript source of truth. -- `supplement_ijds.qmd`: IJDS-style online supplement source. -- `submission/README.md`: IJDS handoff checklist, anonymity guardrails, PDF - draft commands, and SPO+ numbering rule. -- `submission/COVER_LETTER_AND_DISCLOSURE.md`: editor-facing cover letter and - data/code disclosure draft; keep it out of the anonymous reviewer packet - unless the submission system asks for disclosure text. -- `submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md`: target-date readiness - plan and 15-track improvement checklist. -- `submission/CLAIM_AUDIT_MATRIX.md`: claim-to-evidence map and reviewer - objection bank. -- `submission/REPRODUCIBILITY_PACKAGE.md`: IJDS data/code disclosure and - accepted-paper reproducibility plan. -- `submission/TITLE_PAGE_DRAFT.md`: non-anonymous title-page draft for - ScholarOne. -- `submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md`: working text for the official - IJDS disclosure form. -- `submission/SCHOLARONE_FINAL_CHECKLIST.md`: final upload/proof checklist. +Earlier manuscript versions are outside the active capsule and preserved in +Git history and `D:\crpto_legacy`. `CRPTO_ijds.qmd` is the only body source; +never edit generated TeX directly. -The active paper has one method: exact 90% conformal replay, the midpoint -guardrail `q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under -`B_u<=0.28`. A35 is the exact-alpha audit, A36 is the split selector/December -audit, A37 is temporal evaluation, A38 is letter-grade composition, A39 is the -month-cluster bootstrap with loan-level sensitivity, and A40 is the matched -point-PD comparison. OCE/CVaR, SPO+, satisficing, online-style checks, -and Prosper/Freddie replications remain supplement diagnostics; they do not -select or redefine the midpoint policy. Prospective validation, causal variants, -live recalibration, production, and package tracks remain outside the claim. - -## Render Commands +## Build ```powershell -just ijds-evidence -just paper-ijds -just paper-ijds-supplement -just paper-submission -just paper-submission-pdf +just submission-build ``` -The HTML render is the writing preview; the PDF render is an HTML-print -verification draft produced from the anonymous previews. The final submission -PDF should use the official venue template. These Quarto files are the writing -source of truth until the IJDS LaTeX template is applied with double-anonymous -settings. +The official compiler attempts `latexmk`; its robust fallback is +`pdflatex -> bibtex -> pdflatex -> pdflatex`. The first LaTeX pass writes the +auxiliary graph, BibTeX writes the bibliography, and the final two passes +resolve citations, labels, floats, and pagination. + +## Validation + +```powershell +just ijds-active-check +just paper-tex-check +just paper-official-scan +just validate-champion +just type-check +just type-check-fast +git status --short +``` -## Closeout Gates +Page counts and visual QA records belong in `submission/README.md` and must be +regenerated after substantive edits. The project remains pre-freeze until an +explicit submission-freeze decision. -Before tagging a submission release, do a final sweep for stale numbers, -captions, body-vs-supplement placement, and IJDS length. Keep public GitHub, -DagsHub, and MLflow links anonymized in the manuscript unless the venue policy -or cover-letter disclosure requires otherwise. -Use the roadmap and claim audit matrix as the final editorial checklist before -freezing the official `informs4` PDF. +`submission-build` writes paper-facing evidence and document outputs in causal +order. `submission-check` and `ijds-active-check` verify the current outputs +without replaying evidence or a scientific protocol. diff --git a/book/apa.csl b/paper/apa.csl similarity index 100% rename from book/apa.csl rename to paper/apa.csl diff --git a/paper/ijds.css b/paper/ijds.css index 134aee5..e1be777 100644 --- a/paper/ijds.css +++ b/paper/ijds.css @@ -31,10 +31,33 @@ } #quarto-content { - grid-template-rows: [content-top] auto [content-bottom] 0 [page-bottom] !important; + display: block !important; + grid-template-rows: none !important; } #quarto-appendix { - break-after: avoid-page; + break-after: auto !important; + page-break-after: auto !important; + } + + .keep-together { + break-inside: avoid-page !important; + font-size: 0.88em; + line-height: 1.12; + page-break-inside: avoid !important; + } + + .keep-together th, + .keep-together td { + padding: 0.28rem 0.36rem !important; + } + + #refs { + font-size: 0.80em; + line-height: 1.12; + } + + #refs .csl-entry { + margin-bottom: 0.12rem; } } diff --git a/paper/references.bib b/paper/references.bib new file mode 100644 index 0000000..7464255 --- /dev/null +++ b/paper/references.bib @@ -0,0 +1,1477 @@ +@article{angelopoulos2023, + author = {Angelopoulos, Anastasios N. and Bates, Stephen}, + title = {Conformal Prediction: A Gentle Introduction}, + journal = {Foundations and Trends in Machine Learning}, + volume = {16}, + number = {4}, + pages = {494--591}, + year = {2023}, + doi = {10.1561/2200000101}, + url = {https://arxiv.org/abs/2107.07511} +} + +@inproceedings{angelopoulos2024risk, + author = {Angelopoulos, Anastasios N. and Bates, Stephen and Fisch, Adam and Lei, Lihua and Schuster, Tal}, + title = {Conformal Risk Control}, + booktitle = {ICLR 2024}, + year = {2024}, + url = {https://openreview.net/forum?id=33XGfHLtZg} +} + +@article{angelopoulos2025ltt, + author = {Angelopoulos, Anastasios N. and Bates, Stephen and Cand{\`e}s, Emmanuel J. and Jordan, Michael I. and Lei, Lihua}, + title = {Learn then Test: Calibrating Predictive Algorithms to Achieve Risk Control}, + journal = {The Annals of Applied Statistics}, + year = {2025}, + doi = {10.1214/24-AOAS1998}, + url = {https://arxiv.org/abs/2110.01052} +} + +@article{athey2019, + author = {Athey, Susan and Wager, Stefan}, + title = {Estimating Treatment Effects with Causal Forests: An Application}, + journal = {Annals of Statistics}, + year = {2019}, + doi = {10.1214/18-AOS1709} +} + +@article{ayari2026, + author = {Ayari, Helmi and Guetari, Ramzi and Kra{\"\i}em, Naoufel}, + title = {Machine learning powered financial credit scoring: A systematic literature review}, + journal = {Artificial Intelligence Review}, + volume = {59}, + number = {1}, + pages = {13}, + year = {2026}, + doi = {10.1007/s10462-025-11416-2} +} + +@inproceedings{bai2021, + author = {Bai, Yu and Mei, Song and Wang, Huan and Xiong, Caiming}, + title = {Don't Just Blame Over-parametrization for Over-confidence: Theoretical Analysis of Calibration in Binary Classification}, + booktitle = {Proceedings of the 38th International Conference on Machine Learning (ICML)}, + year = {2021}, + pages = {566--576}, + publisher = {PMLR} +} + +@inproceedings{bairaktari2025, + author = {Bairaktari, Konstantina and Wu, Jiayun and Wu, Steven}, + title = {Kandinsky Conformal Prediction: Beyond Class- and Covariate-Conditional Coverage}, + booktitle = {Proceedings of the 42nd International Conference on Machine Learning}, + series = {Proceedings of Machine Learning Research}, + volume = {267}, + pages = {2581--2602}, + publisher = {PMLR}, + year = {2025}, + url = {https://proceedings.mlr.press/v267/bairaktari25a.html} +} + +@article{bao2025croms, + author = {Bao, Yajie and Hu, Yang and Ren, Haojie and Zhao, Peng and Zou, Changliang}, + title = {Optimal Model Selection for Conformalized Robust Optimization ({CROMS})}, + journal = {arXiv preprint arXiv:2507.04716}, + year = {2025}, + url = {https://arxiv.org/abs/2507.04716} +} + +@article{barber2021limits, + author = {Barber, Rina Foygel and Cand{\`e}s, Emmanuel J. and Ramdas, Aaditya and Tibshirani, Ryan J.}, + title = {The Limits of Distribution-Free Conditional Predictive Inference}, + journal = {Information and Inference: A Journal of the IMA}, + volume = {10}, + number = {2}, + pages = {455--482}, + year = {2021}, + doi = {10.1093/imaiai/iaaa017} +} + +@article{barcena2024, + author = {Saavedra, Cayan Atreio Portela B{\'a}rcena and Fachini-Gomes, Juliana Betini and Gomes, Eduardo Monteiro de Castro and Kimura, Herbert}, + title = {Probability of default for lifetime credit loss for {IFRS} 9 using machine learning competing risks survival analysis models}, + journal = {Expert Systems with Applications}, + volume = {249}, + pages = {123607}, + year = {2024}, + doi = {10.1016/j.eswa.2024.123607} +} + +@article{bates2021rcps, + author = {Bates, Stephen and Angelopoulos, Anastasios N. and Lei, Lihua and Malik, Jitendra and Jordan, Michael I.}, + title = {Distribution-Free, Risk-Controlling Prediction Sets}, + journal = {Journal of the ACM}, + volume = {68}, + number = {6}, + pages = {43:1--43:34}, + year = {2021}, + doi = {10.1145/3478535}, + url = {https://arxiv.org/abs/2101.02703} +} + +@book{chen2024creditrisk, + author = {Chen, Colin}, + title = {Practical Credit Risk and Capital Modeling, and Validation: {CECL}, Basel Capital, {CCAR}, and Credit Scoring with Examples}, + year = {2024}, + publisher = {Springer Cham}, + doi = {10.1007/978-3-031-52542-1} +} + +@article{bertsimas2004, + author = {Bertsimas, Dimitris and Sim, Melvyn}, + title = {The Price of Robustness}, + journal = {Operations Research}, + year = {2004}, + doi = {10.1287/opre.1030.0065} +} + +@article{bertsimas2018datadriven, + author = {Bertsimas, Dimitris and Gupta, Vishal and Kallus, Nathan}, + title = {Data-Driven Robust Optimization}, + journal = {Mathematical Programming}, + volume = {167}, + number = {2}, + pages = {235--292}, + year = {2018}, + doi = {10.1007/s10107-017-1125-8} +} + +@article{bertsimas2020prescriptive, + author = {Bertsimas, Dimitris and Kallus, Nathan}, + title = {From Predictive to Prescriptive Analytics}, + journal = {Management Science}, + volume = {66}, + number = {3}, + pages = {1025--1044}, + year = {2020}, + doi = {10.1287/mnsc.2018.3253} +} + +@book{bental2009robust, + author = {Ben-Tal, Aharon and El Ghaoui, Laurent and Nemirovski, Arkadi}, + title = {Robust Optimization}, + publisher = {Princeton University Press}, + year = {2009}, + url = {https://press.princeton.edu/books/hardcover/9780691143682/robust-optimization} +} + +@article{bennett1962, + author = {Bennett, George}, + title = {Probability Inequalities for the Sum of Independent Random Variables}, + journal = {Journal of the American Statistical Association}, + volume = {57}, + number = {297}, + pages = {33--45}, + year = {1962}, + doi = {10.1080/01621459.1962.10482149} +} + +@book{boucheron2013concentration, + author = {Boucheron, St{\'e}phane and Lugosi, G{\'a}bor and Massart, Pascal}, + title = {Concentration Inequalities: A Nonasymptotic Theory of Independence}, + publisher = {Oxford University Press}, + year = {2013}, + doi = {10.1093/acprof:oso/9780199535255.001.0001}, + url = {https://academic.oup.com/book/26549} +} + +@inproceedings{cantelli1929, + author = {Cantelli, Francesco Paolo}, + title = {Sui confini della probabilit{\`a}}, + booktitle = {Atti del Congresso Internazionale dei Matematici: Bologna del 3 al 10 de settembre di 1928}, + volume = {6}, + pages = {47--60}, + year = {1929} +} + +@article{freedman1975, + author = {Freedman, David A.}, + title = {On Tail Probabilities for Martingales}, + journal = {The Annals of Probability}, + volume = {3}, + number = {1}, + pages = {100--118}, + year = {1975}, + doi = {10.1214/aop/1176996452}, + url = {https://projecteuclid.org/journals/annals-of-probability/volume-3/issue-1/On-tail-probabilities-for-martingales/10.1214/aop/1176996452.full} +} + +@article{ghosh2002, + author = {Ghosh, Bimal K.}, + title = {Probability Inequalities Related to Markov's Theorem}, + journal = {The American Statistician}, + volume = {56}, + number = {3}, + pages = {186--190}, + year = {2002}, + doi = {10.1198/000313002119} +} + +@article{hoeffding1963, + author = {Hoeffding, Wassily}, + title = {Probability Inequalities for Sums of Bounded Random Variables}, + journal = {Journal of the American Statistical Association}, + volume = {58}, + number = {301}, + pages = {13--30}, + year = {1963}, + doi = {10.1080/01621459.1963.10500830} +} + +% --- F. Credit Risk, IFRS9 & Survival Analysis --- + +@inproceedings{bostrom2021, + author = {Bostr{\"o}m, Henrik and Johansson, Ulf and L{\"o}fstr{\"o}m, Tuwe}, + title = {Mondrian Conformal Predictive Distributions}, + booktitle = {Proceedings of the Tenth Symposium on Conformal and Probabilistic Prediction and Applications}, + series = {Proceedings of Machine Learning Research}, + volume = {152}, + pages = {24--38}, + publisher = {PMLR}, + year = {2021}, + url = {https://proceedings.mlr.press/v152/bostrom21a.html} +} + +@article{botha2026, + author = {Botha, Melvin and Verster, Tanja}, + title = {Approaches for modelling term-structure of default risk under {IFRS} 9}, + journal = {International Journal of Data Science and Analytics}, + year = {2026}, + url = {https://arxiv.org/abs/2501.04975} +} + +@inproceedings{capitaine2026online, + author = {Capitaine, Aymeric and Haddouche, Maxime and Moulines, Eric and Jordan, Michael I. and Boursier, Etienne and Durmus, Alain}, + title = {Online Decision-Focused Learning}, + booktitle = {ICLR 2026}, + year = {2026}, + doi = {10.48550/arXiv.2505.13564}, + url = {https://arxiv.org/abs/2505.13564} +} + +@article{causalfinance2023, + title = {Causal Inference for Banking, Finance, and Insurance: A Survey}, + year = {2023}, + url = {https://arxiv.org/abs/2307.16427} +} + +% --- I. Calibration --- + +@inproceedings{chan2024inverse, + author = {Chan, Timothy C.Y. and Delage, Erick and Lin, Bo}, + title = {Conformal Inverse Optimization}, + booktitle = {NeurIPS 2024}, + year = {2024}, + url = {https://arxiv.org/abs/2402.01489} +} + +@inproceedings{chenreddy2024, + author = {Chenreddy, Abhilash Reddy and Delage, Erick}, + title = {End-to-End Conditional Robust Optimization}, + booktitle = {Proceedings of the Fortieth Conference on Uncertainty in Artificial Intelligence}, + series = {Proceedings of Machine Learning Research}, + volume = {244}, + pages = {736--748}, + publisher = {PMLR}, + year = {2024}, + url = {https://arxiv.org/abs/2403.04670} +} + +@inproceedings{cortesgomez2025utility, + author = {Cortes-Gomez, Santiago and Patino, Carlos and Byun, Yewon and Wu, Zhiwei Steven and Horvitz, Eric and Wilder, Bryan}, + title = {Utility-Directed Conformal Prediction: A Decision-Aware Framework for Actionable Uncertainty Quantification}, + booktitle = {ICLR 2025}, + year = {2025} +} + +@article{chernozhukov2018, + author = {Chernozhukov, Victor and Chetverikov, Denis and Demirer, Mert and Duflo, Esther and Hansen, Christian and Newey, Whitney and Robins, James}, + title = {Double/Debiased Machine Learning for Treatment and Structural Parameters}, + journal = {The Econometrics Journal}, + volume = {21}, + number = {1}, + pages = {C1--C68}, + year = {2018}, + doi = {10.1111/ectj.12097} +} + +@article{christoffersen1998, + author = {Christoffersen, Peter F.}, + title = {Evaluating Interval Forecasts}, + journal = {International Economic Review}, + volume = {39}, + number = {4}, + pages = {841--862}, + year = {1998}, + doi = {10.2307/2527341} +} + +@article{cpsurvey2025, + author = {Zhou, Xiaofan and Chen, Baiting and Gui, Yu and Cheng, Lu}, + title = {Conformal Prediction: A Data Perspective}, + journal = {ACM Computing Surveys}, + year = {2025}, + doi = {10.1145/3736575} +} + +@inproceedings{ding2023, + author = {Ding, Tiffany and Angelopoulos, Anastasios N. and Bates, Stephen and Jordan, Michael I. and Tibshirani, Ryan J.}, + title = {Class-Conditional Conformal Prediction with Many Classes}, + booktitle = {Advances in Neural Information Processing Systems}, + volume = {36}, + pages = {64555--64576}, + year = {2023}, + url = {https://arxiv.org/abs/2306.09335} +} + +@inproceedings{donti2017, + author = {Donti, Priya L. and Amos, Brandon and Kolter, J. Zico}, + title = {Task-based End-to-end Model Learning in Stochastic Optimization}, + booktitle = {NeurIPS}, + year = {2017}, + url = {https://arxiv.org/abs/1703.04529} +} + +@techreport{ecb2024, + author = {{European Central Bank}}, + title = {{IFRS} 9 overlays and model improvements for novel risks}, + institution = {ECB Banking Supervision}, + year = {2024} +} + +@book{efron1994, + author = {Efron, Bradley and Tibshirani, Robert J.}, + title = {An Introduction to the Bootstrap}, + publisher = {Chapman and Hall/CRC}, + year = {1994} +} + +@article{elmachtoub2022, + author = {Elmachtoub, Adam N. and Grigas, Paul}, + title = {Smart ``Predict, then Optimize''}, + journal = {Management Science}, + volume = {68}, + number = {1}, + year = {2022}, + doi = {10.1287/mnsc.2020.3922} +} + +@article{fantazzini2024, + author = {Fantazzini, Dean}, + title = {Adaptive Conformal Inference for Computing Market Risk Measures}, + journal = {Journal of Risk and Financial Management}, + year = {2024} +} + +% --- E. Decision-Focused Learning / Predict-then-Optimize --- + +@inproceedings{gibbs2021aci, + author = {Gibbs, Isaac and Cand{\`e}s, Emmanuel}, + title = {Adaptive Conformal Inference Under Distribution Shift}, + booktitle = {NeurIPS}, + year = {2021}, + url = {https://arxiv.org/abs/2106.00170} +} + +% --- Additional key references --- + +@article{gibbs2024, + author = {Gibbs, Isaac and Cherian, John J. and Cand{\`e}s, Emmanuel J.}, + title = {Conformal Prediction With Conditional Guarantees}, + journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology}, + volume = {87}, + number = {4}, + pages = {1100--1126}, + year = {2025}, + doi = {10.1093/jrsssb/qkaf008}, + url = {https://arxiv.org/abs/2305.12616} +} + +@article{barber2023beyond, + author = {Barber, Rina Foygel and Cand{\`e}s, Emmanuel J. and Ramdas, Aaditya and Tibshirani, Ryan J.}, + title = {Conformal Prediction Beyond Exchangeability}, + journal = {The Annals of Statistics}, + year = {2023}, + url = {https://arxiv.org/abs/2202.13415} +} + +@inproceedings{liu2024multisource, + author = {Liu, Yi and Levis, Alexander W. and Normand, Sharon-Lise T. and Han, Larry}, + title = {Multi-Source Conformal Inference Under Distribution Shift}, + booktitle = {Proceedings of the 41st International Conference on Machine Learning}, + series = {Proceedings of Machine Learning Research}, + volume = {235}, + year = {2024}, + publisher = {PMLR} +} + +@inproceedings{jonkers2024wcps, + author = {Jonkers, Jef and Van Wallendael, Glenn and Duchateau, Luc and Van Hoecke, Sofie}, + title = {Conformal Predictive Systems Under Covariate Shift}, + booktitle = {Conformal and Probabilistic Prediction with Applications}, + series = {Proceedings of Machine Learning Research}, + volume = {230}, + pages = {1--18}, + year = {2024}, + publisher = {PMLR} +} + +@article{gopalan2022, + author = {Gopalan, Parikshit and Luss, Ronny and Sharan, Vatsal and Urner, Ruth}, + title = {Multicalibrated Partitions for Importance Weights}, + journal = {arXiv preprint arXiv:2103.05853}, + year = {2022} +} + +@inproceedings{guo2017, + author = {Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q.}, + title = {On Calibration of Modern Neural Networks}, + booktitle = {Proceedings of the 34th International Conference on Machine Learning (ICML)}, + year = {2017}, + pages = {1321--1330}, + publisher = {PMLR} +} + +@article{guo2016p2p, + author = {Guo, Yanhong and Zhou, Wenjun and Luo, Chunyu and Liu, Chuanren and Xiong, Hui}, + title = {Instance-Based Credit Risk Assessment for Investment Decisions in {P2P} Lending}, + journal = {European Journal of Operational Research}, + volume = {249}, + number = {2}, + pages = {417--426}, + year = {2016}, + doi = {10.1016/j.ejor.2015.05.050} +} + +@inproceedings{hardt2016, + author = {Hardt, Moritz and Price, Eric and Srebro, Nathan}, + title = {Equality of Opportunity in Supervised Learning}, + booktitle = {Advances in Neural Information Processing Systems}, + volume = {29}, + year = {2016} +} + +% --- L. Quantum Machine Learning & Quantum Conformal --- + +@article{huangfu_parallelizing_2018, + author = {Huangfu, Qi and Hall, J. A. J.}, + title = {Parallelizing the dual revised simplex method}, + journal = {Mathematical Programming Computation}, + volume = {10}, + number = {1}, + pages = {119--142}, + year = {2018}, + doi = {10.1007/s12532-017-0130-5} +} + +@techreport{ifrsboard2024, + author = {{IFRS Board}}, + title = {{SICR} Feedback Analysis}, + institution = {IFRS Foundation}, + year = {2024} +} + +% --- G. ML in Credit Scoring --- + +@article{iutzeler2025dro, + author = {Iutzeler, Franck and Mazoyer, Cl{\'e}ment}, + title = {Risk-controlling Prediction with Distributionally Robust Optimization}, + journal = {Transactions on Machine Learning Research}, + year = {2025} +} + +% --- D. Conformal Prediction in Finance & Portfolios --- + +@inproceedings{johnstone2021, + author = {Johnstone, Chancellor and Cox, Bruce}, + title = {Conformal Uncertainty Sets for Robust Optimization}, + booktitle = {COPA (PMLR v152)}, + year = {2021}, + url = {https://proceedings.mlr.press/v152/johnstone21a.html} +} + +@article{kato2025, + author = {Kato, Masahiro}, + title = {Conformal Predictive Portfolio Selection}, + journal = {arXiv preprint arXiv:2410.16333}, + year = {2025}, + url = {https://arxiv.org/abs/2410.16333} +} + +@inproceedings{kiyani2025, + author = {Kiyani, Shayan and Pappas, George J. and Roth, Aaron and Hassani, Hamed}, + title = {Decision Theoretic Foundations for Conformal Prediction: Optimal Uncertainty Quantification for Risk-Averse Agents}, + booktitle = {Proceedings of the 42nd International Conference on Machine Learning}, + series = {Proceedings of Machine Learning Research}, + volume = {267}, + pages = {30943--30965}, + publisher = {PMLR}, + year = {2025}, + url = {https://proceedings.mlr.press/v267/kiyani25a.html} +} + +@article{lekeufack2023cdt, + author = {Lekeufack, Jordan and Angelopoulos, Anastasios N. and Bajcsy, Andrea and Jordan, Michael I. and Malik, Jitendra}, + title = {Conformal Decision Theory: Safe Autonomous Decisions from Imperfect Predictions}, + journal = {arXiv preprint}, + year = {2023} +} + +@inproceedings{kull2017, + author = {Kull, Meelis and Silva Filho, Telmo and Flach, Peter}, + title = {Beta Calibration: A Well-Founded and Easily Implemented Improvement on Logistic Calibration for Binary Classifiers}, + booktitle = {Proceedings of the 20th International Conference on Artificial Intelligence and Statistics (AISTATS)}, + year = {2017}, + pages = {623--631}, + publisher = {PMLR} +} + +@inproceedings{kumar2019, + author = {Kumar, Ananya and Liang, Percy and Ma, Tengyu}, + title = {Verified Uncertainty Calibration}, + booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, + year = {2019}, + volume = {32} +} + +@article{kupiec1995, + author = {Kupiec, Paul H.}, + title = {Techniques for Verifying the Accuracy of Risk Measurement Models}, + journal = {The Journal of Derivatives}, + volume = {3}, + number = {2}, + pages = {73--84}, + year = {1995}, + doi = {10.3905/jod.1995.407942} +} + +@article{lessmann2015, + author = {Lessmann, Stefan and Baesens, Bart and Seow, Hsin-Vonn and Thomas, Lyn C.}, + title = {Benchmarking state-of-the-art classification algorithms for credit scoring: An update of research}, + journal = {European Journal of Operational Research}, + volume = {247}, + number = {1}, + pages = {124--136}, + year = {2015}, + doi = {10.1016/j.ejor.2015.05.030} +} + +@article{liu2026portfolio, + author = {Liu, Tuo and Dobriban, Edgar and Orabona, Francesco}, + title = {Online Conformal Prediction via Universal Portfolio Algorithms}, + journal = {arXiv preprint arXiv:2602.03168}, + year = {2026}, + url = {https://arxiv.org/abs/2602.03168} +} + +@article{lopezpaz2018c2st, + author = {Lopez-Paz, David and Oquab, Maxime}, + title = {Revisiting Classifier Two-Sample Tests}, + journal = {arXiv preprint arXiv:1610.06545}, + year = {2018}, + url = {https://arxiv.org/abs/1610.06545} +} + +@article{mandi2024, + author = {Mandi, Jayanta and Kotary, James and Berden, Senne and Mulamba, Maxime and Bucarey, Victor and Guns, Tias and Fioretto, Ferdinando}, + title = {Decision-Focused Learning: Foundations, State of the Art, Benchmark and Future Opportunities}, + journal = {Journal of Artificial Intelligence Research}, + volume = {81}, + pages = {1623--1701}, + year = {2024}, + doi = {10.1613/jair.1.15320}, + url = {https://arxiv.org/abs/2307.13565} +} + +@article{manokhin2024, + author = {Manokhin, Valery}, + title = {Awesome Conformal Prediction: A Curated List of Resources}, + journal = {arXiv preprint arXiv:2407.16613}, + year = {2024} +} + +@article{murphy1973, + author = {Murphy, Allan H.}, + title = {A New Vector Partition of the Probability Score}, + journal = {Journal of Applied Meteorology}, + volume = {12}, + number = {4}, + pages = {595--600}, + year = {1973} +} + +@article{noguer2024portfolio, + author = {{Noguer i Alonso}, Miquel}, + title = {Conformal Portfolio Optimization}, + year = {2024}, + url = {https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5011129} +} + +@article{noguer2024survey, + author = {{Noguer i Alonso}, Miquel}, + title = {Conformal Prediction in Finance}, + year = {2024}, + url = {https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4939336} +} + +@incollection{papadopoulos2002, + author = {Papadopoulos, Harris and Proedrou, Kostas and Vovk, Vladimir and Gammerman, Alex}, + title = {Inductive Confidence Machines for Regression}, + booktitle = {Machine Learning: ECML 2002}, + series = {Lecture Notes in Computer Science}, + volume = {2430}, + pages = {345--356}, + publisher = {Springer}, + year = {2002}, + doi = {10.1007/3-540-36755-1_29} +} + +@inproceedings{patel2024, + author = {Patel, Yash P. and Rayan, Sahana and Tewari, Ambuj}, + title = {Conformal Contextual Robust Optimization}, + booktitle = {Proceedings of the 27th International Conference on Artificial Intelligence and Statistics}, + series = {Proceedings of Machine Learning Research}, + volume = {238}, + pages = {2485--2493}, + publisher = {PMLR}, + year = {2024}, + url = {https://proceedings.mlr.press/v238/patel24a.html} +} + +@article{plassier2024, + author = {Plassier, Vincent and Fishkov, Alexander and Guizani, Mohsen and Panov, Maxim and Moulines, Eric}, + title = {Probabilistic Conformal Prediction with Approximate Conditional Validity}, + journal = {ICLR 2025}, + year = {2025}, + url = {https://openreview.net/forum?id=Nfd7z9d6Bb} +} + +% --- C. Conformal Prediction + Robust Optimization --- + +@incollection{platt1999, + author = {Platt, John C.}, + title = {Probabilistic Outputs for Support Vector Machines}, + booktitle = {Advances in Large Margin Classifiers}, + year = {1999}, + publisher = {MIT Press} +} + +@book{powell2026sdam, + author = {Powell, Warren B.}, + title = {Sequential Decision Analytics and Modeling: Modeling with {Python}}, + edition = {2nd}, + year = {2026}, + publisher = {Princeton University}, + note = {Kindle Direct Publishing} +} + +@inproceedings{romano2019, + author = {Romano, Yaniv and Patterson, Evan and Cand{\`e}s, Emmanuel J.}, + title = {Conformalized Quantile Regression}, + booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, + year = {2019}, + url = {https://arxiv.org/abs/1905.03222} +} + +@article{sicr2025, + title = {Defining and comparing {SICR}-events under {IFRS} 9}, + journal = {Annals of Operations Research}, + year = {2025} +} + +@techreport{sr117, + title = {Supervisory Guidance on Model Risk Management}, + author = {{Board of Governors of the Federal Reserve System} and {Office of the Comptroller of the Currency}}, + year = {2011}, + number = {SR 11-7 / OCC 2011-12}, + institution = {Federal Reserve}, + url = {https://www.federalreserve.gov/supervisionreg/srletters/sr1107.htm}, +} + +% --- A. Conformal Prediction -- Foundations --- + +@article{sun2024ptc, + author = {Sun, Chunlin and Liu, Linyu and Li, Xiaocheng}, + title = {Predict-then-Calibrate: A New Perspective of Robust Contextual {LP}}, + journal = {arXiv preprint arXiv:2305.15686}, + year = {2024}, + url = {https://arxiv.org/abs/2305.15686} +} + +@article{taquet2025mapie, + author = {Taquet, Vianney and Blot, Vincent and Morzadec, Thomas and Lacombe, Louis and Brunel, Nicolas}, + title = {{MAPIE}: an open-source library for distribution-free uncertainty quantification}, + journal = {arXiv preprint arXiv:2207.12274}, + year = {2022}, + doi = {10.48550/arXiv.2207.12274}, + url = {https://arxiv.org/abs/2207.12274} +} + +% --- Additional references used in book chapters --- + +@inproceedings{vaicenavicius2019, + author = {Vaicenavicius, Juozas and Widmann, David and Andersson, Carl and Lindsten, Fredrik and Roll, Jacob and Sch{\"o}n, Thomas B.}, + title = {Evaluating Model Calibration in Classification}, + booktitle = {Proceedings of the 22nd International Conference on Artificial Intelligence and Statistics (AISTATS)}, + year = {2019}, + pages = {3459--3467}, + publisher = {PMLR} +} + +@book{vovk2005, + author = {Vovk, Vladimir and Gammerman, Alexander and Shafer, Glenn}, + title = {Algorithmic Learning in a Random World}, + year = {2005}, + publisher = {Springer}, + doi = {10.1007/978-3-031-06649-8} +} + +@inproceedings{vovk2014, + author = {Vovk, Vladimir and Petej, Ivan}, + title = {Venn-{Abers} Predictors}, + booktitle = {UAI}, + year = {2014}, + url = {https://arxiv.org/abs/1211.0025} +} + +@article{winkler1972, + author = {Winkler, Robert L.}, + title = {A Decision-Theoretic Approach to Interval Estimation}, + journal = {Journal of the American Statistical Association}, + volume = {67}, + number = {337}, + pages = {187--191}, + year = {1972}, + doi = {10.1080/01621459.1972.10481224} +} + +@article{xia2017, + author = {Xia, Yufei and Liu, Chuanzhe and Li, YuYing and Liu, Nana}, + title = {A boosted decision tree approach using {Bayesian} hyper-parameter optimization for credit scoring}, + journal = {Expert Systems with Applications}, + volume = {78}, + pages = {225--241}, + year = {2017}, + doi = {10.1016/j.eswa.2017.02.017} +} + +@article{yang2026multidistribution, + author = {Yang, Yuqi and Jin, Ying}, + title = {Multi-Distribution Robust Conformal Prediction}, + journal = {arXiv preprint arXiv:2601.02998}, + year = {2026}, + doi = {10.48550/arXiv.2601.02998}, + url = {https://arxiv.org/abs/2601.02998} +} + +@inproceedings{yeh2025training, + author = {Yeh, Christopher and Christianson, Nicolas and Wierman, Adam and Yue, Yisong}, + title = {Conformal Risk Training: End-to-End Optimization of Conformal Risk Control}, + booktitle = {NeurIPS 2025}, + year = {2025}, + url = {https://arxiv.org/abs/2510.08748} +} + +@article{yeh2026, + author = {Yeh, Christopher and Christianson, Nicolas and Wu, Alan and Wierman, Adam and Yue, Yisong}, + title = {End-to-End Conformal Calibration for Optimization Under Uncertainty}, + journal = {Transactions on Machine Learning Research}, + year = {2025}, + note = {Published 12/2025; arXiv v2 posted 2026}, + url = {https://arxiv.org/abs/2409.20534} +} + +@inproceedings{zadrozny2002, + author = {Zadrozny, Bianca and Elkan, Charles}, + title = {Transforming classifier scores into accurate multiclass probability estimates}, + booktitle = {KDD}, + year = {2002}, + doi = {10.1145/775047.775151} +} + +@article{zhao2025robust, + author = {Zhao, Lingjie and Jiang, Hansheng and Qi, Wei}, + title = {Conformal Robust Optimization and Satisficing for Prescriptive Analytics with Black-Box Predictors}, + journal = {SSRN Electronic Journal}, + year = {2026}, + note = {AISTATS 2026 Workshop on Towards Trustworthy Predictions: Uncertainty, Calibration and Conformal Methods}, + url = {https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5338354} +} + +@article{zhou2025credo, + author = {Zhou, Wenbin and Orfanoudaki, Agni and Zhu, Shixiang}, + title = {Conformalized Decision Risk Assessment}, + journal = {arXiv preprint arXiv:2505.13243}, + year = {2025}, + url = {https://arxiv.org/abs/2505.13243} +} + +@article{zhou2026creme, + author = {Zhou, Wenbin and Zhu, Shixiang}, + title = {Calibrating Decision Robustness via Inverse Conformal Risk Control}, + journal = {arXiv preprint arXiv:2510.07750}, + year = {2026}, + url = {https://arxiv.org/abs/2510.07750} +} + +@inproceedings{cresswell2024, + author = {Cresswell, Jesse C. and Sui, Yi and Kumar, Bhargava and Vouitsis, No{\"e}l}, + title = {Conformal Prediction Sets Improve Human Decision Making}, + booktitle = {Proceedings of the 41st International Conference on Machine Learning (ICML)}, + year = {2024}, + publisher = {PMLR}, + url = {https://arxiv.org/abs/2401.13744} +} + +@inproceedings{zhou2024, + author = {Zhou, Yanfei and Sesia, Matteo}, + title = {Conformal Classification with Equalized Coverage for Adaptively Selected Groups}, + booktitle = {NeurIPS 2024}, + year = {2024}, + url = {https://arxiv.org/abs/2405.15106} +} + +% --- M. Tail-risk measures (CVaR / OCE) -- referenced by Table A12 --- + +@article{rockafellar2000cvar, + author = {Rockafellar, R. Tyrrell and Uryasev, Stanislav}, + title = {Optimization of Conditional Value-at-Risk}, + journal = {The Journal of Risk}, + volume = {2}, + number = {3}, + pages = {21--41}, + year = {2000}, + doi = {10.21314/JOR.2000.038} +} + +@article{bental2007oce, + author = {Ben-Tal, Aharon and Teboulle, Marc}, + title = {An Old-New Concept of Convex Risk Measures: The Optimized Certainty Equivalent}, + journal = {Mathematical Finance}, + volume = {17}, + number = {3}, + pages = {449--476}, + year = {2007}, + doi = {10.1111/j.1467-9965.2007.00311.x} +} + +% --- N. Conformal under distribution/covariate shift & foundations --- + +@inproceedings{tibshirani2019covshift, + author = {Tibshirani, Ryan J. and Foygel Barber, Rina and Cand\`{e}s, Emmanuel J. and Ramdas, Aaditya}, + title = {Conformal Prediction Under Covariate Shift}, + booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, + volume = {32}, + year = {2019}, + url = {https://arxiv.org/abs/1904.06019} +} + +@book{angelopoulos2024foundations, + author = {Angelopoulos, Anastasios N. and Barber, Rina Foygel and Bates, Stephen}, + title = {Theoretical Foundations of Conformal Prediction}, + year = {2024}, + publisher = {arXiv (monograph)}, + url = {https://arxiv.org/abs/2411.11824} +} + +@article{guan2023localized, + author = {Guan, Leying}, + title = {Localized Conformal Prediction: A Generalized Inference Framework for Conformal Prediction}, + journal = {Biometrika}, + volume = {110}, + number = {1}, + pages = {33--50}, + year = {2023}, + doi = {10.1093/biomet/asac040} +} + +% --- O. Fair lending: proxy methods and policy reports --- + +@techreport{cfpb2014bisg, + author = {{Consumer Financial Protection Bureau}}, + title = {Using Publicly Available Information to Proxy for Unidentified Race and Ethnicity: A Methodology and Assessment}, + institution = {CFPB}, + year = {2014}, + url = {https://www.consumerfinance.gov/data-research/research-reports/using-publicly-available-information-to-proxy-for-unidentified-race-and-ethnicity/} +} + +@techreport{finreglab2023fairness, + author = {{FinRegLab}}, + title = {Explainability and Fairness in Machine Learning for Credit Underwriting: Policy Analysis}, + institution = {FinRegLab}, + year = {2023}, + url = {https://finreglab.org/research/explainability-and-fairness-in-machine-learning-for-credit-underwriting/} +} + +% --- P. Thesis-library sources integrated from Papers_tesis --- + +@techreport{albanesi2024credit, + author = {Albanesi, Stefania and Vamossy, Domonkos F.}, + title = {Credit Scores: Performance and Equity}, + institution = {National Bureau of Economic Research}, + type = {NBER Working Paper}, + number = {32917}, + year = {2024}, + url = {https://www.nber.org/papers/w32917} +} + +@article{blattner2021noise, + author = {Blattner, Laura and Nelson, Scott}, + title = {How Costly is Noise? Data and Disparities in Consumer Credit}, + journal = {arXiv preprint arXiv:2105.07554}, + year = {2021}, + url = {https://arxiv.org/abs/2105.07554} +} + +@article{brevoort2016creditinvisibles, + author = {Brevoort, Kenneth P. and Grimm, Philipp and Kambara, Michelle}, + title = {Credit Invisibles and the Unscored}, + journal = {Cityscape}, + volume = {18}, + number = {2}, + pages = {9--34}, + year = {2016}, + url = {https://ssrn.com/abstract=2743007} +} + +@article{fuster2022predictably, + author = {Fuster, Andreas and Goldsmith-Pinkham, Paul and Ramadorai, Tarun and Walther, Ansgar}, + title = {Predictably Unequal? The Effects of Machine Learning on Credit Markets}, + journal = {Journal of Finance}, + year = {2022}, + url = {https://ssrn.com/abstract=3072038} +} + +@article{khandani2010consumer, + author = {Khandani, Amir E. and Kim, Adlar J. and Lo, Andrew W.}, + title = {Consumer Credit-Risk Models via Machine-Learning Algorithms}, + journal = {Journal of Banking \& Finance}, + volume = {34}, + number = {11}, + pages = {2767--2787}, + year = {2010}, + doi = {10.1016/j.jbankfin.2010.06.001} +} + +@techreport{basel2015ecl, + author = {{Basel Committee on Banking Supervision}}, + title = {Guidance on Credit Risk and Accounting for Expected Credit Losses}, + institution = {Bank for International Settlements}, + year = {2015}, + url = {https://www.bis.org/bcbs/publ/d350.htm} +} + +@techreport{jagtiani2019altdata, + author = {Jagtiani, Julapa and Lemieux, Catharine}, + title = {The Roles of Alternative Data and Machine Learning in Fintech Lending: Evidence from the {LendingClub} Consumer Platform}, + institution = {Federal Reserve Bank of Philadelphia}, + type = {Working Paper}, + number = {18-15}, + year = {2019}, + doi = {10.21799/frbp.wp.2018.15} +} + +@article{bhattacharyya2026groupweighted, + author = {Bhattacharyya, Aabesh and Barber, Rina Foygel}, + title = {Group-Weighted Conformal Prediction}, + journal = {Electronic Journal of Statistics}, + volume = {20}, + number = {1}, + pages = {1171--1199}, + year = {2026}, + doi = {10.1214/26-EJS2506}, + url = {https://arxiv.org/abs/2401.17452} +} + +@inproceedings{schutte2024robust, + author = {Schutte, Noah and Postek, Krzysztof and Yorke-Smith, Neil}, + title = {Robust Losses for Decision-Focused Learning}, + booktitle = {Proceedings of the 33rd International Joint Conference on Artificial Intelligence (IJCAI)}, + year = {2024}, + doi = {10.24963/ijcai.2024/538}, + url = {https://arxiv.org/abs/2310.04328} +} + +@article{einbinder2024labelnoise, + author = {Einbinder, Bat-Sheva and Feldman, Shai and Bates, Stephen and Angelopoulos, Anastasios N. and Gendler, Asaf and Romano, Yaniv}, + title = {Label Noise Robustness of Conformal Prediction}, + journal = {Journal of Machine Learning Research}, + volume = {25}, + year = {2024}, + url = {https://jmlr.org/papers/v25/23-1549.html} +} + +@article{hu2026crc, + author = {Hu, Yang and Tan, Jieren and Zou, Changliang and Bao, Yajie and Ren, Haojie}, + title = {Conformal Robustness Control: A New Strategy for Robust Decision}, + journal = {ICLR 2026}, + year = {2026}, + url = {https://openreview.net/forum?id=bt4Ahpemmi} +} + +@techreport{kawasumi2026ordinal, + author = {Kawasumi, Ayumi and Kato, Masahiro and Duan, Lei}, + title = {Conformal Prediction for Ordinal Credit Scoring}, + institution = {Japanese Society for Artificial Intelligence, Special Interest Group on Financial Informatics}, + type = {SIG-FIN Technical Report}, + number = {036-20}, + year = {2026}, + doi = {10.11517/jsaisigtwo.2026.FIN-036_128} +} + +@article{angelopoulos2025gradient, + author = {Angelopoulos, Anastasios N. and Jordan, Michael I. and Tibshirani, Ryan J.}, + title = {Gradient Equilibrium in Online Learning: Theory and Applications}, + journal = {Journal of Machine Learning Research}, + volume = {26}, + year = {2025}, + url = {https://arxiv.org/abs/2501.08330} +} + +@misc{angelopoulos2026nonmonotonic, + author = {Angelopoulos, Anastasios N.}, + title = {Conformal Risk Control for Non-Monotonic Losses}, + year = {2026}, + note = {arXiv preprint} +} + +% --- Q. Parent-project intake: survival/LGD & credit OR (related-work + thesis CIF) --- + +@article{ptakchmielewska2024survival, + author = {Ptak-Chmielewska, Aneta and Kopciuszewski, Pawel}, + title = {Random Survival Forests and {Cox} Regression in Loss Given Default Estimation}, + journal = {The Journal of Credit Risk}, + year = {2024}, + doi = {10.21314/JCR.2024.3656} +} + +@article{chi2019p2p, + author = {Chi, Guotai and Ding, Shijie and Peng, Xiankun}, + title = {Data-Driven Robust Credit Portfolio Optimization for Investment Decisions in {P2P} Lending}, + journal = {Mathematical Problems in Engineering}, + volume = {2019}, + pages = {1902970}, + year = {2019}, + doi = {10.1155/2019/1902970} +} + +@article{babaei2020p2p, + author = {Babaei, Golnoosh and Bamdad, Shahrooz}, + title = {A Multi-Objective Instance-Based Decision Support System for Investment Recommendation in Peer-to-Peer Lending}, + journal = {Expert Systems with Applications}, + volume = {150}, + pages = {113278}, + year = {2020}, + doi = {10.1016/j.eswa.2020.113278} +} + +@article{aior2025lendingclub, + author = {Torkian, Vajiheh and Bamdad, Shahrooz and Sarfaraz, Amir Homayoun}, + title = {Integrating {AI} and {OR} for Investment Decision-Making in Emerging Digital Lending Businesses: A Risk-Return Multi-Objective Optimization Approach}, + journal = {Journal of the Operational Research Society}, + volume = {77}, + number = {3}, + pages = {716--735}, + year = {2026}, + note = {Published online 2025}, + doi = {10.1080/01605682.2025.2498652} +} + +@article{zheng2026twostage, + author = {Zheng, Chenlu and Zhu, Jianping and Weng, Futian and Zhang, Zhiyuan and Feng, Chong and Wang, Linhui}, + title = {A Two-Stage Machine Learning Method for Personal Credit Risk Scoring with Fragmentary Data}, + journal = {Expert Systems with Applications}, + volume = {299}, + pages = {130268}, + year = {2026}, + doi = {10.1016/j.eswa.2025.130268} +} + +@article{boosting2025default, + author = {Nguyen, Nhat and Ngo, Duy}, + title = {Comparative Analysis of Boosting Algorithms for Predicting Personal Default}, + journal = {Cogent Economics \& Finance}, + volume = {13}, + number = {1}, + pages = {2465971}, + year = {2025}, + doi = {10.1080/23322039.2025.2465971} +} + +@article{financialriskcp2025, + author = {{Preprints.org}}, + title = {A Financial Risk Assessment Model Combining Conformal Prediction and {Cox} Proportional Hazards}, + journal = {Preprints.org}, + year = {2025}, + url = {https://www.preprints.org/manuscript/202601.1407} +} + +@inproceedings{niculescu2005, + author = {Niculescu-Mizil, Alexandru and Caruana, Rich}, + title = {Predicting Good Probabilities with Supervised Learning}, + booktitle = {Proceedings of the 22nd International Conference on Machine Learning (ICML)}, + year = {2005}, + pages = {625--632}, + doi = {10.1145/1102351.1102430} +} + +% --- R. External multidataset credit sources --- + +@misc{prosperLoanData, + author = {{Prosper Funding LLC}}, + title = {Where Can I Download Data about Loans through Prosper?}, + year = {2026}, + howpublished = {Help Center article and loan-level data access page}, + url = {https://help.prosper.com/hc/en-us/articles/210013083-Where-can-I-download-data-about-loans-through-Prosper}, + note = {Accessed 5 June 2026} +} + +@misc{freddieMacSfLoanLevel, + author = {{Freddie Mac}}, + title = {Single Family Loan-Level Dataset}, + year = {2026}, + howpublished = {Public loan-level credit performance dataset documentation}, + url = {https://www.freddiemac.com/research/datasets/sf-loanlevel-dataset}, + note = {Accessed 5 June 2026} +} + +@misc{mushava2023classimbalance, + author = {Mushava, Jonah}, + title = {Credit Scoring with Class Imbalance Data: An Out-of-Sample and Out-of-Time Perspective}, + publisher = {Mendeley Data}, + year = {2023}, + version = {3}, + doi = {10.17632/bzr2rxttvz.3}, + url = {https://data.mendeley.com/datasets/bzr2rxttvz/3} +} + +@misc{homeCreditDefaultRisk, + author = {{Home Credit Group}}, + title = {Home Credit Default Risk}, + year = {2018}, + howpublished = {Kaggle competition dataset}, + url = {https://www.kaggle.com/competitions/home-credit-default-risk/data}, + note = {Archived source only; not promoted in the CRPTO main external claim} +} + +% --- S. IJDS venue fit and newly promoted decision references --- + +@article{das2023creditgraph, + author = {Das, Sanjiv and Huang, Xin and Adeshina, Soji and Yang, Patrick and Bachega, Leonardo}, + title = {Credit Risk Modeling with Graph Machine Learning}, + journal = {INFORMS Journal on Data Science}, + volume = {2}, + number = {2}, + pages = {197--217}, + year = {2023}, + doi = {10.1287/ijds.2022.00018} +} + +@article{yang2025costaware, + author = {Yang, Mochen and Bi, Xuan}, + title = {Cost-Aware Calibration of Classifiers}, + journal = {INFORMS Journal on Data Science}, + volume = {4}, + number = {2}, + pages = {101--113}, + year = {2025}, + doi = {10.1287/ijds.2024.0038} +} + +@article{fernandezloria2022causaldecision, + author = {Fernandez-Loria, Carlos and Provost, Foster}, + title = {Causal Decision Making and Causal Effect Estimation Are Not the Same...and Why It Matters}, + journal = {INFORMS Journal on Data Science}, + volume = {1}, + number = {1}, + pages = {4--16}, + year = {2022}, + doi = {10.1287/ijds.2021.0006} +} + +@article{fernandezloria2025observational, + author = {Fernandez-Loria, Carlos and Provost, Foster}, + title = {Observational vs. Experimental Data When Making Automated Decisions Using Machine Learning}, + journal = {INFORMS Journal on Data Science}, + volume = {4}, + number = {3}, + pages = {197--229}, + year = {2025}, + doi = {10.1287/ijds.2023.0012} +} + +@article{falconer2026replication, + author = {Falconer, Thomas and Kazempour, Jalal and Pinson, Pierre}, + title = {Toward Replication-Robust Analytics Markets}, + journal = {INFORMS Journal on Data Science}, + volume = {5}, + number = {2}, + pages = {155--170}, + year = {2026}, + doi = {10.1287/ijds.2025.0075} +} + +@article{goldfarb2003robustportfolio, + author = {Goldfarb, Donald and Iyengar, Garud}, + title = {Robust Portfolio Selection Problems}, + journal = {Mathematics of Operations Research}, + volume = {28}, + number = {1}, + pages = {1--38}, + year = {2003}, + doi = {10.1287/moor.28.1.1.14260} +} + +@article{delage2010dro, + author = {Delage, Erick and Ye, Yinyu}, + title = {Distributionally Robust Optimization Under Moment Uncertainty with Application to Data-Driven Problems}, + journal = {Operations Research}, + volume = {58}, + number = {3}, + pages = {595--612}, + year = {2010}, + doi = {10.1287/opre.1090.0741} +} + +@inproceedings{zhao2016p2pportfolio, + author = {Zhao, Hongke and Liu, Qi and Wang, Guifeng and Ge, Yong and Chen, Enhong}, + title = {Portfolio Selections in {P2P} Lending: A Multi-Objective Perspective}, + booktitle = {Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}, + pages = {2075--2084}, + year = {2016}, + doi = {10.1145/2939672.2939861} +} + +@article{serrano2016profitscoring, + author = {Serrano-Cinca, Carlos and Guti{\'e}rrez-Nieto, Bego{\~n}a}, + title = {The Use of Profit Scoring as an Alternative to Credit Scoring Systems in Peer-to-Peer ({P2P}) Lending}, + journal = {Decision Support Systems}, + volume = {89}, + pages = {113--122}, + year = {2016}, + doi = {10.1016/j.dss.2016.06.014} +} + +@inproceedings{liu2021riskbounds, + author = {Liu, Heyuan and Grigas, Paul}, + title = {Risk Bounds and Calibration for a Smart Predict-then-Optimize Method}, + booktitle = {Advances in Neural Information Processing Systems}, + volume = {34}, + pages = {22083--22094}, + year = {2021} +} + +% --- T. Literature expansion scan 2026-07-08 --- + +@article{sadana2025contextual, + author = {Sadana, Utsav and Chenreddy, Abhilash and Delage, Erick and Forel, Alexandre and Frejinger, Emma and Vidal, Thibaut}, + title = {A Survey of Contextual Optimization Methods for Decision-Making Under Uncertainty}, + journal = {European Journal of Operational Research}, + volume = {320}, + number = {2}, + pages = {271--289}, + year = {2025}, + doi = {10.1016/j.ejor.2024.03.020} +} + +@article{xu2025profit_uncertainty_credit, + author = {Xu, Yong and Kou, Gang and Ergu, Daji}, + title = {Profit-Based Uncertainty Estimation with Application to Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {325}, + number = {2}, + pages = {303--316}, + year = {2025}, + doi = {10.1016/j.ejor.2025.03.007} +} + +@article{xu2024profit_risk_credit, + author = {Xu, Yong and Kou, Gang and Peng, Yi and Ding, Kexing and Ergu, Daji and Alotaibi, Fahd S.}, + title = {Profit- and Risk-Driven Credit Scoring Under Parameter Uncertainty: A Multiobjective Approach}, + journal = {Omega}, + volume = {125}, + pages = {103004}, + year = {2024}, + doi = {10.1016/j.omega.2023.103004} +} + +@inproceedings{farinhas2024nonexchangeable_crc, + author = {Farinhas, Antonio and Zerva, Chrysoula and Ulmer, Dennis and Martins, Andre F. T.}, + title = {Non-Exchangeable Conformal Risk Control}, + booktitle = {International Conference on Learning Representations}, + year = {2024}, + url = {https://arxiv.org/abs/2310.01262} +} + +@article{hegazy2025valid_selection_conformal_sets, + author = {Hegazy, Mahmoud and Aolaritei, Liviu and Jordan, Michael I. and Dieuleveut, Aymeric}, + title = {Valid Selection Among Conformal Sets}, + journal = {arXiv preprint arXiv:2506.20173}, + year = {2025}, + url = {https://arxiv.org/abs/2506.20173} +} + +@article{hullman2025conformal_human_decision, + author = {Hullman, Jessica and Wu, Yifan and Xie, Dawei and Guo, Ziyang and Gelman, Andrew}, + title = {Conformal Prediction and Human Decision Making}, + journal = {arXiv preprint arXiv:2503.11709}, + year = {2025}, + url = {https://arxiv.org/abs/2503.11709} +} + +@article{djeundje2025dynamic_loan_portfolio_profitability, + author = {Djeundje, Viani B. and Crook, Jonathan and Andreeva, Galina}, + title = {The Devil in the Details: Dynamic Prediction of Loan Portfolio Profitability with Macroeconomic Drivers Through Multi-State Modelling}, + journal = {European Journal of Operational Research}, + volume = {327}, + number = {2}, + pages = {703--715}, + year = {2025}, + doi = {10.1016/j.ejor.2025.07.008} +} + +@article{stratigakos2026decision_calibrated_sets, + author = {Stratigakos, Akylas and Wen, Honglin and Spyrou, Elina and Pinson, Pierre}, + title = {Decision-Calibrated Prediction Sets for Robust Power System Operations}, + journal = {arXiv preprint arXiv:2606.02081}, + year = {2026}, + url = {https://arxiv.org/abs/2606.02081} +} + +@article{chen2026polyhedral_conformal_ro, + author = {Chen, Shuyi and Zhou, Wenbin and Zhu, Shixiang}, + title = {Learning Polyhedral Conformal Sets for Robust Optimization}, + journal = {arXiv preprint arXiv:2605.08506}, + year = {2026}, + url = {https://arxiv.org/abs/2605.08506} +} + +@article{wang2026optimal_decision_prediction_sets, + author = {Wang, Tao and Dobriban, Edgar}, + title = {Optimal Decision-Making Based on Prediction Sets}, + journal = {arXiv preprint arXiv:2602.00989}, + year = {2026}, + url = {https://arxiv.org/abs/2602.00989} +} + +@article{zheng2026counterfactual_decision_sets, + author = {Zheng, Yurui and Jin, Ying}, + title = {Prediction Sets for Counterfactual Decisions: Coverage, Optimality, and Conformal Prediction}, + journal = {arXiv preprint arXiv:2607.02206}, + year = {2026}, + url = {https://arxiv.org/abs/2607.02206} +} + +@article{huang2026oce_rcps, + author = {Huang, Jiayi and Farzaneh, Amirmohammad and Simeone, Osvaldo}, + title = {Optimized Certainty Equivalent Risk-Controlling Prediction Sets}, + journal = {arXiv preprint arXiv:2602.13660}, + year = {2026}, + url = {https://arxiv.org/abs/2602.13660} +} + +@article{baesens2026foundation_credit_risk, + author = {Baesens, Bart and Goethals, Andreas and Lessmann, Stefan and De Vos, Simon and Bravo, Cristian and Martens, David and Medina-Olivares, Victor and Mues, Christophe and Oskarsdottir, Maria and vanden Broucke, Seppe and Verdonck, Tim and Verbeke, Wouter}, + title = {Foundation Models for Credit Risk Prediction: A Game Changer?}, + journal = {arXiv preprint arXiv:2605.18147}, + year = {2026}, + url = {https://arxiv.org/abs/2605.18147} +} + +@article{distaso2025business_cycle_losses, + author = {Distaso, Walter and Roccazzella, Francesco and Vrins, Frederic}, + title = {Business Cycle and Realized Losses in the Consumer Credit Industry}, + journal = {European Journal of Operational Research}, + volume = {323}, + number = {3}, + pages = {1024--1039}, + year = {2025}, + doi = {10.1016/j.ejor.2024.12.026} +} + +@article{ballegeer2025explanation_stability, + author = {Ballegeer, Matteo and Bogaert, Matthias and Benoit, Dries F.}, + title = {Evaluating the Stability of Model Explanations in Instance-Dependent Cost-Sensitive Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {326}, + number = {3}, + pages = {630--640}, + year = {2025}, + doi = {10.1016/j.ejor.2025.05.039} +} + +@article{wiberg2025ai_or, + author = {Wiberg, Holly and Dai, Tinglong and Lam, Henry and Kulkarni, Radhika}, + title = {Synergizing Artificial Intelligence and Operations Research: Perspectives from {INFORMS} Fellows on the Next Frontier}, + journal = {INFORMS Journal on Data Science}, + volume = {5}, + number = {1}, + pages = {14--23}, + year = {2025}, + doi = {10.1287/ijds.2025.0077} +} + +@article{morucci2022robust_matching_uncertainty, + author = {Morucci, Marco and Noor-E-Alam, Md. and Rudin, Cynthia}, + title = {A Robust Approach to Quantifying Uncertainty in Matching Problems of Causal Inference}, + journal = {INFORMS Journal on Data Science}, + volume = {1}, + number = {2}, + pages = {156--171}, + year = {2022}, + doi = {10.1287/ijds.2022.0020} +} + +@article{navaspalencia2020, + author = {Navas-Palencia, Guillermo}, + title = {Optimal Binning: Mathematical Programming Formulation}, + journal = {Expert Systems with Applications}, + volume = {158}, + pages = {113508}, + year = {2020}, + doi = {10.1016/j.eswa.2020.113508}, + url = {https://arxiv.org/abs/2001.08025} +} + +@article{chen2025cost_sensitive_adversarial, + author = {Chen, Qiyuan and Al Kontar, Raed and Nouiehed, Maher and Yang, X. Jessie and Lester, Corey}, + title = {Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial Data Augmentation}, + journal = {INFORMS Journal on Data Science}, + volume = {4}, + number = {1}, + pages = {1--19}, + year = {2025}, + doi = {10.1287/ijds.2022.0033} +} + +@article{ausset2022censoring, + author = {Ausset, Guillaume and Cl{\'e}men{\c{c}}on, Stephan and Portier, Fran{\c{c}}ois}, + title = {Empirical Risk Minimization under Random Censorship}, + journal = {Journal of Machine Learning Research}, + volume = {23}, + number = {5}, + pages = {1--59}, + year = {2022}, + url = {https://jmlr.org/papers/v23/19-450.html} +} + +@article{li2023online_loans, + author = {Li, Zhiyong and Li, Aimin and Bellotti, Anthony and Yao, Xiao}, + title = {The Profitability of Online Loans: A Competing Risks Analysis on Default and Prepayment}, + journal = {European Journal of Operational Research}, + volume = {306}, + number = {2}, + pages = {968--985}, + year = {2023}, + doi = {10.1016/j.ejor.2022.08.013} +} + +@article{lyocsa2022profit, + author = {Ly{\'o}csa, {\v{S}}tefan and Va{\v{s}}ani{\v{c}}ov{\'a}, Petra and Hadji Misheva, Branka and Vateha, Marko D{\'a}vid}, + title = {Default or Profit Scoring Credit Systems? Evidence from European and {US} Peer-to-Peer Lending Markets}, + journal = {Financial Innovation}, + volume = {8}, + number = {1}, + pages = {32}, + year = {2022}, + doi = {10.1186/s40854-022-00338-5} +} diff --git a/paper/submission/.gitignore b/paper/submission/.gitignore index 22e1914..f3615a9 100644 --- a/paper/submission/.gitignore +++ b/paper/submission/.gitignore @@ -1,6 +1,7 @@ # Do NOT vendor the official INFORMS template or build artifacts. -# Authors download informs4.cls + informs2014.bst from the INFORMS author portal -# and drop them here locally; they must not be committed. +# Authors download the style kit from the INFORMS author portal and drop it +# here locally. The reviewed revision is pinned by informs_style_assets.json; +# publisher files and the private title page must not be committed. informs4.cls informs*.bst *.bst @@ -12,6 +13,8 @@ INFORMS-IJDS-Template.* Style-Instructions.pdf sample.bib Sample-Figure.* +IJDS-datadisclosureform-*.pdf +IJDS_Reproducibility*.docx # LaTeX build artifacts *.aux *.bbl @@ -23,3 +26,5 @@ Sample-Figure.* *.fdb_latexmk # Compiled submission PDF (regenerate locally) CRPTO_ijds_submission.pdf +TITLE_PAGE_PRIVATE.md +COVER_LETTER_PRIVATE.md diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index fc792fd..909f462 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -1,49 +1,59 @@ # IJDS Claim Audit Matrix -This matrix is the editorial guardrail for the active calibration-selected -midpoint policy. Numeric authority is -`ijds_policy_governance.json` plus A35--A40. +Use this editor-facing matrix with the active evidence manifest. It is not a +reviewer manuscript and does not authorize claims beyond the registry. -| Claim | Evidence | Main reviewer risk | Defensible wording | -|---|---|---|---| -| The 90% conformal recipe is replayed exactly. | A35; stored endpoint replay max error `6.67e-16`. | "Exact" is mistaken for universal conditional validity. | Exact refers to numerical reconstruction of the frozen finite-sample recipe. Coverage remains marginal/Mondrian under its assumptions. | -| The 99% setting is not decision-useful here. | A35: width `0.988215`; `93.5424%` of upper endpoints equal one. | Reviewer thinks 90% was chosen only to improve economics. | The 90% level is the frozen recipe's reference level and preserves materially more ranking resolution; alpha sensitivity is fully reported. | -| The final policy is simple. | `q=(p+u)/2`, `tau=0.17`; one linear policy family. | Complexity or hidden nonlinear logic. | Point PD prices expected loss; the midpoint is used only in the risk constraint. | -| Final ranking does not use OOT outcomes or statistical assumptions. | A36; outcomes stored separately from a 12-column frame, nine candidates, five eligible, zero outcome/Markov fields. | Historical OOT inspection makes "untouched holdout" false. | Call it a retrospective lockbox replay with an outcome-free final ranking code path, not preregistration or a pristine prospective trial. | -| The selector has a declared stable rule. | A36: full budget, effective-PD feasibility, deterministic `B_u<=0.28`, then maximum expected objective; winner stable on `[0.259036,0.290491)`. | The endpoint cap appears chosen to force the midpoint. | Report the complete 3x3 grid, cap-change boundaries, and the independent December replay. | -| Selector stability is not selected-set validity. | December reselects `linear-005` before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | The 90% conformal label is read as 90% funded-set coverage. | Lead with the observed miss: stable outcome-free selection does not transfer marginal conformal validity to optimizer-selected weights. | -| The selected funded set has an exact accounting audit. | Full OOT: return `$179,327.59`, default `0.039375`, miscoverage `0.036875`, endpoint `0.258051`, `B_u+V=0.294926`. | Accounting is confused with a coverage theorem. | The identity is deterministic after outcomes; it is not nominal selected-set coverage. | -| The Markov statement is secondary and conditional. | Threshold `0.574279`; tail-probability bound `0.316228` under weighted validity. | Bound is too loose or presented as a hard risk cap. | Use it as sensitivity only. Operational controls are `tau`, midpoint exposure, and observed funded-set metrics. | -| A40 is a matched point-PD comparison. | Same candidates, budget, concentration, LGD, solver, and `tau=0.17`. | Comparator changes multiple semantics or sees labels. | Only the risk score changes; neither optimization reads OOT outcomes. | -| Robustness has a measured price. | A40: `8.678%` realized-return cost and `7.9025` percentage-point default reduction. | Default and miscoverage reductions are conflated. | Report default reduction (`7.9025` pp) separately from miscoverage reduction (`0.5025` pp). | -| The midpoint is not the safest CRPTO policy. | A40: 75% blend return `$172,939.50`, default `0.035875`, threshold `0.516624`. | Selected point is sold as dominant. | It is the highest calibration expected-objective candidate under the declared screen. | -| Performance is temporally heterogeneous. | A37: CRPTO wins strongly in 2018H2; point PD dominates 2019H2 and 2020+. | Full-panel result is overgeneralized. | State the reversals in body and abstract; no universal dominance claim. | -| Funded-set composition is correctly labeled. | A38 uses letter grade recovered from `sub_grade` and stores conformal group separately. | Score-quantile groups are mistaken for loan grades. | Call A38 a business composition audit, not fairness certification. | -| Bootstrap uncertainty is bounded. | A39 primary month-cluster return interval `$163,421.14`--`$193,551.65`; funded-loan sensitivity `$162,706.17`--`$193,924.74`. | Interval is read as full model/selection uncertainty. | Both are fixed-allocation diagnostics; model, conformal recipe, selector, and optimizer remain fixed. | -| Earlier methods do not multiply the contribution. | Supplement A1--A34. | Paper reads as several papers or an uncontrolled tournament. | OCE/CVaR, SPO+, online-style checks, and external replications are diagnostics or context, not active selectors. | -| Reproducibility is substantive evidence quality. | Run tags, configs, A35--A40 builder, claim-sync tests, manifest validation. | Tooling is presented as the only novelty. | Lead with decision method and empirical implication; reproducibility makes them auditable. | - -## Do Not Claim + + + + + + + + + + + + + + + + + -- pristine prospective or preregistered OOT evaluation; -- nominal conformal validity for optimizer-selected funded weights; -- universal dominance over point PD or decision-focused learning; -- causal return or default effects; -- legal fair-lending certification; -- live post-2020 Lending Club performance; -- that external Prosper/Freddie diagnostics replicate the final midpoint - selector exactly. +| Claim object | Active evidence | Permitted wording | Forbidden inference | +|---|---|---|---| +| Population | 2,925,493 raw rows; 640,543 eligible design rows | Exhaustive population under declared horizon, dates, schema, and observability rules | All raw rows share one estimand | +| Endpoint | 364,814 resolved; 12,076 unresolved among 376,890 primary candidates | Terminal status reconstructed as observable by the cutoff | Verified point-in-time archive snapshot | +| Endpoint reasons | 307,842 fully paid and 56,972 charged off are resolved; 11,551 are nonterminal, 47 terminate after the cutoff, and 478 lack a reconstructible availability date | The five reasons partition every primary candidate | Identified missingness mechanism or observed operational event dates | +| Coverage | Under the active six-month endpoint, all 40 model-window upper bounds are below 0.90; largest 0.897726 | Coverage does not transport here across five protocol-locked specifications | Universal conformal invalidity, endpoint-lag invariance, or selected-set validity | +| Binary geometry | W7 prevalence 0.101703 and quantile 0.888435; W8 prevalence 0.097147 and quantile 0.111801 | Pattern matches the constant-score phase mechanism | Finite-sample causal explanation | +| Fit-label timing | Crossing persists at 0, 3, and 6 months; 8 and 12 months fail strict >99% retention | Stable across fit-label lags satisfying the rule | Invariance to arbitrary label delay | +| Evaluation-endpoint timing | Coverage upper bounds below 0.90: 40/40 at lags 0, 3, 6, and 8; 39/40 at lag 12, maximum 0.900411 | The active six-month claim is contract-specific; the complete lag grid is reported without endpoint selection | Lag-universal 40/40 coverage failure or selection of the six-month endpoint | +| Endpoint decision sensitivity | Payoff lower/cross is 32/16 at lags 0, 3, 6, and 8 and 31/17 at 12; default higher/cross is 33/15 then 32/16; miscoverage is 40/8 throughout; all 216 broad envelopes cross at every lag | No opposite one-sided direction emerges, and broad-support nonidentification survives the grid | Endpoint-robust adverse direction or a preferred lag | +| Timing design | Fit-label timing refits residual recipes; endpoint timing holds recipes and allocations fixed | Two separate one-factor sensitivities, not a factorial design | Joint lag robustness across 25 untested combinations | +| Objective ruler .25 | Payoff [-9,134.34, 5,603.66]; default/miscoverage [-0.0068, 0.1265] pp | All three metrics are unidentified in all eight windows | A favorable endpoint | +| Objective ruler .50/.75 | .50 adverse in 8/8; .75 payoff/default cross in 7/8 | Direction changes with coordinate | Coordinate-free ordering | +| Normalized ruler | .25/.50 adverse; .75 payoff crosses in 1/8 | Same relative score relaxation, different opportunity cost | Neutral comparison or preferred ruler | +| Portfolio structure | Complete 36-scenario budget--purpose-cap--LGD grid; every scenario has at least 17 adverse default and 21 adverse miscoverage cells; zero scenarios are uniformly favorable or adverse | Direction remains conditional after varying three active structural assumptions | Scenario winner, structural invariance, universal harm, or deployment guidance | +| Exact support | 216/216 broad envelopes cross zero; default 72/72 development envelopes cross | No universal direction over declared supports | Universal comparator quantification | +| Solver stability | 7,297 evaluated rows; 0 tie-sensitive reversed-order reruns | Deterministic stability at evaluated caps | Continuous-frontier uniqueness | +| Credit controls | Five coverage specifications; one portfolio learner | Same below-target finding across the protocol-locked specifications under the active endpoint | Model promotion or WOE/IV novelty | +| Missingness encoding | Active sentinels, explicit indicators, and native nullable features each have 8/8 upper bounds below 0.90; maxima 0.882597, 0.884332, and 0.880037 | Coverage finding recurs across three semantics-preserving encodings | Missingness mechanism, encoding winner, or portfolio robustness | +| Second origin | Both retrospective origins have 8/8 upper bounds below 0.90; maxima 0.882597 and 0.874768 | Two-origin retrospective recurrence | Independent replication, temporal invariance, or prospective validation | +| Fit-label completion | All four scenarios have 8/8 upper bounds below 0.90; the W7--W8 crossing remains in three of four | Coverage nontransport survives the declared scenario family, while the geometric path is not scenario-invariant | Sharp bounds over all $2^{215}$ assignments, preregistration, or a universal phase transition | +| Allocation granularity | USD 25 flooring changes any evaluated rate by at most 0.001284 percentage points over 1,440 portfolios | The continuous relaxation is numerically adequate for this archive and lot rule | Integer optimality, reoptimization, or robustness to another lot rule | +| Payoff proxy | Plug-in `(1-p)r-p*LGD`; status-indexed `(1-Y)r-Y*LGD` | Coherent standardized proxy under the declared binary endpoint | IRR, cash-flow return, NPV, or welfare | +| Identification width | For a fixed contrast, width is exactly $\sum_{i\in U}|d_i(1)-d_i(0)|$; reported payoff widths span \$14,738 to \$373,705.31 across the six tracks | Finite-archive sensitivity to unresolved exposure disagreement is algebraically transparent; rate metrics retain their declared capital normalizer | Sampling uncertainty, confidence interval, or an unresolved-count-only ordering across metrics | -## Required Headline Numbers +## Nonnegotiable Boundaries -- selected return: `$179,327.59`; -- selected default / miscoverage: `0.039375 / 0.036875`; -- `Gamma_CP / Gamma_residual`: `0.176102 / 0.088051`; -- endpoint / observed accounting / conditional threshold: - `0.258051 / 0.294926 / 0.574279`; -- point-PD return: `$196,369.14`; -- return cost / default reduction: `8.678% / 7.9025` pp; -- selector: `5/9` eligible, selected `tau=0.17`, `gamma=0.50`. -- deterministic selector cap / stability interval: `0.28 / [0.259036, 0.290491)`; -- December audit default / miscoverage: `0.145650 / 0.124925`; -- primary month-cluster return interval: `$163,421.14`--`$193,551.65`. +- No selected learner, window, taxonomy, endpoint lag, gamma, ruler, + coordinate, structural scenario, cap, comparator, or policy. +- No causal, prospective, confirmatory, deployment, Markov, or fair-lending + claim. +- Overlapping windows and repeated allocations are not independent replications. +- Sharp bounds are finite-archive partial identification, not confidence + intervals. +- The paper is one ML--conformal--optimization audit; none of the three + components is presented as a separate successor paper. diff --git a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md index cf80bbd..92014bc 100644 --- a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md +++ b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md @@ -1,86 +1,82 @@ -# IJDS Cover Letter and Disclosure Draft - -Editor-facing material only. Do not include it in the double-anonymous reviewer -packet unless ScholarOne requests the corresponding disclosure text. - -## Cover Letter - -Dear Editors, - -We submit "CRPTO: A Calibration-Selected Conformal Guardrail for Credit -Portfolios" for consideration at the *INFORMS Journal on Data Science*. The -paper treats credit allocation as data science for decisions, -not as a credit-scoring leaderboard. A frozen calibrated PD model is combined -with an exactly replayed 90% Mondrian conformal endpoint. The resulting -midpoint score, `q=(p+u)/2`, constrains a `$1M` portfolio while point PD remains -in the expected-return objective. - -The final policy is selected from nine round-number candidates on November -2017 using a deterministic endpoint cap. Outcomes are stored separately from -its 12-column ranking frame, which contains no assumption-conditional -statistics. An outcome-free December replay selects the same rule; opening -outcomes afterward reveals -miscoverage `0.124925`, so the paper explicitly does not infer selected-set -validity from policy stability. On 276,869 out-of-time Lending Club loans, the -fixed policy earns `$179,327.59`, with weighted default `0.039375`. -A matched point-PD allocation earns `$196,369.14` with weighted default -`0.118400`. The paper reports both the `8.678%` return cost and the `7.9025` -percentage-point default reduction, together with temporal periods in which -the point-PD decision performs better. We therefore position CRPTO as an -auditable retrospective return-risk guardrail, not as a universal winner or -prospective deployment guarantee. - -The submission contributes an explicit prediction-to-decision contract, an -exact conformal replay, a temporally separated selector/audit, matched economic -comparisons, and a file-backed reproducibility package. These features align -with IJDS's emphasis on data, innovative methodology, decision relevance, and -reproducible evidence. +# Cover Letter and Disclosure Draft + +Dear Editor, + +Please consider **"CRPTO: An Identification Audit of Binary Conformal Credit +Portfolio Optimization"** for publication in *INFORMS Journal on Data Science*. + +The paper studies a common but under-audited handoff: a predictive uncertainty +object becomes a coefficient in a constrained optimization problem. CRPTO keeps +credit-risk prediction, binary conformal intervals, and monthly portfolio +optimization in one system, then asks which conclusions survive explicit +timing, outcome-observability, and comparator contracts. + +The contribution is an identification audit rather than a policy contest. +First, the paper derives a prevalence-threshold discontinuity for constant-score +binary absolute-residual intervals and separates that mechanism from the +empirical varying-score path with a joint fit-label completion sensitivity. +Second, it defines objective-matched and +normalized-score rulers and an exact basis-endpoint support audit for comparing +non-affine risk scores. It also gives an exact identity linking each contrast's +identification width to the sum of unresolved loan-wise attainable +contribution ranges. Third, it provides a status-independent, maturity-restricted +evaluation with outcomes physically isolated until allocations are frozen and +with sharp common-outcome bounds for unresolved loans. + +The full archive audit covers 2,925,493 rows and an exhaustive 640,543-loan +eligible design. The primary OOT panel contains 376,890 candidates: 364,814 +outcomes are reconstructed as observable by the evaluation cutoff and 12,076 +remain unresolved. All 40 all-candidate coverage upper bounds are below 0.90 +across five retrospectively protocol-locked learner specifications; the largest +is 0.8977. The endpoint census partitions all candidates into 307,842 fully paid +and 56,972 charged off by the cutoff, 11,551 nonterminal, 47 terminal after the +cutoff, and 478 terminal with no reconstructible availability date. The +stratum-level W7--W8 quantile change persists under every +fit-label lag retaining more than 99% of each fitting month. A separate complete +evaluation-endpoint lag grid reports 0, 3, 6, 8, and 12 months without selection; +the six-month slice exactly reproduces the active endpoint evaluation. The +below-target coverage finding also recurs across three feature-semantics- +preserving missingness encodings and the only additional feasible retrospective +origin; neither exercise selects a representation or constitutes independent +validation. +All four declared scenarios for 215 labels unavailable at fitting retain +8/8 below-target coverage upper bounds. The all-default scenario removes the +W7--W8 geometric crossing, so the paper reports that mechanism as diagnostic +rather than universal. + +The decision result has a narrow interpretation. Objective-matched coordinate .25 +crosses zero for payoff, default, and miscoverage in all eight windows; .50 is +adverse; and .75 is mostly unidentified. Normalized-score comparisons answer a +different opportunity-cost question. An exact point-cap audit places zero in +all 216 broad-support envelopes. No model, window, gamma, ruler, coordinate, +comparator, or policy is selected. +The complete 36-scenario budget--concentration--LGD grid restores neither a +universally favorable nor a universally adverse ordering. A deterministic USD +25 floor-with-cash diagnostic changes evaluated rates by at most 0.0013 +percentage points; it supports the continuous relaxation without claiming an +optimized integer policy. + +The manuscript addresses a decision-making problem at the +predictive-to-prescriptive interface. It combines executable information +boundaries, exact optimization geometry, partial identification, and complete +reporting of the protocol-locked paths. It makes no causal, selected-set, +prospective, deployment, or fair-lending claim. The distributed archive is +explicitly not described as a verified point-in-time snapshot, and standardized +payoff is not presented as investor return. + +The submission is original, is not under review elsewhere, and uses a separate +online supplement. Code, environment lock, immutable evidence metadata, and +raw-data reconstruction instructions are prepared under the journal's data and +code policy. + +## Generative AI Disclosure + +OpenAI Codex was used for code review, test generation, editorial revision, and +reproducibility checks under author direction. The author independently +verified the analysis, citations, numerical evidence, and final wording and +accepts full responsibility for the submitted work. Generative AI is not an +author and did not determine the scientific conclusions. Sincerely, -[Author details supplied separately] - -## Data and Code Availability - -The body and supplement are double-anonymous. During review they refer to a -reproducible companion without exposing author-identifying URLs. The IJDS Data -and Code Disclosure Form will state that, when venue policy permits, the -companion includes: - -- source code, configurations, and manuscript sources; -- A35--A40 evidence tables and active governance metadata; -- DVC metadata and artifact pointers for large processed data and model files; -- source instructions for Lending Club, Prosper, and Freddie/Mendeley data - rather than unauthorized redistribution; -- exact-alpha and calibration-selector replay commands; -- manifest, claim-sync, and publication-integrity tests; -- commands for regenerating tables, previews, and the official-template PDF. - -No secrets, tokens, private storage credentials, local usernames, or machine -paths belong in the reviewer package. - -Suggested ScholarOne prose: - -> Code, configurations, manuscript sources, and evidence-generation scripts -> will be released under the journal's accepted-paper reproducibility process. -> Public-source raw data are disclosed through source instructions; large or -> license-constrained artifacts are provided through documented pointers and -> integrity hashes where redistribution terms permit. - -## Anonymity Handling - -- Upload the `informs4` PDF built with `dblanonrev` as the manuscript. -- Upload the anonymous supplement separately. -- Keep the title page, affiliation, acknowledgements, repository ownership, - and personal URLs outside reviewer-facing files. -- Use this file and ScholarOne fields for editor-only disclosure timing. - -## Editorial Boundary - -The final ranking code is outcome-free with respect to OOT policy selection, -but earlier project development inspected the static OOT corpus. The manuscript -therefore says "retrospective lockbox replay," not "preregistered" or -"untouched holdout." Marginal/Mondrian coverage is not promoted to nominal -validity under optimizer-selected funded weights. OCE/CVaR, SPO+, online-style -checks, and external datasets remain diagnostics or context rather than -additional active methods. +The Author diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index 1696887..973b86e 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -1,17 +1,10 @@ %% ===================================================================== -%% CRPTO -- INFORMS Journal on Data Science (IJDS) submission manuscript -%% ===================================================================== -%% Double-anonymous official-template handoff synchronized with -%% paper/CRPTO_ijds.qmd. Build with latexmk, or use the Windows fallback: -%% -%% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -%% bibtex CRPTO_ijds_submission -%% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -%% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -%% -%% Pass 1 writes the .aux file, BibTeX writes the .bbl, pass 2 resolves the -%% bibliography and cross-references, and pass 3 stabilizes labels and pages. +%% GENERATED FROM paper/CRPTO_ijds.qmd. DO NOT EDIT THIS FILE DIRECTLY. +%% Build: uv run python scripts/build_ijds_submission_tex.py +%% Compile: latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +%% Windows fallback: pdflatex, bibtex, pdflatex, pdflatex. %% ===================================================================== +%% Generated manuscript template. Edit paper/CRPTO_ijds.qmd, not the output TeX. \documentclass[ijds,dblanonrev]{informs4} \usepackage{eqndefns-left} @@ -26,11 +19,20 @@ \ECRepeatTheorems \MANUSCRIPTNO{} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} \usepackage{amsmath,amssymb,mathtools} -\usepackage{booktabs} +\usepackage{booktabs,longtable,array,calc} \usepackage{graphicx} \usepackage{placeins} +\usepackage{float} +\usepackage{tabularx} +\usepackage{xcolor} +\usepackage{hyperref} +\hypersetup{hidelinks} \graphicspath{{../../reports/crpto/figures/}} +\floatplacement{figure}{H} +\setlength{\emergencystretch}{2em} \usepackage{natbib} \bibpunct[, ]{(}{)}{,}{a}{}{,} \def\bibfont{\small} @@ -39,12 +41,26 @@ \def\newblock{\ } \def\BIBand{and} +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\providecommand{\passthrough}[1]{#1} +\newsavebox\pandocbox +\newcommand*\pandocbounded[1]{% + \sbox\pandocbox{#1}% + \ifdim\wd\pandocbox>\linewidth + \resizebox{\linewidth}{!}{\usebox\pandocbox}% + \else + \usebox\pandocbox + \fi} + + \begin{document} \RUNAUTHOR{Anonymous} -\RUNTITLE{A Calibration-Selected Conformal Guardrail for Credit Portfolios} +\RUNTITLE{Binary Conformal Geometry Audit} -\TITLE{CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios} +\TITLE{CRPTO: An Identification Audit of Binary Conformal Credit +Portfolio Optimization} \ARTICLEAUTHORS{% \AUTHOR{} @@ -52,540 +68,1347 @@ } \ABSTRACT{% -Credit models matter only through the decisions they change. We study how -finite-sample predictive uncertainty can constrain a loan portfolio after a -probability-of-default (PD) model has been frozen. Conformal Robust -Predict-Then-Optimize (CRPTO) recomputes a 90\% Mondrian conformal upper endpoint -exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places -$q_i$ in a portfolio-risk constraint while retaining point PD in the economic -objective. Nine round-number policies are ranked on November 2017 without -default, realized-return, miscoverage, or assumption-conditional columns; an -outcome-free December replay selects the same rule before its outcomes are -opened. That audit misses nominal funded-set coverage, making the statistical -boundary observable rather than rhetorical. The policy is then replayed on -276{,}869 out-of-time Lending Club loans. -It funds 308 loans and earns \$179{,}327.59 on a \$1M budget, with weighted -default 0.039375, weighted miscoverage 0.036875, and conformal endpoint budget -0.258051. A matched point-PD allocation earns \$196{,}369.14 but has weighted -default 0.118400 and endpoint budget 0.921317. Thus CRPTO pays 8.678\% of -realized return for a 7.9025 percentage-point default reduction; the advantage -reverses in some temporal slices, so we do not claim universal dominance. The -contribution is a small, auditable prediction-to-decision guardrail with an -exact replay, a deterministic selector, and explicit statistical boundaries.% +How should predictive uncertainty be audited after it becomes a +constraint coefficient in an optimizer? Conformal Risk-Aware +Predict-Then-Optimize (CRPTO) combines a Platt-scaled credit-default +score, 90\%-target binary absolute-residual intervals, and monthly +loan-allocation linear programs. Our retrospectively locked design uses +all 640,543 eligible 36-month loans in the declared temporal roles, +fixes score strata, reports eight complete residual windows, constructs +status-independent menus, and freezes allocations before reading +evaluation outcomes. Because the archive is not a verified point-in-time +snapshot, we reconstruct availability at September 30, 2020 and retain +12,076 of 376,890 primary candidates as unresolved. Under the declared +six-month rule, all 40 model-window sharp coverage upper bounds are +below 0.90 across five credit-risk specifications (maximum 0.8977). The +CatBoost pattern recurs under three missing-value encodings and one +later retrospective origin. Four declared fitting-label scenarios for +215 unavailable labels also leave all 32 scenario-window upper bounds +below 0.90. A derived binary threshold mechanism is consistent with an +observed W7--W8 change in interval geometry, but the all-default +scenario removes that crossing; the mechanism is diagnostic, not +universal. For decisions, two outcome-blind common-frontier rulers and +three coordinates compare the point and full-upper scores. Sharp +common-outcome bounds, an exact width identity, and basis enumeration +show that direction depends on the comparator: all 216 broad point-cap +envelopes include zero, and no ruler, coordinate, or policy is selected. +Deterministic USD 25 rounding changes evaluated rates by at most 0.0013 +percentage points. CRPTO contributes a reproducible audit of the +machine-learning--conformal--optimization interface: coverage transport, +interval geometry, unresolved outcomes, and comparator design must be +distinguished before making a portfolio claim.% } -\KEYWORDS{conformal prediction; predict-then-optimize; credit risk; portfolio -optimization; calibration; reproducible data science} +\KEYWORDS{conformal prediction; predict-then-optimize; comparator +design; credit risk; portfolio selection; temporal transport; partial +identification} \maketitle -\section{Introduction}\label{sec:intro} - -Credit allocation is a contextual optimization problem. A lender estimates a -probability of default (PD) and then chooses which loans to fund under capital, -concentration, and risk constraints \citep{sadana2025contextual}. Credit-scoring -research has strengthened the first step through discrimination benchmarks, -probability calibration, and cost-aware evaluation -\citep{lessmann2015,chen2024creditrisk,yang2025costaware}. Yet a calibrated -probability does not specify how much model uncertainty a portfolio should -bear. A point-PD optimizer can concentrate capital in loans that appear -attractive where the predictive model is least certain. - -Conformal prediction offers finite-sample coverage language under explicit -exchangeability conditions \citep{vovk2005,angelopoulos2023}, while robust -optimization makes uncertainty operational through feasibility sets -\citep{bertsimas2004,goldfarb2003robustportfolio}. Joining them is attractive, -but an applied decision paper still has to answer three questions. Which -conformal level is informative rather than nearly vacuous? How is a policy -selected without using the outcomes on which it is later reported? What -economic value is lost when conformal uncertainty actually changes the funded -set? - -CRPTO answers those questions with one deliberately simple policy. A frozen, -calibrated CatBoost model produces $p_i$. An exactly replayed 90\% Mondrian -recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, -\begin{equation}\label{eq:midpoint} -q_i=p_i+0.5(u_i-p_i)=\frac{p_i+u_i}{2}, -\end{equation} -inside the risk constraint. The economic objective remains point-PD expected -net return. This separation is substantive: $p_i$ prices expected loss, while -$q_i$ limits the uncertainty the funded portfolio may carry. It also removes -nonlinear caps, tail rules, and uncertainty penalties that made an earlier -research frontier difficult to explain. - -The empirical design uses a temporal Lending Club panel. The conformal recipe -is fit inside the calibration period, November 2017 ranks a declared -$3\times3$ grid, and December replays the outcome-free selector before auditing -the fixed decision. The ranking artifact contains no defaults, realized -returns, miscoverage, or assumption-conditional statistics. The fixed rule is -then evaluated on January 2018 through September 2020 originations. Earlier -project development inspected this static OOT corpus, so the final run is a -transparent retrospective lockbox replay, not a pristine prospective or -preregistered trial. - -The paper makes three contributions. First, it gives an auditable -prediction-to-decision construction in which the economic objective and the -conformal guardrail have separate roles. Second, it replaces approximate -cross-alpha scaling with an exact replay, a deterministic endpoint screen, and -a temporally separated selector audit. Third, it reports the price and limits -of that guardrail against matched point-PD and more-conservative comparators, -including a pre-OOT funded-set coverage miss and temporal slices where CRPTO -wins and slices where it does not. The novelty is the closed, inspectable -decision protocol for a frozen credit model, not a claim that conformal -prediction, robust optimization, or credit scoring is individually new. - -\begin{figure}[t] -\centering -\includegraphics[width=0.92\textwidth]{crpto_fig1_journal_pipeline.png} -\caption{CRPTO carries a frozen calibrated PD through an exact conformal -replay, a simple portfolio guardrail, and a funded-set audit.} -\label{fig:pipeline} -\end{figure} - -\section{Related Work}\label{sec:related} - -CRPTO sits at the intersection of conformal prediction, robust optimization, -and decision-focused learning. Split conformal methods provide marginal -coverage without a parametric posterior, while Mondrian variants condition on -declared partitions \citep{vovk2005,bostrom2021}. Conditional and weighted -extensions require additional structure, and exact conditional coverage is -generally unavailable without restrictive assumptions -\citep{barber2021limits,barber2023beyond,jonkers2024wcps}. We therefore -distinguish population or partition coverage from coverage after a portfolio -has adaptively reweighted the loans. - -Data-driven robust and contextual optimization translate predictive -uncertainty into decisions -\citep{bertsimas2018datadriven,bertsimas2020prescriptive,sadana2025contextual}. -Recent work uses conformal sets in robust optimization -\citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. CRPTO is an applied -complement: it retains a frozen credit PD model, exposes the funded rows and -uncertainty premium, and compares the resulting allocation with a matched -point-PD decision. - -Decision-calibrated prediction sets and inverse conformal risk control go -further by calibrating downstream violation or regret and, after robustness -selection, using a separate split to restore risk-estimation validity -\citep{zhou2026creme,stratigakos2026decision_calibrated_sets}. CRPTO does not -import that guarantee into a batch portfolio where the optimizer chooses the -funded weights. It separates a deterministic endpoint screen from an -independent post-selection audit and reports when funded-set miscoverage misses -its nominal target. - -Decision-focused learning and SPO+ train predictions against downstream regret -\citep{donti2017,elmachtoub2022,mandi2024}. That is a different institutional -choice. CRPTO asks what can be done after a calibrated model already exists and -must remain frozen for governance. A synthetic SPO+ diagnostic remains in the -online supplement; it is not mixed with the real-dollar portfolio results. - -Credit-allocation research already covers profit scoring, P2P investment -recommendation, robust loan portfolios, rejection, and multiobjective risk -\citep{guo2016p2p,zhao2016p2pportfolio,serrano2016profitscoring,chi2019p2p, -babaei2020p2p,xu2025profit_uncertainty_credit,xu2024profit_risk_credit}. -Conformal credit scoring also means that the safe claim is not first use of -conformal prediction in credit \citep{kawasumi2026ordinal}. The remaining gap -is a file-backed protocol that shows exactly how a conformal endpoint changes a -budgeted credit decision and what that change costs. - -\begin{table}[t] -\centering -\small -\caption{Closest-work boundary for CRPTO.} -\label{tab:closest} -\begin{tabular}{p{0.20\textwidth}p{0.34\textwidth}p{0.37\textwidth}} -\toprule -Family & Existing contribution & CRPTO boundary \\ -\midrule -Credit scoring & Calibrated and cost-aware PD & PD is an input contract; no AUC-leadership claim. \\ -Robust credit portfolios & Economic selection under uncertainty & Exact conformal endpoint becomes a simple guardrail. \\ -Conformal robust optimization & Coverage-backed uncertainty sets & Frozen credit stack, funded-set audit, matched economics. \\ -Decision-focused learning & Training-time regret reduction & Frozen predictor and post-hoc auditability. \\ -Decision-risk calibration and valid selection & Calibrated decision losses or validity after set selection & Deterministic selector plus diagnostic audit; selected-set validity is not claimed. \\ -\bottomrule -\end{tabular} -\end{table} - -\section{Data and Evaluation Design}\label{sec:data} - -The data are Lending Club retail loans originated from 2007 through 2020. The -feature contract contains only origination-time information. Model development -and evaluation use temporal rather than random splits. - -\begin{table}[t] -\centering -\small -\caption{Temporal Lending Club design.} -\label{tab:splits} -\begin{tabular}{lrrp{0.42\textwidth}} -\toprule -Split & Period & Loans & Role \\ -\midrule -Train & Jun 2007--Mar 2017 & 1{,}346{,}311 & Fit PD model and calibrator. \\ -Conformal fit & Mar--Oct 2017 & 142{,}550 & Estimate frozen conformal recipe. \\ -Policy selection & Nov 2017 & 14{,}943 & Rank nine outcome-free policies. \\ -Calibration audit & Dec 2017 & 20{,}695 & Replay selector, then open outcomes. \\ -OOT evaluation & Jan 2018--Sep 2020 & 276{,}869 & Freeze-then-evaluate portfolio decisions. \\ -\bottomrule -\end{tabular} -\end{table} - -The frozen recipe uses the most recent 75\% of the original calibration pool -(178{,}188 rows). Within it, 142{,}550 rows estimate conformal quantiles, -14{,}943 November rows select the policy, and 20{,}695 December rows form a -pre-OOT audit. The loader stores holdout outcomes separately from the 12-column -policy frame, which contains identifiers, origination context, PD endpoints, -amounts, and rates. Ranking cannot access outcomes, realized returns, -miscoverage, or the assumption-conditional Markov quantity. December first -reruns that outcome-free selector and only then joins outcomes for the decision -audit. - -The final OOT panel covers several regimes, including the 2020 disruption. We -report the full panel and five temporal slices. Each slice solves the same fixed -policy on a fresh \$1M budget; slice returns are comparable stress evaluations, -not components that sum to the full-panel value. - -\section{Method}\label{sec:method} - -\subsection{Calibrated PD and Exact Conformal Replay} - -Let $p_i\in[0,1]$ be calibrated PD. The frozen CatBoost layer has AUC 0.7139, -Brier score 0.1544, and expected calibration error about 0.0070. These values -are not a leaderboard claim; probability quality matters because the economic -objective consumes PD. - -The conformal recipe partitions calibrated scores into five score-quantile -Mondrian cells. Calibration loan $j$ receives scaled residual -\begin{equation}\label{eq:score} -s_j=\frac{|Y_j-p_j|}{\sqrt{\max\{p_j(1-p_j),10^{-6}\}}}. -\end{equation} -Each cell uses the finite-sample higher quantile at frozen used alpha 0.095 for -target $\alpha=0.10$. Recorded group and temporal factors may widen but never -narrow an interval. The resulting upper endpoint is clipped to $[0,1]$. - -The replay reconstructs every setting from the frozen result payload. At the -90\% reference level it reproduces stored point, lower, and upper vectors with -maximum absolute error below $6.67\times10^{-16}$. This matters because an -earlier exploratory sweep scaled 90\% radii with average widths from another -conformal family; those values are not used in this manuscript. - -\subsection{One Portfolio Policy} - -Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, -$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ risk tolerance. -CRPTO solves -\begin{equation}\label{eq:portfolio} +\section{Introduction}\label{sec-introduction} + +Predictive models create value through decisions, yet a property +established for a predictor need not survive the rule that consumes it. +A probability of default can be calibrated in a population while an +optimizer concentrates capital in a systematically different subset. A +conformal prediction set can attain marginal or groupwise coverage +before selection while missing outcomes among the observations that +receive positive exposure. Even if the predictive object were stable, a +comparison can still fail when two scores are assigned the same +numerical threshold: the score and its cap jointly define a feasible +decision problem. + +The usual retrospective workflow creates additional hazards. A loan that +has reached an endpoint by an evaluation cutoff has a label, whereas a +current or later-resolved loan may not. Treating a later archive as if +it were a verified historical snapshot, or filtering to resolved +outcomes before constructing the candidate menu, uses information +unavailable at the decision date. Pooling several years of originations +into one allocation similarly lets a decision made today choose from +tomorrow's loans. Finally, optimizing one payoff and evaluating another +can make an apparent improvement an accounting artifact. These are +estimand defects rather than presentation details. + +CRPTO studies this handoff after making timing, observability, geometry, +and comparator contracts explicit. A temporally trained CatBoost model +and Platt calibrator produce point score \(p_i\). Score strata are fixed +from all 2011 predictions, independently of outcomes. We then report the +complete set of eight consecutive six-month residual windows beginning +in 2012 and ending no later than January 2013. No result selects, +weights, or removes a window. Four coverage-only controls span a +separately fitted numeric logistic model, domain-constrained monotonic +CatBoost, a platform-signal WOE/IV scorecard, and a pricing-excluded +application WOE/IV scorecard. Each has its own Platt map and 2011 +taxonomy; none is selected from OOT outcomes or enters portfolio +optimization. The resulting clipped residual interval \([\ell_i,u_i]\) +predicts the observed binary outcome. It is neither a confidence +interval for latent individual PD nor the convex hull of its +intersection with \(\{0,1\}\). Its upper endpoint enters a guardrail +score + +\begin{equation}\protect\phantomsection\label{eq-score}{ +q_i(\gamma)=(1-\gamma)p_i+\gamma u_i, +}\end{equation} + +in the portfolio risk constraint. Guardrail and point-score policies +maximize the same model-implied objective over identical monthly menus, +budgets, loan bounds, and purpose constraints. The complete diagnostic +score path is \(\gamma\in\{0,0.25,0.50,0.75,1\}\). Its primary empirical +contrast is the full-upper-score endpoint minus the point-score +endpoint. No development outcome selects a score, and no OOT result +promotes a gamma. + +The comparator is part of the estimand. We therefore use two rulers +constructed without policy-development or OOT evaluation outcomes on the +common attainable frontier. We call a construction +\emph{evaluation-outcome-blind} when earlier model and conformal fitting +may use historical labels but policy construction reads neither +development nor OOT evaluation outcomes. The primary ruler imposes the +same model-implied objective floor for every score and hence matches +plug-in opportunity cost. The secondary ruler uses the same relative +relaxation from each score's minimum-risk allocation to the common +plug-in optimum; it is invariant to positive affine score +transformations but does not match opportunity cost. Both rulers are +evaluated at coordinates \(\{0.25,0.50,0.75\}\). A supporting audit +checks these finite tracks against same-cap, development-matched, +contemporaneous funded-moment, and exact point-cap comparators. The +exact HiGHS basis frontier avoids fixed-grid interpolation over the +declared point-cap support. + +The empirical message is an identification result rather than a winner. +Under the declared six-month availability rule, all 40 model-by-window +all-candidate OOT coverage upper bounds are below 0.90. The primary +CatBoost pattern recurs under three semantics-preserving missing-value +encodings, one later retrospective origin, and four declared fit-label +scenarios. One score stratum crosses the nominal miscoverage prevalence +threshold between W7 and W8, coinciding with an abrupt change in +conformal geometry, but the crossing disappears when every label +unavailable at its fitting cutoff is completed as default. The empirical +coverage failure is therefore more stable than that particular geometric +explanation. In the portfolio audit, no endpoint ordering survives all +rulers and coordinates. Objective-matched .25 crosses zero for all three +metrics in every window; .50 is adverse in all eight; and .75 leaves +payoff and default unidentified in seven. Normalized .25 and .50 are +adverse, while normalized .75 leaves payoff unidentified in one window +and does not equalize plug-in opportunity cost. Independently, every +broad-support exact-cap payoff, default, and miscoverage envelope +crosses zero. + +The paper makes three contributions. + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + It gives an executable decision-interface audit: freeze the + population, information set, predictive object, and allocations; join + outcomes only afterward; then report complete perturbation families + without selecting a favorable result. +\item + It derives binary prevalence-sensitive geometry and sharp + common-outcome contrast bounds, including an exact width + decomposition. The completion sensitivity separates a stable + nontransport finding from a nonuniversal geometric explanation. +\item + It defines two common-frontier rulers and an exact basis-endpoint + support audit, showing why score, ruler, coordinate, and comparator + support jointly define the portfolio estimand. The negative conclusion + is constructive: coverage does not transport here and no universal + portfolio direction is identified. +\end{enumerate} + +This audit was retrospectively protocol-locked and hash-verified after +the archive had been inspected; it is not a prospective trial, +preregistration, or causal estimate. The purpose is to identify which +conclusions are properties of the predictive object and which are +artifacts of the decision comparison. + +\section{Related Work}\label{sec-related} + +\subsection{From predictive quality to decision +quality}\label{from-predictive-quality-to-decision-quality} + +Research on decision quality distinguishes estimation quality from the +quality of the action induced by an estimate. Fernandez-Loria and +Provost show that a useful decision ranking and an accurate effect +estimate are different objects \citep{fernandezloria2022causaldecision}, +and later make the identifying assumptions behind observational decision +rules explicit \citep{fernandezloria2025observational}. Cost-aware +calibration similarly evaluates probability errors through their +downstream asymmetry rather than through calibration alone +\citep{yang2025costaware}. Das et al.~link credit-modeling choices to +empirical design and reproducibility \citep{das2023creditgraph}. More +broadly, the AI--OR interface is valuable when prediction, mathematical +optimization, and operational interpretation form one auditable chain +\citep{wiberg2025ai_or}. + +Decision-focused learning trains predictors against an optimization loss +\citep{donti2017, elmachtoub2022, mandi2024}. Contextual optimization +and predict-then-optimize methods instead preserve a modular predictor +and expose how forecast errors enter the decision +\citep{bertsimas2020prescriptive, sadana2025contextual}. CRPTO takes the +modular route because the research question concerns governance of a +frozen score, not a new credit-scoring leaderboard. The baseline and +guardrail therefore share the model, payoff, budget, concentration +limits, solver, and monthly candidate menus. They cannot, however, share +a numeric risk threshold by label alone: once the score changes, the +threshold defines a different feasible set. CRPTO treats comparator +alignment as part of the empirical design rather than an implementation +default. + +\subsection{What conformal coverage does and does not +transport}\label{what-conformal-coverage-does-and-does-not-transport} + +Split conformal prediction supplies finite-sample marginal coverage +under exchangeability \citep{vovk2005, angelopoulos2023}. Exact +conditional coverage is generally unattainable without strong +restrictions \citep{barber2021limits}, and departures from +exchangeability require an explicit discrepancy, weighting, or +adaptation mechanism +\citep{tibshirani2019covshift, gibbs2021aci, barber2023beyond, farinhas2024nonexchangeable_crc}. +Selecting among valid conformal objects can itself invalidate them; +recent work constructs stable or otherwise controlled selection +procedures precisely because validity is not closed under arbitrary +data-dependent choice \citep{hegazy2025valid_selection_conformal_sets}. + +Conformal uncertainty sets have also entered portfolio, robust, and +contextual optimization \citep{kato2025, johnstone2021, patel2024}. Kato +studies asset-return portfolio selection, while Sun et al.~calibrate +predict-then-optimize contextual linear programs \citep{sun2024ptc}. +Those designs supply useful decision-level benchmarks, but their +sampling units and guarantees do not transfer to binary credit +allocation with reconstructed endpoints. Recent work instead calibrates +decision loss, operational violations, or the miscoverage--regret +frontier rather than using predictive coverage as a proxy +\citep{yeh2025training, zhou2026creme, stratigakos2026decision_calibrated_sets}. +Wang and Dobriban derive risk-optimal actions and prediction sets from +an explicit decision loss +\citep{wang2026optimal_decision_prediction_sets}, while Zheng and Jin +couple coverage to counterfactual actions when outcomes depend on the +induced policy \citep{zheng2026counterfactual_decision_sets}. CRPTO does +not compete with those methods by claiming a new selected-set theorem. +Its accepted-loan archive cannot identify a counterfactual funding +effect. It asks a complementary empirical question: when a simple +conformal score is attached to a conventional credit LP, where does its +apparent risk effect come from, and where does candidate coverage fail +to follow? + +The closest decision-level methods also clarify what this audit cannot +borrow. CROMS selects conformal models using downstream robust-decision +risk \citep{bao2025croms}; decision-theoretic conformal prediction +begins from the agent's loss \citep{kiyani2025}; and inverse conformal +risk control calibrates a robustness choice against decision regret +\citep{zhou2026creme}. Those methods require labeled decision contexts +under their stated sampling conditions. CRPTO instead has eleven common +development menus, fifteen later monthly decisions, and direct evidence +against temporal coverage transport. Treating hundreds of thousands of +loans as independent decision contexts would ignore that each action is +one coupled monthly allocation. We therefore use these methods to define +the correct target, not to import a finite-sample guarantee. + +\subsection{Credit maturity and economic +evaluation}\label{credit-maturity-and-economic-evaluation} + +Credit scoring and profit scoring are related but distinct. High +discrimination does not determine which loan is profitable +\citep{lessmann2015}, and Lending Club studies have long shown that +interest rate, default, recoveries, and portfolio constraints jointly +shape the investment decision +\citep{serrano2016profitscoring, lyocsa2022profit}. Recent +uncertainty-aware profit models reinforce the need to evaluate the +economic target directly +\citep{xu2024profit_risk_credit, xu2025profit_uncertainty_credit}. +Credit-risk auditability also motivates transparent scorecards and +economically signed constraints alongside flexible learners. WOE/IV +binning provides a supervised, inspectable representation, while +mathematical-programming binning can enforce bin size and monotonic +structure \citep{navaspalencia2020}. We use these devices as model-class +controls: they test whether the transport result depends on CatBoost, +incumbent grade/pricing signals, or unconstrained nonlinear shape. They +are not a scorecard-superiority claim. + +Maturity is equally central. Ignoring random censoring can bias +empirical risk \citep{ausset2022censoring}. In online lending, default +and prepayment are competing events whose timing changes portfolio +profitability \citep{li2023online_loans}, and dynamic portfolio models +track state transitions and cash flows rather than a single binary +reward \citep{djeundje2025dynamic_loan_portfolio_profitability}. Our +status-indexed payoff proxy is deliberately simpler. It is useful for +isolating the decision effect of PD and conformal uncertainty, but it is +not an internal rate of return, a discounted cash-flow estimate, or a +substitute for survival analysis. + +\subsection{Closest-work boundary}\label{closest-work-boundary} + +CRPTO lies between several mature literatures, so its contribution +cannot be that any one ingredient is new. Classical and data-driven +robust optimization make the price of protection explicit +\citep{bertsimas2004, bertsimas2018datadriven, goldfarb2003robustportfolio}. +P2P lending research already combines credit scores, returns, and +portfolio constraints +\citep{guo2016p2p, zhao2016p2pportfolio, chi2019p2p, babaei2020p2p}. +Conformal robust optimization carries coverage-backed sets into +downstream decisions \citep{johnstone2021, patel2024, hu2026crc}, while +valid-selection and decision-calibration methods directly target the +inferential break caused by choosing a set or action +\citep{hegazy2025valid_selection_conformal_sets, zhou2026creme, stratigakos2026decision_calibrated_sets}. + +The active CRPTO role is narrower: it audits what happens when a +conventional, frozen credit score receives a simple conformal +upper-score constraint. It does not retrain through the optimizer, +calibrate a selected-set loss, or claim selected-set validity. Its +theoretical additions concern this specific handoff: a binary-residual +threshold discontinuity, comparator-feasibility results, and an exact +identification-width identity for unresolved binary outcomes. Its +empirical additions are a maturity-restricted credit protocol, complete +residual-window reporting, an exact comparator frontier, coherent +economic comparison, and sharp treatment of unresolved outcomes. + +Fernandez-Loria and Provost motivate the distinction between an +intermediate estimate and the action it induces, Yang and Bi make +downstream cost central to calibration, Das et al.~link credit-modeling +choices to empirical design, and Wiberg et al.~frame the AI--OR +interface. The unit of analysis is the combined predictive and decision +system: the discrete geometry of the interval, the allocation it +induces, and the comparator support required to interpret that +allocation. Online Supplement Table S13 preserves the detailed +closest-work matrix while keeping the manuscript focused on the active +estimands. + +\section{Data and Locked Evaluation Design}\label{sec-data} + +\subsection{Decision unit, target, and +estimand}\label{decision-unit-target-and-estimand} + +The decision unit is an issue month. For month \(t\), the candidate set +\(\mathcal I_t\) contains only loans observable in that month, and each +policy maps the same menu into dollar exposures with a fresh USD 1 +million budget. This is different from ranking the full archive once: no +April 2016 decision can fund a May 2016 loan, and no capital is carried +across months. Equal monthly budgets make the pooled exposure-weighted +metric equivalent to the average of the 15 monthly dollar-weighted +metrics. + +The endpoint is terminal default reconstructed as observable by +September 30, 2020. A Fully Paid status is available at the month-end of +\texttt{last\_pymnt\_d}; a Charged Off status is available at that +month-end plus six calendar months. The latter is a conservative +modeling assumption, not the known operational charge-off date. The +distributed archive is not a verified point-in-time snapshot: its +last-payment field extends through December 2020 and its +last-credit-pull field through October 2020. We therefore date endpoint +availability from servicing fields and keep later terminal statuses +unresolved. The policy estimands are historical guardrail-minus-point +differences in a status-indexed standardized payoff proxy, +exposure-weighted terminal default, and exposure-weighted interval +miscoverage over the same menus. They are not treatment effects: funding +does not cause the recorded status, rejected-loan outcomes are +unavailable, and no behavioral response to deployment is modeled. + +There are therefore three distinct populations in the analysis: the +candidate rows to which predictive coverage refers, the listed loan +amounts that define available exposure, and the optimizer-selected +funded dollars that define the decision result. Treating those +populations as interchangeable would erase the mechanism the paper is +designed to measure. + +\subsection{Status-independent loan +universe}\label{status-independent-loan-universe} + +The source is the Lending Club 2007--2020Q3 public research archive. A +full raw-file audit scans 2,925,493 rows, of which 2,925,492 are valid +dated loans, across all 142 columns. There are 2,060,077 36-month and +865,415 60-month contracts. The active design retains 640,543 rows: +every loan eligible under the declared 36-month horizon, dates, schema, +and observability rules, not a sample or computational row cap. Its +evaluation panel contains 465,117 loans: 376,890 primary candidates and +88,227 extension candidates. Candidate membership depends on issue +month, term, and fields observable at origination. It never depends on +whether the reconstructed endpoint is resolved. + +Adding the excluded rows would change the estimand, not merely increase +sample size. Sixty-month loans have a different outcome horizon; 48 +fields have negligible early support but near-complete later coverage; +and the intervening cohorts are heavily duration-censored at the March +2016 information cutoff. Available labels are 59,910/162,570 for 2014, +28,878/283,173 for 2015, and 1,110/96,120 for 2016Q1, with no observed +bads in the last group. Training a terminal-default classifier on those +resolved subsets would condition on maturity. A survival estimand could +use them, but it would change the target, theory, and paper rather than +make this binary protocol more complete. + +Status strings containing \texttt{Charged\ Off} are positive and strings +containing \texttt{Fully\ Paid} are negative only when their +reconstructed availability is no later than the cutoff. The reason +taxonomy partitions all 376,890 candidates: 307,842 are Fully Paid by +the cutoff, 56,972 are Charged Off by the cutoff, 11,551 remain +nonterminal or unresolved, 47 have reconstructed terminal availability +after the cutoff, and 478 have terminal status but no reconstructible +availability date. Thus 11,551 + 47 + 478 = 12,076 outcomes remain +unrestricted. Exact \texttt{Default} and every other nonterminal status +are included in the 11,551-row category. This is a cutoff-specific +terminal classification endpoint, not a lifetime hazard or causal +response. Unresolved loans remain candidates and enter sharp bounds +after allocations are frozen. + +The optimizer uses listed loan amount as available exposure. A full +primary OOT reconciliation finds only USD 18,000 of +requested-minus-funded difference over 376,890 loans, a funded ratio of +0.999996. The distinction is empirically immaterial here while remaining +explicit in the estimand. + +The chronology has shared PD and probability-calibration blocks, a +13-month residual pool, one common outcome-free development block, and +one common OOT panel. Every consecutive six-month residual window +beginning January--August 2012 is reported; all end by January 2013. +February--December 2013 supplies the same eleven development menus for +every window. Policy-development outcomes are neither required nor read. +The first primary month is April 2016. Its window ends in June 2017, at +least 39 months before the September 2020 cutoff for a 36-month +contract. Contract age nevertheless does not force administrative +resolution, which is why 12,076 primary candidates remain unresolved. +July--September 2017 is retained as a more heavily censored extension +rather than silently discarded. + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\linewidth - 8\tabcolsep) * \real{0.1667}} + >{\raggedleft\arraybackslash}p{(\linewidth - 8\tabcolsep) * \real{0.2222}} + >{\raggedleft\arraybackslash}p{(\linewidth - 8\tabcolsep) * \real{0.2222}} + >{\raggedleft\arraybackslash}p{(\linewidth - 8\tabcolsep) * \real{0.2222}} + >{\raggedright\arraybackslash}p{(\linewidth - 8\tabcolsep) * \real{0.1667}}@{}} +\caption{Active data blocks. Residual-window rows overlap by +construction and are not independent +samples.}\label{tbl-protocol}\tabularnewline +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Block +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Issue months +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Rows +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Labels available/read +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +Role +\end{minipage} \\ +\midrule\noalign{} +\endfirsthead +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Block +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Issue months +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Rows +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedleft +Labels available/read +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +Role +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +PD development & 2007-06--2010-12 & 17,433 & 17,392 & train/validate \\ +Probability calibration & 2011-01--2011-12 & 14,101 & 14,077 & Platt fit +and taxonomy \\ +Residual pool & 2012-01--2013-01 & 49,007 & 48,857 & eight overlapping +six-month recipes \\ +Policy development & 2013-02--2013-12 & 94,885 & not read & common +outcome-free frontier construction \\ +Primary OOT & 2016-04--2017-06 & 376,890 & post-freeze only & locked +evaluation \\ +Censored extension & 2017-07--2017-09 & 88,227 & post-freeze only & +stress only \\ +\end{longtable} + +\subsection{Information boundary}\label{information-boundary} + +The implementation materializes two ID-keyed panels. The decision panel +contains issue date, amount, purpose, contractual rate, point score, +conformal endpoints, and the frozen score stratum. It rejects outcome, +status-indexed payoff, miscoverage, or outcome-derived columns. The +outcome panel contains the reconstructed endpoint and is joined only +after the solver returns an allocation. IDs must align one-to-one; +partial joins fail. This physical separation does not create a new +statistical theorem, but it makes the timing claim testable in code. + +Label-dependent fitting is additionally restricted by an information +cutoff of March 31, 2016. A Fully Paid label becomes available at the +month-end of its last payment; a Charged Off label is conservatively +dated six calendar months after that month-end. The retained shares are +99.765\% in PD development, 99.830\% in probability calibration, and +99.694\% over the residual pool. Every residual month exceeds 99\%; the +minimum is 99.296\% in January 2013. Missing dates and labels arriving +after the cutoff are excluded from fitting, not from the candidate +universe. + +The outcome-free protocol froze two learner score vectors, 64 residual +recipes, 51,117 solve records, and 5,001,617 funded rows before any +outcome join. Its maximum absolute C2 match residual is +\(8.33\times10^{-17}\). The active evaluator verifies that freeze by +SHA-256, reconstructs status availability, and injects window-specific +endpoints only after the shared point allocation is loaded. It changes +neither predictions, recipes, comparators, nor allocations. Online +Supplement Appendix F records the active lineage and replay contract. + +The two-ruler diagnostic has a separate outcome-free freeze. It contains +6,240 solves and 622,455 funded rows over eight windows, 26 +development/OOT months, five gamma values, three interior coordinates, +and two rulers. Reversed-ID reruns and a separate GLOP implementation +check the endpoint allocations. A hash-locked evaluator then joins the +same outcome panel without refitting, resolving, or selecting a track. +Its 48 window cells are six specification tracks observed through eight +overlapping residual recipes, not 48 replications. + +\subsection{Identification safeguards}\label{identification-safeguards} + +The reusable audit has five stages. First, define status-independent +monthly menus and dated label availability. Second, fit scores, +taxonomies, and every residual recipe without evaluation outcomes. +Third, declare comparator rulers, solve the complete policy grid, and +persist allocations. Fourth, join one keyed outcome panel and compute +sharp candidate and paired-policy bounds. Fifth, report every declared +window, model control, ruler, coordinate, support, and sensitivity; a +claim survives only its named perturbation family. This order prevents +outcome filtering, future-menu look-ahead, payoff mismatch, and +result-dependent comparator choice. It does not restore exchangeability, +identify lifetime cash flows, or turn a finite grid into universal +support. Online Supplement Appendices A and G give the full audit map +and boundaries. + +\section{Method}\label{sec-method} + +\subsection{Platt-scaled default score and credit-risk +controls}\label{platt-scaled-default-score-and-credit-risk-controls} + +A CatBoost classifier uses 29 numeric and 9 categorical origination-time +features. Hyperparameters are fixed in the executable protocol: 500 +trees, depth 6, learning rate 0.04, class balancing, Bernoulli +subsampling, and a time-aware ordering. The last 20\% of PD-development +months form a temporal validation tail. A logistic Platt map is then +fitted on the 2011 raw margins, separate from both model training and +conformal fitting. + +The prediction model is deliberately not tuned against downstream +outcomes. Canonical seed 42 is inherited from the locked design. Four +protocol-locked coverage-only specifications alter model class or +credit-risk information: + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + numeric-feature logistic regression; +\item + CatBoost with 20 domain-signed monotonic constraints, such as + increasing risk in interest rate, debt burden, inquiries, and + adverse-history flags and decreasing risk in income, FICO, and credit + age; +\item + a 26-field OptBinning WOE/IV logistic scorecard including borrower, + contract, grade, subgrade, interest, and derived pricing signals; and +\item + a 19-field pricing-excluded application WOE/IV scorecard excluding + grade, subgrade, interest, installment, and derived platform-pricing + signals. +\end{enumerate} + +Each WOE process is fit only on PD development, uses two to eight bins, +a 5\% minimum bin share, and automatic monotonic trend +\citep{navaspalencia2020}. Every learner receives its own 2011 Platt map +and fixed score taxonomy. On the Platt block, AUCs in primary/control +order are 0.676327, 0.656880, 0.678091, 0.660533, and 0.662401; Brier +scores range from 0.090054 to 0.091000. All five are frozen before the +primary OOT outcome join and use every eligible row. The four controls +never enter portfolio optimization, and OOT outcomes select neither +model nor feature. These diagnostics test specification dependence; they +are not a predictive leaderboard or a claim that WOE/IV is novel. + +The full-file contract identifies two active-input exceptions to the +95\% raw coverage rule. Missing delinquency recency is structurally +meaningful and is mapped to the frozen no-recent-delinquency convention; +bankruptcy count has 92.17\% minimum early coverage and missing values +map to no recorded bankruptcy. Observed-value indicators are retained by +feature engineering for auditability, but the active portfolio model +does not add them as post hoc predictors. A separately frozen +sensitivity preserves the two feature definitions while comparing the +active sentinel convention, explicit missing indicators, and +CatBoost-native nullable values. It selects no encoding and does not +identify a missingness mechanism. + +\subsection{Exact binary-outcome Mondrian +intervals}\label{exact-binary-outcome-mondrian-intervals} + +Let \(Y_i\in\{0,1\}\) denote terminal default and \(p_i\) the +Platt-scaled score. Five groups are defined by quantiles of all 2011 +Platt-scaled scores, without using residual-window labels. The frozen +CatBoost edges are \(0.008219\), \(0.050265\), \(0.080369\), +\(0.111894\), \(0.153262\), and \(0.475854\). Within fixed group \(g\), +the conformity score on an availability-safe residual window is +\(s_i=|Y_i-p_i|\). The complete specification reports all eight +consecutive six-month windows beginning January--August 2012. Every +included month exceeds 99\% label retention under the six-month lag. For +\(n_g\) observations and target \(\alpha=0.10\), the finite-sample rank +is + +\begin{equation}\protect\phantomsection\label{eq-rank}{ +k_g=\left\lceil(n_g+1)(1-\alpha)\right\rceil, +}\end{equation} + +and \(c_g\) is the \(k_g\)-th ordered residual. A future score assigned +to group \(g(i)\) receives + +\begin{equation}\protect\phantomsection\label{eq-interval}{ +[\ell_i,u_i]= +\left[\max\{0,p_i-c_{g(i)}\},\min\{1,p_i+c_{g(i)}\}\right]. +}\end{equation} + +The interval predicts the observed binary outcome. It is not a +confidence interval for latent individual PD. No holdout-learned +widening, floor, taxonomy adaptation, or post-cutoff label enters any +recipe. Taxonomies with 1, 2, and 10 groups are closed coverage +diagnostics. Each of the four controls repeats this construction with +its own scores and edges. + +For binary \(Y\), miscoverage has the useful identity + +\begin{equation}\protect\phantomsection\label{eq-binary-miss}{ +m_i=\mathbf 1\{Y_i=0,\ell_i>0\}+ + \mathbf 1\{Y_i=1,u_i<1\}. +}\end{equation} + +Thus a default is counted as covered whenever \(u_i=1\), even though +such an endpoint offers little discrimination to the optimizer. +Conversely, a narrow low-score interval with \(u_i<1\) misses every +realized default. This geometry is central to interpreting funded-set +coverage. + +\subsection{Coherent objective and status-indexed payoff +proxy}\label{coherent-objective-and-status-indexed-payoff-proxy} + +For contractual annual rate \(r_i\) and fixed loss given default +\(\lambda=0.45\), define the per-dollar status-indexed standardized +payoff proxy + +\begin{equation}\protect\phantomsection\label{eq-status-payoff}{ +\pi_i(Y_i)=(1-Y_i)r_i-Y_i\lambda, +}\end{equation} + +with model-implied plug-in objective coefficient + +\begin{equation}\protect\phantomsection\label{eq-expected-payoff}{ +\bar\pi_i=(1-p_i)r_i-p_i\lambda. +}\end{equation} + +This coefficient equals the conditional expectation of the proxy only if +\(p_i\) equals the conditional default probability. We use it as a +model-implied plug-in objective and evaluate the corresponding +status-indexed proxy, not as proof of conditional calibration. The pair +intentionally omits payment timing, principal amortization, prepayment, +fees, recoveries, and discounting. We therefore do not call the endpoint +profit, cash-flow return, NPV, welfare, or IRR. + +In month \(t\), let \(a_{it}\) be dollar exposure to loan \(i\), bounded +by its listed amount \(A_i\). For a policy \((\tau,\gamma)\), the LP is + +\begin{equation}\protect\phantomsection\label{eq-lp}{ \begin{aligned} -\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\max_{a_{it}}\quad & \sum_{i\in\mathcal I_t}a_{it}\bar\pi_i \\ \text{s.t.}\quad -& \sum_i x_i a_i\le B, \\ -& \sum_i x_i a_iq_i\le\tau\sum_i x_i a_i, \\ -& 0\le x_i\le\bar x_i, -\end{aligned} -\end{equation} -with the existing eligibility and concentration constraints. The selected -policy uses $\tau=0.17$ and Equation~\eqref{eq:midpoint}. The objective uses -$p_i$, not $q_i$: expected economics and uncertainty feasibility are separate -contracts. The matched point-PD baseline changes only the risk score to -$q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and -risk tolerance fixed. - -For funded-exposure weights $w_i=x_ia_i/\sum_jx_ja_j$, define -\begin{equation}\label{eq:gamma} -\begin{aligned} -\Gamma_{\mathrm{CP}}&=\sum_iw_i(u_i-p_i),\\ -\Gamma_{\mathrm{int}}&=\sum_iw_i(q_i-p_i),\\ -\Gamma_{\mathrm{res}}&=\sum_iw_i(u_i-q_i),\\ -B_u&=\sum_iw_iu_i. +& \sum_i a_{it}=B, \\ +& \sum_i a_{it}q_i(\gamma)\le \tau B, \\ +& \sum_{i:\,purpose_i=k}a_{it}\le 0.25B \quad\forall k,\\ +& 0\le a_{it}\le A_i, \end{aligned} -\end{equation} -For the midpoint policy, -$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. - -\subsection{Calibration-Only Final Selector} - -The candidate set crosses $\tau\in\{0.15,0.17,0.19\}$ with -$\gamma\in\{0.25,0.50,0.75\}$ in -$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, -at least 99.9\% of budget is allocated, the effective-PD cap holds, and -\begin{equation}\label{eq:screen} -B_u\le0.28 -\end{equation} -on November 2017. Among eligible candidates, the rule maximizes expected -point-PD objective. Five of nine pass; the selected row is -$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic and does not use -the weighted-validity assumption below. The midpoint remains selected for -every cap in $[0.259036,0.290491)$, and an outcome-free December replay selects -the same row. - -\begin{table}[t] -\centering -\small -\caption{Temporally separated selector examples; the supplement reports all nine rows.} -\label{tab:selector} -\resizebox{\textwidth}{!}{% -\begin{tabular}{lrrrrrl} -\toprule -Candidate & $\tau$ & $\gamma$ & Nov. objective & Nov. $B_u$ & Dec. $B_u$ & Status \\ -\midrule -Low guardrail & 0.17 & 0.25 & \$109{,}885.56 & 0.375105 & 0.396751 & Ineligible \\ -Selected midpoint & 0.17 & 0.50 & \$99{,}387.12 & 0.259036 & 0.262082 & Selected twice \\ -Conservative blend & 0.17 & 0.75 & \$93{,}760.13 & 0.202259 & 0.203504 & Eligible \\ -Higher tolerance & 0.19 & 0.50 & \$102{,}671.87 & 0.290491 & 0.294861 & Ineligible \\ -\bottomrule -\end{tabular}} -\end{table} - -The selector's input whitelist contains no outcomes or -assumption-conditional quantities. Earlier research iterations used the OOT -corpus; the narrow claim is that this final ranking code path does not. - -\subsection{Accounting and Statistical Boundary} - -Let $Z_i=\mathbf1\{Y_i>u_i\}$ and $V=\sum_iw_iZ_i$. Since -$Y_i\le u_i+Z_i$ for every funded loan, -\begin{equation}\label{eq:accounting} -\sum_iw_iY_i\le B_u+V -\end{equation} -holds deterministically. It is funded-set accounting, not a coverage theorem. - -If one additionally assumes weighted funded-set validity, -$\mathbb E[V]\le\alpha$, Markov's inequality gives -\begin{equation}\label{eq:markov} -\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) -\le\sqrt{\alpha}. -\end{equation} -The assumption does not follow from marginal split conformal because the -optimizer chooses $w_i$ from $p_i$ and $u_i$. Equation~\eqref{eq:markov} is -therefore secondary, assumption-conditional sensitivity. The supplement gives -the proof and selected-set boundary \citep{hegazy2025valid_selection_conformal_sets}. - -\section{Results}\label{sec:results} - -\subsection{Exact 90\% Conformal Evidence} - -At target $\alpha=0.10$, exact OOT coverage is 0.934836, average width is -0.788879, minimum score-partition coverage is 0.926310, minimum letter-grade -coverage is 0.926797, and 51.7873\% of upper endpoints equal one. The intervals -are conservative and broad, as expected for a binary outcome on the probability -scale. - -The exact alpha sensitivity explains why the paper does not headline a 99\% -interval. At target alpha 0.01, coverage is 0.996720, but average width is -0.988215 and 93.5424\% of upper endpoints equal one. Such endpoints carry -almost no ranking information for a portfolio. The selected 90\% level is the -frozen recipe's reference level and preserves more decision resolution. - -\subsection{Pre-OOT Selector and Decision Audit} - -November selects the midpoint policy from five eligible rows. Applying the -same outcome-free rule to December again selects \texttt{linear-005}; outcomes -are opened only after that replay. - -\begin{table}[t] -\centering -\small -\caption{Independent December 2017 post-selection decision audit.} -\label{tab:calibration-audit} -\resizebox{\textwidth}{!}{% -\begin{tabular}{lrrrrr} -\toprule -Policy & Funded & Realized return & Weighted default & Miscoverage & $B_u$ \\ -\midrule -Selected 50/50 CRPTO & 193 & \$53{,}313.05 & 0.145650 & 0.124925 & 0.262082 \\ -Conservative 75\% blend & 191 & \$38{,}379.50 & 0.155250 & 0.134525 & 0.203504 \\ -Point-PD matched $\tau$ & 169 & \$89{,}732.35 & 0.185650 & 0.058300 & 0.888071 \\ -\bottomrule -\end{tabular}} -\end{table} - -The selected policy remains below its operational tolerance -($0.145650<0.17$) but misses nominal funded-set coverage -($V=0.124925>0.10$). Its deterministic accounting right-hand side is -$B_u+V=0.387007$. Thus outcome-free, month-stable selection does not create -selected-set conformal validity. This negative audit motivates the two-layer -reporting of deterministic endpoint exposure and observed miscoverage. - -\subsection{Full OOT Funded-Set Audit} - -The fixed midpoint policy allocates the full \$1M budget across 308 loans. Its -expected point-PD objective is \$168{,}271.56; realized return is -\$179{,}327.59. Weighted default is 0.039375 and weighted miscoverage is -0.036875. - -\begin{table}[t] -\centering -\small -\caption{Exact full-OOT audit of the selected midpoint policy.} -\label{tab:active} -\begin{tabular}{lr} -\toprule -Quantity & Value \\ -\midrule -Weighted point PD & 0.081949 \\ -Weighted midpoint score & 0.170000 \\ -$\Gamma_{\mathrm{CP}}$ & 0.176102 \\ -$\Gamma_{\mathrm{int}}$ & 0.088051 \\ -$\Gamma_{\mathrm{res}}$ & 0.088051 \\ -Endpoint budget $B_u$ & 0.258051 \\ -Observed accounting bound $B_u+V$ & 0.294926 \\ -Conditional Markov threshold & 0.574279 \\ -\bottomrule -\end{tabular} -\end{table} - -The observed weighted outcome 0.039375 is below the exact accounting right-hand -side 0.294926. Under the additional weighted-validity assumption, the 0.574279 -event threshold has probability bound $\sqrt{0.10}=0.316228$. This loose -statement is not a direct default cap; the operational controls are $\tau$, -$q_i$, and the funded-set diagnostics. - -A fixed-allocation bootstrap over 31 origination-month clusters gives a 95\% -return interval of \$163{,}421.14--\$193{,}551.65 from 5{,}000 draws. A -funded-loan sensitivity gives \$162{,}706.17--\$193{,}924.74. Neither resamples -the model, conformal recipe, selector, or optimizer. The month-level scheme is -primary because the temporal results reject a naive independence story. - -\subsection{Matched Comparators} - -\begin{table}[t] -\centering -\small -\caption{Full-OOT matched decision comparison.} -\label{tab:matched} -\resizebox{\textwidth}{!}{% -\begin{tabular}{lrrrrrr} -\toprule -Policy & Funded & Return & Default & Miscoverage & $B_u$ & Threshold \\ -\midrule -Selected 50/50 CRPTO & 308 & \$179{,}327.59 & 0.039375 & 0.036875 & 0.258051 & 0.574279 \\ -Conservative 75\% blend & 312 & \$172{,}939.50 & 0.035875 & 0.035875 & 0.200396 & 0.516624 \\ -Point-PD matched $\tau$ & 225 & \$196{,}369.14 & 0.118400 & 0.041900 & 0.921317 & 1.237545 \\ -\bottomrule -\end{tabular}} -\end{table} - -Relative to point PD, selected CRPTO gives up \$17{,}041.55, or 8.678\% of -realized return. Weighted default falls by 7.9025 percentage points, -miscoverage by 0.5025 percentage points, and the endpoint-plus-Markov threshold -by 66.3266 percentage points. The default contrast is much larger than the -miscoverage contrast: the guardrail changes the funded loans but does not create -a dramatic selected-set coverage gain. - -The 75\% blend lowers default by 0.35 percentage points and the threshold by -0.057655 relative to the midpoint, but costs another \$6{,}388.08. The -calibration selector chooses the midpoint because it has the highest expected -objective under the declared screen, not because it dominates every risk -metric. - -\subsection{Temporal Heterogeneity} - -\begin{table}[t] -\centering -\small -\caption{Fixed-policy temporal stress evaluation; each row uses a fresh \$1M budget.} -\label{tab:temporal} -\begin{tabular}{lrrrr} -\toprule -Period & CRPTO return & Point-PD return & CRPTO default & Point-PD default \\ -\midrule -2018H1 & \$92{,}530.73 & \$118{,}101.99 & 0.106703 & 0.190825 \\ -2018H2 & \$156{,}185.51 & \$95{,}603.58 & 0.026725 & 0.236728 \\ -2019H1 & \$123{,}590.69 & \$144{,}281.46 & 0.077325 & 0.170275 \\ -2019H2 & \$110{,}251.95 & \$256{,}966.20 & 0.103250 & 0.023775 \\ -2020+ & \$99{,}689.54 & \$218{,}629.14 & 0.083775 & 0.016900 \\ -\bottomrule -\end{tabular} -\end{table} - -CRPTO is economically and statistically attractive in 2018H2, when the point- -PD portfolio concentrates in realized defaults. In 2019H2 and 2020+, point PD -earns more and defaults less. The result is therefore a full-period return-risk -trade-off with temporal heterogeneity, not a universal robustness premium. - -\subsection{Funded-Set Composition} - -Grades C and D represent 31.36\% and 58.68\% of funded exposure. Grade F is -only 1.30\% but has realized default 0.269231. The supplement reports every -letter grade and reconciles to the full allocation. Letter grade is recovered -from the original subgrade field; the score-quantile conformal group is stored -separately. This is a business-risk composition audit, not legal fair-lending -certification. - -\section{Robustness, Comparators, and Implications}\label{sec:implications} - -The online supplement keeps diagnostics that help challenge the one active -method without turning the paper into several methods: exact alpha saturation, -all selector cells, temporal results, grade composition, bootstrap, and matched -comparators. Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, and -Freddie/Mendeley analyses are supporting diagnostics or external context. They -do not select the active policy. - -For a credit committee, the useful control is not the Markov threshold alone. -The committee chooses a conformal level, a transparent blend $\gamma$, and a -portfolio tolerance $\tau$, then reads economic cost, endpoint exposure, -realized default, and temporal behavior together. Here the midpoint has a clear -interpretation: half of the conformal premium enters the enforceable risk score -and half remains visible as residual endpoint exposure. - -The matched baseline turns that interpretation into a decision. A committee -that accepts the full-period evidence pays 8.678\% of realized return for a -7.9025 percentage-point default reduction. A more conservative committee can -use the 75\% blend and pay another roughly \$6.4K. A committee emphasizing -2019H2 or 2020 should reject the static rule or require a new temporal -recalibration protocol. CRPTO exposes that choice rather than hiding it behind -a single score. - -\section{Reproducibility and Limitations}\label{sec:limitations} - -The research bundle versions code, manuscript sources, configurations, tables, -figures, and governance files. Heavy data and model artifacts are stored -separately and checked by hashes. The evidence builder regenerates A35--A40 -from frozen exact-alpha and policy outputs. Claim-sync tests verify that the -body, supplement, and this official TeX surface share policy settings, selector -counts, and numeric anchors. - -Several limitations bound the contribution. Lending Club retail origination -ended in 2020, so the historical panel cannot establish live performance. The -final selector is outcome-free with respect to OOT ranking, but earlier project -development inspected the same OOT corpus. The evaluation is retrospective, -not prospective. The conformal intervals are broad because the outcome is -binary; more than half of 90\% OOT upper endpoints equal one. Mondrian coverage -does not imply validity under optimizer-selected funded weights. The December -audit makes this visible: the same policy is reselected without outcomes, yet -funded-set miscoverage is 0.124925. The Markov statement therefore requires an -explicit assumption and is absent from policy selection. Temporal slices show -that point PD can dominate both return and default. Public data also do not -support a legal fair-lending certification or a causal interpretation. - -A focused next step is a genuinely prospective or formally selection-valid -protocol frozen before a new evaluation period -\citep{farinhas2024nonexchangeable_crc,hegazy2025valid_selection_conformal_sets, -zhou2026creme}. -That extension is not hidden as an acceptance criterion for this retrospective -decision audit. - -For double-anonymous review, author-identifying repository URLs are omitted. -Code/data access and artifact instructions can be disclosed under the journal's -policy without embedding credentials or identity in the anonymous PDF. - -\section{Conclusion}\label{sec:conclusion} - -CRPTO shows how a frozen credit model can become an auditable portfolio -decision without a maze of policy variants. An exact 90\% conformal replay -produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a nine-cell -November selector fixes $\tau=0.17$ under $B_u\le0.28$. December independently -selects the same rule and then misses nominal funded-set coverage; policy -stability is therefore not relabeled as conformal validity. On the -full panel, the policy earns \$179{,}327.59, with weighted default 0.039375, -miscoverage 0.036875, $\Gamma_{\mathrm{CP}}=0.176102$, -$\Gamma_{\mathrm{res}}=0.088051$, endpoint 0.258051, observed accounting bound -0.294926, and conditional Markov threshold 0.574279. Against matched point PD, -it pays 8.678\% of realized return for a 7.9025 percentage-point default -reduction. Temporal reversals keep the claim narrow: CRPTO is an inspectable -retrospective return-risk guardrail, not a universal winner, prospective -deployment guarantee, or new credit-scoring leaderboard. +}\end{equation} + +where \(B=\$1\) million in every month. The point-score baseline sets +\(\gamma=0\). The objective is identical across policies, so differences +arise from the risk score and the loans made feasible by it. The LP +permits fractional dollar exposure. A separately frozen implementation +diagnostic floors each positive exposure to a USD 25 lot and holds the +residual as cash; it does not reoptimize or define an integer-policy +estimand. + +\subsection{Common-frontier rulers and supporting comparator +audit}\label{common-frontier-rulers-and-supporting-comparator-audit} + +Let \(\mathcal A_t\) contain the budget, loan-bound, and purpose +constraints in Equation~\ref{eq-lp}, let \(v_i=\bar\pi_i\), and let +\(s_i(\gamma)=q_i(\gamma)\) for \(\gamma\in\{0,0.25,0.50,0.75,1\}\). For +every month and score, define the minimum funded score, the common +unconstrained plug-in optimum, and the score of that optimum as + +\begin{equation}\protect\phantomsection\label{eq-ruler-anchors}{ +m_{\gamma t}=B^{-1}\min_{a\in\mathcal A_t}s(\gamma)^\top a, +\quad z_t^*=\max_{a\in\mathcal A_t}v^\top a, +\quad o_{\gamma t}=B^{-1}s(\gamma)^\top a_t^*. +}\end{equation} + +The primary \textbf{objective-matched ruler} compares scores at a common +plug-in opportunity cost. Let \(z^{\min}_{\gamma t}\) be the objective +attained by a minimum-score portfolio and set +\(z_t^L=\max_\gamma z^{\min}_{\gamma t}\). At +\(\rho\in\{0.25,0.50,0.75\}\), every score solves + +\begin{equation}\protect\phantomsection\label{eq-objective-ruler}{ +\min_{a\in\mathcal A_t}s(\gamma)^\top a +\quad\text{s.t.}\quad +v^\top a\ge z_t^L+\rho(z_t^*-z_t^L). +}\end{equation} + +The shared floor is a model-implied objective, not true expected return. +The secondary \textbf{normalized-score ruler} instead sets + +\begin{equation}\protect\phantomsection\label{eq-normalized-ruler}{ +c_{\gamma t}(\eta)=m_{\gamma t} + +\eta(o_{\gamma t}-m_{\gamma t}),\qquad +\eta\in\{0.25,0.50,0.75\}, +}\end{equation} + +and maximizes \(v^\top a\) subject to +\(s(\gamma)^\top a\le Bc_{\gamma t}(\eta)\). Equal \(\eta\) is invariant +to a positive affine rescaling of a score, but it is not equal default +risk, objective sacrifice, or operational tolerance. At coordinate one +both rulers reach the verified common plug-in optimum, so the endpoint +allocation contrast is structurally null. The empirical diagnostic is +therefore the complete declared three-coordinate interior grid, not a +continuous-frontier claim. + +For each ruler and coordinate, the frozen contrast is \(\gamma=1\) minus +\(\gamma=0\). Interior gamma values verify the path but cannot be +selected as winners. Every allocation is outcome-free; unresolved +outcomes enter only through the common-outcome bounds after the freeze. + +A supporting audit checks the finite grid against declared point-cap +comparators. C0 copies the numerical cap, C1 uses the eleven-menu +development point-score support, and C2 matches the frozen guardrail's +funded point-score moment on the same menu. C2 is an outcome-free +decomposition, not a deployable policy. HiGHS basis ranging then +enumerates 3,067 exact point caps over development support and {[}0.05, +0.12{]} broad stress. Nine historical \((\tau,\gamma)\) pairs index this +audit; they are not a closed policy family or promotion candidates. Full +definitions are in Online Supplement Appendix C. No OOT outcome chooses +any learner, window, gamma, ruler, coordinate, cap, policy, or +comparator. + +\section{Audit Theory and Estimands}\label{sec-theory} + +The theory separates four objects that are often conflated: score units, +comparator stringency, binary interval geometry, and unresolved-outcome +identification. None of the statements assumes funded-set conformal +validity or makes one ruler uniquely correct. Appendix D contains the +proofs. + +\subsection{Comparator non-invariance}\label{comparator-non-invariance} + +Let \(\mathcal F_s(\tau)\) denote the allocations satisfying +Equation~\ref{eq-lp} when the risk score is \(s\). All nonrisk +constraints and the objective are held fixed. + +\textbf{Proposition 1 (what comparator matching preserves).} Suppose the +budget binds, \(\sum_i a_i=B\). + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\tightlist +\item + If \(s_i=\kappa p_i+b\) with \(\kappa>0\), then +\end{enumerate} + +\begin{equation}\protect\phantomsection\label{eq-affine-cap}{ +\sum_i a_i s_i\le \tau_s B +\quad\Longleftrightarrow\quad +\sum_i a_i p_i\le \frac{\tau_s-b}{\kappa}B. +}\end{equation} + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\setcounter{enumi}{1} +\tightlist +\item + If \(u_i\ge p_i\) and the same numerical cap is copied to + \(q_i(\gamma)\), then +\end{enumerate} + +\begin{equation}\protect\phantomsection\label{eq-feasible-nesting}{ +\mathcal F_q(\tau)\subseteq\mathcal F_p(\tau). +}\end{equation} + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\setcounter{enumi}{2} +\tightlist +\item + For \(\widetilde s=\kappa s+b\), the normalized ruler transforms as + \(c_{\widetilde s}(\eta)=\kappa c_s(\eta)+b\), so +\end{enumerate} + +\begin{equation}\protect\phantomsection\label{eq-normalized-invariance}{ +\widetilde s^\top a\le Bc_{\widetilde s}(\eta) +\quad\Longleftrightarrow\quad +s^\top a\le Bc_s(\eta). +}\end{equation} + +\begin{enumerate} +\def\labelenumi{\arabic{enumi}.} +\setcounter{enumi}{3} +\tightlist +\item + If C2 sets \(c=(p^\top a^q)/B\) for a guardrail allocation \(a^q\), + that allocation is feasible for the point-score LP and +\end{enumerate} + +\begin{equation}\protect\phantomsection\label{eq-c2-dominance}{ +V_p(c)\ge \bar\pi^\top a^q. +}\end{equation} + +Parts 1 and 3 identify exact invariances. Parts 2 and 4 show why +same-cap and funded-moment matching are not neutral performance tests: +they mechanically order feasible plug-in objectives but do not order +realized payoff, default, or miscoverage. Equality of one optimizer is +also insufficient to establish affine score equivalence. + +\subsection{Binary interval geometry}\label{binary-interval-geometry} + +\textbf{Proposition 2 (binary threshold geometry).} Fix \(0\le p<1/2\) +and let \(Y\sim\mathrm{Bernoulli}(\pi)\). Under the lower population +\((1-\alpha)\)-quantile convention, + +\begin{equation}\protect\phantomsection\label{eq-binary-phase}{ +c_{1-\alpha}= +\begin{cases} +p, & \pi\le\alpha,\\ +1-p, & \pi>\alpha. +\end{cases} +}\end{equation} + +Thus the clipped interval changes from \([0,2p]\) to \([0,1]\) as +prevalence moves above \(\alpha\); coverage jumps from \(1-\pi\) to one +and the upper decision score from \(2p\) to one. For any varying-score +interval, Equation~\ref{eq-binary-miss} gives the exact two-case +miscoverage decomposition: \(u=1\) covers every default, whereas every +default with \(u<1\) is missed. Endpoint saturation can therefore +improve coverage while removing ranking information. The constant-score +result is a mechanism, not a proof for an empirical stratum with varying +\(p_i\). + +\subsection{Sharp bounds for unresolved +outcomes}\label{sharp-bounds-for-unresolved-outcomes} + +For an unresolved loan, \(Y_i\) may be either 0 or 1. Default therefore +lies in \([0,1]\), payoff in \([-\lambda,r_i]\), and miscoverage in the +two attainable values implied by Equation~\ref{eq-binary-miss}. + +\textbf{Proposition 3 (sharp common-outcome bounds and width).} Let +\(U\) index unrestricted outcomes and write an additive fixed-allocation +metric as \(T(Y)=C+\sum_{i\in U}g_i(Y_i)\). Then + +\begin{equation}\protect\phantomsection\label{eq-sharp-bounds}{ +T_L=C+\sum_{i\in U}\min_{y\in\{0,1\}}g_i(y),\qquad +T_U=C+\sum_{i\in U}\max_{y\in\{0,1\}}g_i(y) +}\end{equation} + +are sharp because each endpoint is attained by a joint assignment of all +unresolved outcomes. This is partial identification under unrestricted +binary completion, not a sampling confidence interval. + +For all-candidate coverage, let \(m_i(0)\) and \(m_i(1)\) be the two +attainable miscoverage values. With \(m_i^L=\min_y m_i(y)\) and +\(m_i^U=\max_y m_i(y)\), the sharp coverage interval is + +\begin{equation}\protect\phantomsection\label{eq-sharp-coverage}{ +\left[1-\frac{1}{n}\sum_i m_i^U, + 1-\frac{1}{n}\sum_i m_i^L\right]. +}\end{equation} + +For paired policies, subtracting marginal intervals is not sharp. +Instead, form the union of funded IDs, retain each signed exposure +difference, and assign the same unresolved \(Y_i\) once to both +policies. If \(d_i(Y_i)\) is loan \(i\)'s contrast contribution, then + +\begin{equation}\protect\phantomsection\label{eq-identification-width}{ +T_U-T_L=\sum_{i\in U}\left|d_i(1)-d_i(0)\right|. +}\end{equation} + +For exposure difference \(\delta_i=a_i^A-a_i^B\), the dollar +payoff-proxy width is \(\sum_{i\in U}|\delta_i|(r_i+\lambda)\). Let +\(N_A,N_B\) be the declared positive capital normalizers and define +\(\delta_i^w=a_i^A/N_A-a_i^B/N_B\). Payoff-rate width is +\(\sum_{i\in U}|\delta_i^w|(r_i+\lambda)\), default-rate width is +\(\sum_{i\in U}|\delta_i^w|\), and miscoverage-rate width is +\(\sum_{i\in U}|\delta_i^w|\,|m_i(1)-m_i(0)|\). The fully invested +policies use \(N_A=N_B=B\); the floor-with-cash diagnostic also uses +committed capital \(B\) rather than renormalizing away cash. Hence +unresolved outcomes alone do not determine identification loss: width +depends on where the policies disagree in exposure and, for miscoverage, +on prediction-set geometry. These are finite-archive identification +intervals, not confidence intervals. + +\subsection{Declared comparator sensitivity +envelope}\label{declared-comparator-sensitivity-envelope} + +Let comparator \(j\) produce a sharp contrast interval \([L_j,U_j]\) for +a fixed guardrail and metric. For a declared finite set \(\mathcal J\), +define + +\begin{equation}\protect\phantomsection\label{eq-multiverse}{ +\mathcal I_{\mathcal J}= +\left[\min_{j\in\mathcal J}L_j,\max_{j\in\mathcal J}U_j\right]. +}\end{equation} + +The declared comparator sensitivity envelope \(\mathcal I_{\mathcal J}\) +summarizes endpoints over a named finite comparator set. A sign survives +only when the envelope lies strictly on one side of zero; this is +deterministic design sensitivity, not universal quantification over +baselines. + +\textbf{Proposition 4 (basis-endpoint sufficiency).} Within one optimal +LP basis, the point allocation is affine in cap \(c\). Against a fixed +guardrail, observed-outcome contrast contributions are therefore affine. +For an unresolved binary outcome, the sharp lower contribution is the +minimum of two affine functions and the sharp upper contribution is +their maximum. The lower bound is concave and the upper bound convex on +that basis range, so their adverse extrema occur at its endpoints. +Evaluating every support endpoint and every HiGHS basis-ranging endpoint +therefore gives the exact sharp sensitivity envelope over the declared +cap interval, up to solver tolerance. + +Together these statements separate exact algebra, finite-archive partial +identification, and empirical sensitivity. Online Supplement Appendix D +gives the proof map and Appendix G states the corresponding claim +boundaries. + +\section{Results}\label{sec-results} + +\subsection{Coverage fails across the complete residual-window +specification}\label{coverage-fails-across-the-complete-residual-window-specification} + +Under the active six-month evaluation endpoint, every five-stratum +recipe attains its finite-sample fit rank, but none transports 90\% +all-candidate coverage to the primary OOT panel. CatBoost resolved +coverage ranges from 0.8659 to 0.8792. After retaining all 12,076 +unresolved candidates, its sharp coverage bounds range from {[}0.8425, +0.8696{]} to {[}0.8570, 0.8826{]}. The separately calibrated logistic +control is closer to nominal: resolved coverage ranges from 0.8732 to +0.8931 and the largest upper bound is 0.8962. Nevertheless, all eight +logistic upper bounds also remain below 0.90. The finding is therefore +not unique to CatBoost, while its magnitude is learner-dependent. Four +additional credit-risk specifications test that conclusion more broadly +below. + +\begin{figure} + +\centering{ + +\includegraphics[width=0.94\linewidth,height=\textheight,keepaspectratio]{../../reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png} + +} + +\caption{\label{fig-coverage}Resolved coverage (markers) and sharp +all-candidate bounds (error bars) for the primary CatBoost and +numeric-logistic specifications. The dashed line is nominal 90\% +coverage.} + +\end{figure}% + +Online Supplement Table S6 reports all eight exact window bounds and +widths; the figure retains their complete nonselective pattern in the +main text. + +All five all-candidate sharp upper bounds remain below 0.90 in all eight +windows, including the monotonic learner and the pricing-excluded +application scorecard. The latter has the largest across-window upper +endpoint, 0.8977, so even assigning every unresolved candidate to +coverage does not reach 0.90. + +\begin{longtable}[]{@{}lrrrr@{}} +\caption{Five-model coverage-specification audit. \(U_{\max}\) is the +largest all-candidate coverage upper bound across eight windows. +Predictive metrics use 364,814 resolved outcomes; coverage retains all +376,890 candidates. Full hulls are in Online Supplement Table S2. No OOT +metric selects a model.}\label{tbl-credit-controls}\tabularnewline +\toprule\noalign{} +Score & OOT AUC & Brier & Slope & \(U_{\max}\) \\ +\midrule\noalign{} +\endfirsthead +\toprule\noalign{} +Score & OOT AUC & Brier & Slope & \(U_{\max}\) \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +CatBoost & 0.6406 & 0.1299 & 0.7954 & 0.8826 \\ +Logistic & 0.6420 & 0.1288 & 0.5432 & 0.8962 \\ +Monotone CB & 0.6520 & 0.1286 & 0.8351 & 0.8865 \\ +Platform WOE & 0.6331 & 0.1295 & 0.9187 & 0.8949 \\ +Pricing-excluded WOE & 0.6129 & 0.1302 & 0.7098 & 0.8977 \\ +\end{longtable} + +All five scores underpredict later default on average and have +calibration slopes below one. Their AUC, calibration, WOE/IV, and PSI +diagnostics are reported in the Online Supplement as descriptive +controls, not as a model contest or an explanation of the coverage gap. + +Two complete sensitivities narrow the specification concern. The active +sentinel encoding, explicit missing indicators, and native nullable +CatBoost features each have all eight coverage upper bounds below 0.90, +with maxima 0.882597, 0.884332, and 0.880037. At a later retrospective +origin, all eight CatBoost upper bounds are again below 0.90, with +maximum 0.874768. No encoding, model, or origin is selected. These are +three semantics-preserving encoding paths and two retrospective origins, +not evidence about the missingness mechanism, independent replications, +or prospective temporal invariance. + +A separate four-scenario sensitivity addresses 215 labels unavailable at +their own fitting cutoffs: 41 in PD development, 24 in Platt +calibration, and 150 in conformal fitting. It refits the complete +score-and-recipe pipeline after retaining only observed labels, +completing every unavailable label as nondefault, completing every one +as default, or using the later terminal archive. Across all four +scenarios and eight windows, every overall coverage upper bound remains +below 0.90; the scenario maxima range from 0.8826 to 0.884669. No +completion is selected. + +The result is specific to the declared archive and multi-year transport +gap; it does not say that conformal prediction fails under +exchangeability. It occurs before the optimizer selects a funded set and +therefore cannot identify a preferred portfolio rule. + +\subsection{A prevalence crossing coincides with a change in binary +interval +geometry}\label{a-prevalence-crossing-coincides-with-a-change-in-binary-interval-geometry} + +The large W8 width change is concentrated in CatBoost stratum 2. Fit +prevalence declines from 0.1167 in W1 to 0.1017 in W7, then to 0.0971 in +W8. That final change crosses \(\alpha=0.10\). The fitted residual +quantile remains near 0.89 through W7 but falls from 0.8884 to 0.1118 in +W8; mean OOT width in the stratum falls from 0.9843 to 0.2076. Its +discrete intersection simultaneously moves from 0.96\% \(\{0,1\}\) and +99.04\% \(\{0\}\) in W7 to 0\% \(\{0,1\}\), 99.74\% \(\{0\}\), and +0.26\% empty in W8. + +\begin{figure} + +\centering{ + +\includegraphics[width=0.94\linewidth,height=\textheight,keepaspectratio]{../../reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png} + +} +\caption{\label{fig-phase}CatBoost stratum-2 fit prevalence and residual +quantile across all eight windows. The W7--W8 prevalence crossing +coincides with a discontinuous quantile change.} + +\end{figure}% + +This empirical pattern matches the mechanism in Proposition 2, but +scores vary within the stratum and the proposition is not used as a +finite-sample proof. The retrospectively protocol-locked fit-label +sensitivity gives the same threshold crossing at 0, 3, and 6 months; +their minimum monthly retention is 99.71\%, 99.67\%, and 99.30\%, +respectively. At 8 and 12 months the W7 quantile also lies near 0.112, +so the crossing disappears, but those specifications retain only 98.68\% +and 97.45\% in their weakest month and fail the strict greater-than-99\% +rule. The evidence thus supports stability across lags satisfying the +strict retention rule, not a claim that prevalence causally produced the +empirical change. Nor does the narrower W8 interval repair transport: +stratum-2 all-candidate coverage is only {[}0.8225, 0.8547{]}. A +narrower interval can simply expose more rare defaults as misses. + +The fit-label scenario sensitivity separates this explanation from the +coverage result. The observed-only, all-nondefault, and hindsight +scenarios retain the W7--W8 crossing. Under the all-default scenario, +however, W7 and W8 prevalences are 0.105974 and 0.100287 and both +residual quantiles remain near 0.889, so the crossing disappears even +though all eight overall coverage upper bounds remain below 0.90. +Because model and calibration refits are nonlinear, these four scenarios +are not sharp bounds over all \(2^{215}\) label assignments. The phase +mechanism explains one observed path; it is not invariant to every +plausible fit-label completion. + +This is the \textbf{fit-label timing} axis: it changes which 2012 labels +enter conformal fitting and therefore refits the residual recipes. The +\textbf{evaluation-endpoint availability} axis below instead holds +scores, recipes, supports, and allocations fixed while changing which +outcomes are observable at the September 2020 cutoff. The two one-factor +sensitivities were not crossed factorially, so they do not establish +joint lag robustness. + +\subsection{Endpoint direction changes with ruler and +coordinate}\label{endpoint-direction-changes-with-ruler-and-coordinate} + +The two-ruler freeze contains 6,240 portfolios, and the evaluator +reports all 720 monthly and 48 complete-window endpoint contrasts. +Table~\ref{tbl-two-ruler} reports \(\gamma=1\) minus \(\gamma=0\); +\emph{adverse} means lower payoff and higher default/miscoverage, while +\emph{crossing} means the sharp interval contains zero. Each row is one +ruler-coordinate track over the same fifteen OOT menus. The eight +windows are overlapping residual specifications, so the table is a +six-track census, not independent replications or votes. + +\begin{longtable}[]{@{} + >{\raggedright\arraybackslash}p{(\linewidth - 6\tabcolsep) * \real{0.2500}} + >{\raggedright\arraybackslash}p{(\linewidth - 6\tabcolsep) * \real{0.2500}} + >{\raggedright\arraybackslash}p{(\linewidth - 6\tabcolsep) * \real{0.2500}} + >{\raggedright\arraybackslash}p{(\linewidth - 6\tabcolsep) * \real{0.2500}}@{}} +\caption{Six protocol-locked endpoint tracks. Full sharp hulls are in +Online Supplement Table S9.}\label{tbl-two-ruler}\tabularnewline +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Ruler +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.25 +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.50 +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.75 +\end{minipage} \\ +\midrule\noalign{} +\endfirsthead +\toprule\noalign{} +\begin{minipage}[b]{\linewidth}\raggedright +Ruler +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.25 +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.50 +\end{minipage} & \begin{minipage}[b]{\linewidth}\raggedright +.75 +\end{minipage} \\ +\midrule\noalign{} +\endhead +\bottomrule\noalign{} +\endlastfoot +Objective matched & All three cross in 8/8; 4 active months & Adverse in +8/8 & Payoff/default cross in 7/8; miscoverage adverse in 8/8 \\ +Normalized score & Adverse in 8/8 & Adverse in 8/8 & Payoff adverse in +7/8 and crossing in 1/8; default/miscoverage adverse in 8/8 \\ +\end{longtable} + +Under objective matching, coordinate .25 changes only four months. The +allocation contrast is identical across windows to cents: 44 loan-month +positions change and one-way turnover is USD 155,937.27 across USD 15 +million. Under the reconstructed endpoint, however, its status-indexed +payoff-proxy bound is {[}-USD 9,134.34, USD 5,603.66{]}, and its default +and miscoverage bounds are both {[}-0.0068, 0.1265{]} percentage points. +The repeated allocation is therefore one unidentified contrast, not +eight independent confirmations. Proposition 3 reconciles these hulls: +in every window the unresolved exposure disagreement contributes exactly +USD 14,738 of payoff-proxy width and 0.1333 percentage points of default +and miscoverage width. At .50, every sharp window bound gives lower +payoff and higher default and miscoverage. At .75, payoff and default +cross zero in seven windows; only W8 is adverse for both, while +miscoverage is higher in all eight. + +The 12,076 unresolved outcomes have three distinct recorded reasons: +11,551 nonterminal or otherwise unresolved statuses, 47 terminal +outcomes whose reconstructed availability follows the cutoff, and 478 +terminal statuses with no reconstructible availability date. Treating +the latter 525 as one timing category would overstate what the archive +identifies. They remain unrestricted in the same common-outcome bounds; +no missingness mechanism is assumed. + +The normalized ruler gives lower payoff and higher default and +miscoverage in all .25 and .50 cells. At .75, default and miscoverage +remain higher in all eight windows, while payoff is lower in seven and +crosses zero in one. It also gives \(\gamma=1\) between USD 28,263 and +USD 557,294 more optimized plug-in objective than \(\gamma=0\) across +the declared coordinates. The ruler therefore compares the same relative +score relaxation, not the same opportunity cost. Its status-indexed sign +cannot be interpreted as a causal effect or combined with the +objective-matched tracks as evidence for a preferred endpoint. Across +all 48 cells, payoff is lower in 32 and crosses zero in 16; default is +higher in 33 and crosses in 15; miscoverage is higher in 40 and crosses +in 8. + +The complete retrospective endpoint-availability grid changes the +Charged Off administrative lag to 0, 3, 6, 8, and 12 months without +refitting any score, recipe, support, ruler, or allocation. Coverage +upper bounds remain below 0.90 in 40/40 cells at lags 0, 3, 6, and 8, +but in 39/40 at 12 months, where the maximum is 0.900411. No opposite +one-sided portfolio direction appears, and all 216 broad-support +envelopes include zero at every lag. The six-month slice reconciles to +the active evaluation in all 120 coverage cells, 48 two-ruler contrasts, +and 648 exact-support envelopes. Full counts are in Online Supplement +Appendix E. These results make the 40/40 statement +endpoint-contract-specific and do not select the six-month endpoint. + +\subsection{Structural assumptions do not restore a universal +direction}\label{structural-assumptions-do-not-restore-a-universal-direction} + +A separate outcome-free freeze crosses monthly budgets of USD 0.5, 1, +and 2 million, maximum purpose shares of 0.20, 0.25, 0.30, and 1.00, and +LGD values of 0.25, 0.45, and 0.65. The evaluation reports the complete +36-scenario Cartesian product without selecting a scenario. Each +scenario retains both rulers, all three coordinates, and all eight +residual windows, giving 48 guardrail-minus-point cells per metric and +1,728 cells over the structural grid. + +Every scenario contains at least 17 cells with higher default and at +least 21 with higher miscoverage. No scenario makes all three metrics +favorable in all 48 cells, but no scenario makes all three metrics +adverse in all 48 cells either. At least one favorable cell appears in +26 scenarios for payoff and 20 each for default and miscoverage. The +baseline scenario reconciles exactly to the active two-ruler result. +Thus budget, concentration, and LGD change the mix of adverse, +favorable, and unidentified cells, but do not supply a +scenario-independent ordering. Online Supplement Tables S9D--S9E report +the complete census and binding diagnostics. + +The active portfolios are continuous LP allocations. A deterministic USD +25 floor-with-cash diagnostic covers all 1,440 baseline portfolios and +143,175 positive source rows; 2,985 exposures change and monthly +residual cash is USD 25--75. Across the 96 ruler-coordinate-window +endpoint tracks, the largest absolute perturbations are 0.000345 +percentage points in payoff rate, 0.001284 in default rate, and 0.001190 +in miscoverage rate. This supports numerical adequacy of the continuous +relaxation at this granularity for the observed archive. It is not an +integer reoptimization or a claim about other lot rules. + +\subsection{Exact point-cap support checks the finite +diagnostic}\label{exact-point-cap-support-checks-the-finite-diagnostic} + +The supporting C2 exercise reconciles Proposition 1, part 4, in all +1,080 cells: its maximum funded-score residual is \(8.33\times10^{-17}\) +and point-minus-guardrail plug-in objective never falls below numerical +tolerance. Named C0/C1/C2 status-indexed directions nevertheless +disagree; their full census is reported in the Online Supplement rather +than treated as another policy contest. + +The exact point-cap frontier evaluates 3,067 distinct caps. Over +{[}0.05, 0.12{]} broad stress, all 216 policy-by-window-by-metric +envelopes cross zero. The development-admissible support is narrower: +default still crosses zero in all 72 cells; payoff is lower in 6 and +crosses in 66; miscoverage is higher in 27 and crosses in 45. All 27 W8 +envelopes cross zero. + +The evaluated-cap numerical audit covers 7,297 point-cap rows. Although +2,941 bases are primal-degenerate, none has a near-zero nonbasic reduced +cost, and reversing candidate order changes no allocation beyond +\(1.45\times10^{-14}\). This supports deterministic stability at the +evaluated caps; it is not a claim of uniqueness over an unenumerated +continuous joint frontier. + +The exact frontier is not a seventh track. It checks the finite +two-ruler grid against a continuous point-cap support. Together the +analyses show that status-indexed direction depends on residual recipe, +comparator support, ruler, frontier coordinate, and declared portfolio +structure; none identifies a universally safer or more profitable +conformal endpoint. + +\section{Discussion}\label{sec-discussion} + +\subsection{What the audit +establishes}\label{what-the-audit-establishes} + +The guardrail changes a score, feasible region, and allocation; copying +a numeric threshold cannot separate them. Objective matching fixes +model-implied opportunity cost, whereas normalization fixes a +scale-invariant relative score relaxation. Neither is neutral. The +objective-matched result changes from unidentified at .25 to adverse at +.50 and mostly unidentified at .75; the mostly adverse normalized result +compares different plug-in objective levels. Selecting a ruler or +coordinate by its sign would recreate the identification problem. + +C0 proves mechanical same-threshold nesting; C2 removes one funded +point-score moment; and the exact cap frontier shows that neither is a +unique counterfactual. None of the 216 broad-stress envelopes has a +fixed sign. Some payoff and miscoverage signs survive the narrower +development support, but default never does and all three metrics become +unidentified in W8. This exact support result is the safeguard against +overinterpreting a finite grid, not a reason to privilege the broadest +support after outcomes. + +The complete structural grid adds a different safeguard. It shows that +the absence of a universally favorable direction is not an artifact of +one budget, purpose cap, or LGD, while favorable cells under many +scenarios prevent the opposite claim of universal harm. Structural +sensitivity therefore sharpens the identification boundary; it does not +identify a preferred scenario. + +The upper score is pointwise larger and, at a shared cap, mechanically +more restrictive; neither fact supplies OOT coverage or funded-set +validity. \emph{Robust} must name a protected quantity and perturbation +\citep{bertsimas2004, bertsimas2018datadriven}. Here it describes only +the observed coverage failure across complete windows and five learner +specifications, not causal benefit, selected-set validity, or policy +dominance. CRPTO therefore uses \emph{risk-aware} in its expanded name. + +\subsection{Implications for decision-focused data +science}\label{implications-for-decision-focused-data-science} + +Three implications may transfer beyond credit, without establishing +external validity. First, define the decision estimand: name the matched +quantity and binding constraints, and use exact support analysis when LP +structure permits. Second, preserve the decision information set. +Status-independent menus, dated labels, complete eligible windows, and +unresolved-outcome bounds prevent later status from redefining an +earlier choice. Third, separate predictive, geometric, and decision +claims: marginal coverage is not funded-set validity, narrower binary +intervals need not improve decisions, controls are not a leaderboard, +and optimized and evaluated payoffs must agree. + +\subsection{Ethical and governance +implications}\label{ethical-and-governance-implications} + +Calling one credit score \emph{safer} can redirect capital because of a +copied cap rather than better predictive validity. Policy claims should +disclose the comparator, calibration window, binding constraints, and +unresolved-outcome treatment. The accepted-loan archive has neither +rejected-applicant counterfactuals nor a disparate-impact design; CRPTO +audits model-risk claims and does not authorize deployment or +fair-lending conclusions. + +\section{Limitations}\label{sec-limitations} + +One discontinued accepted-loan archive cannot establish live, external, +or fair-lending performance. It is not a verified point-in-time +snapshot, and its terminal labels and status-indexed payoff proxy omit +cash-flow timing, prepayment, recoveries, fees, discounting, and capital +costs. Sixty-month loans, immature cohorts, and 48 late-schema fields +are excluded; disclosed mappings cover delinquency recency and legacy +bankruptcy count. The encoding audit does not identify a missingness +mechanism. + +The learners, overlapping windows, and second origin share one +chronology, so they are neither independent nor prospective +replications. Temporal separation does not restore exchangeability, +candidate coverage does not imply funded-set coverage, and the +constant-score theorem identifies a mechanism rather than the +varying-score empirical path. The four fit-label scenarios are not +exhaustive nonlinear bounds. Likewise, the finite ruler grid, one-moment +C2, and point-cap support are not a continuous joint frontier; the +structural grid is stylized, and USD 25 flooring is not integer +optimization. The inspected archive is neither a pristine lockbox nor a +preregistration. + +\section{Reproducibility}\label{sec-reproducibility} + +One registry binds every active lineage and 31 DVC pointers. Publication +replay verifies frozen artifacts and regenerates 18 tables and three +figures transactionally; scientific replay uses each lineage's protocol +commit and \texttt{uv.lock}, not the current source tree. Online +Supplement Appendix F specifies both modes. + +\section{Conclusion}\label{sec-conclusion} + +CRPTO shows why connecting machine learning, conformal prediction, and +optimization creates an identification problem rather than a +self-validating pipeline. Coverage nontransport persists across the +declared learners and fit-label scenarios, but the observed geometric +crossing is not scenario-invariant. Portfolio direction changes with the +outcome-blind ruler and coordinate, and all 216 broad-support exact +envelopes include zero. + +CRPTO therefore selects no model or policy. Its executable audit keeps +unresolved outcomes, coverage, binary geometry, sharp bounds, and +comparator support distinct: portfolio direction is meaningful only +after its estimand and comparator are specified. + +\clearpage \bibliographystyle{informs2014} -\bibliography{../../book/references} +\bibliography{../references} \end{document} diff --git a/paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md b/paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md index 6888403..c37b3e7 100644 --- a/paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md +++ b/paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md @@ -1,115 +1,85 @@ # IJDS Data and Code Disclosure Form Draft -This is a working draft for completing the official IJDS Data and Code -Disclosure Form in ScholarOne. It is not the official form and should not be -uploaded as a substitute unless the submission system asks for free-form -supporting text. - -Official policy: - - -## Suggested Official Form Position - -Use the official IJDS form, not this draft, for ScholarOne. The closest current -position is: - -- **Code:** released at acceptance, including `src/`, `scripts/`, tests, Quarto - sources, `pyproject.toml`, `uv.lock`, `justfile`, DVC metadata, and - manuscript sources. -- **Data:** partially releasable. Derived tables, figures, status JSONs, - manifest hashes, and DVC pointers can be shared. Raw Lending Club, Prosper, - Freddie/Mendeley and Home Credit files should be obtained from their source - pages or shared only through a journal-approved mechanism if source terms and - file sizes permit. -- **Review-stage access:** anonymized archive or controlled access if editors - request verification before acceptance. - -## Short Disclosure Statement - -The paper is computational and relies on public-source credit-risk datasets, -derived processed artifacts, frozen model outputs, and reproducible code. During -double-anonymous review, the manuscript describes a reproducible companion -package without exposing author-identifying URLs. At acceptance, the author will -release a public reproducibility package containing source code, manuscript -sources, paper table/figure generation commands, frozen artifact metadata, -manifest hashes, an executable drift harness for the prediction-to-decision -certificate chain, and instructions for obtaining or reconstructing the raw data. - -## Data Sources - -| Data source | Role in paper | Disclosure plan | -|---|---|---| -| Lending Club retail-loan data | Main static credit-risk panel and promoted funded-set certificate. | Use `RAW_DATA_SOURCE_NOTES.md`; provide source/acquisition instructions, schema notes, cleaning pipeline, DVC pointers or processed artifacts when redistribution is allowed. | -| Prosper loan data | Frozen external marketplace-loan economic replication. | Use `RAW_DATA_SOURCE_NOTES.md`; provide source notes and generated summary artifacts; do not claim a new exact certificate. | -| Freddie/Mendeley mortgage panel | Frozen external mortgage-credit economic replication. | Use `RAW_DATA_SOURCE_NOTES.md`; provide source notes and generated summary artifacts; do not claim a new exact certificate. | -| Home Credit | Audited but not promoted because it lacks the required economic exposure/return contract. | Mention only as archived/non-promoted source context if needed. | - -## Code Availability - -The accepted-paper package should include: - -- Python package code under `src/`. -- Pipeline and export scripts under `scripts/`. -- Quarto manuscript, supplement, and book sources. -- Tests and guardrails under `tests/`. -- `pyproject.toml`, `uv.lock`, `justfile`, `dvc.yaml`, and `dvc.lock`. -- Commands for regenerating tables, figures, evidence summaries, journal - package files, and local PDF previews. - -## Artifact Availability - -The accepted-paper package should include or point to: - -- `EXTRACTION_MANIFEST.json`. -- Frozen JSON status files under `models/`. -- Paper tables under `reports/crpto/tables/`. -- Paper figures under `reports/crpto/figures/`. -- DVC metadata and remote-access instructions for large processed/model - artifacts when allowed. -- The opt-in drift harness that recomputes conformal intervals and certificate - summaries from frozen PD artifacts and checks zero endpoint drift under the - locked dependency stack. - -## Reproducibility Commands - -```powershell -just setup-base -just smoke -just validate-champion -just tables -just figures -just evidence -just journal-package -just paper-submission -just paper-submission-pdf -``` - -Optional artifact-aware check when DVC access is configured: - -```powershell -uv run dvc status --no-updates -uv run dvc status -c -r dagshub -``` - -## Non-Routine Rerun Boundary - -The following stages are not routine reproduction steps because they would -change the frozen champion or reopen protected search: - -```text -crpto.pd.champion -crpto.conformal.intervals -crpto.conformal.validation -crpto.portfolio.optimization -crpto.portfolio.bound_exact_eval -``` - -Any protected rerun requires a new run tag and drift report. - -## Double-Anonymous Review Note - -For initial review, reviewer-facing files should omit public repository URLs, -author-identifying remote names, local paths, credentials, and acknowledgements. -If editors request reproducibility verification during review, provide an -anonymized archive or controlled-access bundle consistent with the journal's -instructions. +Transfer this content into the official two-page form effective March 5, 2025: +. This file +is a field map, not a substitute for the signed official form. + +## Page 1 + +**Title of manuscript:** CRPTO: An Identification Audit of Binary Conformal +Credit Portfolio Optimization + +**Policy confirmation:** Check the box confirming familiarity with and +agreement to comply with the IJDS Data and Code Disclosure Policy. + +**Legitimate access confirmation:** Check the box confirming legitimate access +to the Lending Club research archive and that the provisions governing its use +do not prohibit this research. Reconfirm this statement against the actual +acquisition source before submission. + +**Data-use ethics:** Select **Yes, considerations should be highlighted** and +enter: + +> The study concerns historical consumer-credit decisions. The archive covers +> accepted loans and does not contain counterfactual outcomes for rejected +> applicants. The analysis does not use protected attributes, authorize +> lending decisions, estimate causal effects, or claim fair-lending compliance. +> Terminal status and a simplified payoff compress payment timing, competing +> events, recoveries, fees, and capital costs. The distributed archive is not a +> verified point-in-time snapshot, so endpoint observability is reconstructed +> under a disclosed conservative timing assumption and unresolved outcomes are +> retained in sharp bounds. These limitations constrain deployment and equity +> interpretations. + +## Page 2 + +**Selected sharing option:** **Option 6**, because all active analysis code and +aggregate evidence can be released, while only a partial data set can be made +public. The author agrees to complete the IJDS reproducibility report. + +**Explanation box:** + +> (a) The accepted-paper package will release the complete active analysis +> code, tests, environment lock, configuration, protocol and claim registries, +> aggregate publication tables and figures, evidence metadata, and scripts for +> reconstructing every result from the source archive. It will also release +> non-row-level schema, census, and audit summaries. (b) The 1.7 GB Lending Club +> CSV and row-level derived score, allocation, and outcome artifacts will not be +> redistributed by the authors. The original file was acquired as a historical +> public research archive, but there is no stable issuer-maintained download and +> redistribution terms for copies and row-level derivatives must be respected. +> The package records the exact raw-file SHA-256, dimensions, schema checks, and +> acquisition/reconstruction instructions so a reader with a lawful copy can +> reproduce the analysis. The distinction is therefore complete code plus +> aggregate evidence, but no raw or row-level loan data. + +## Verified Data Facts + +- Raw identity: `Loan_status_2007-2020Q3.csv`, 2,925,493 rows, 142 columns, + SHA-256 `5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f`. +- Active population: 640,543 eligible 36-month loans under the declared dates, + schema, horizon, and origination-observability rules; no sampling. +- Primary OOT: 376,890 candidates, 364,814 outcomes reconstructed as observable + by September 30, 2020, and 12,076 unresolved outcomes retained in sharp + binary bounds. +- Endpoint reasons: 307,842 fully paid and 56,972 charged off by the cutoff; + 11,551 nonterminal; 47 terminal after the cutoff; and 478 terminal with a + missing availability date. These categories partition the primary panel. +- Endpoint timing: Fully Paid is available at the month-end of `last_pymnt_d`; + Charged Off is available at that month-end plus six calendar months. The + latter is a modeling assumption, not an observed operational charge-off date. + +## Package Boundary + +The release includes `pyproject.toml`, `uv.lock`, active source and claim +registries and executable claim ledger, the complete `src` package, +active experiment/build scripts, 31 DVC +pointers, aggregate evidence, canonical QMD, generated TeX, bibliography, and +the scientific, type, drift, anonymity, compilation, and visual-QA gates. +Credentials, local DVC configuration, absolute paths, protected extraction +artifacts, and copyrighted publisher assets are excluded. + +Recheck the selected option, acquisition rights, repository destination, and +the exact set of releasable data immediately before the official form is +signed. A change in what can legally be shared requires changing Option 6 and +its explanation, not silently changing the release package. diff --git a/paper/submission/EDITOR_ONLY_REPRODUCIBILITY_CROSSWALK.md b/paper/submission/EDITOR_ONLY_REPRODUCIBILITY_CROSSWALK.md new file mode 100644 index 0000000..96ed711 --- /dev/null +++ b/paper/submission/EDITOR_ONLY_REPRODUCIBILITY_CROSSWALK.md @@ -0,0 +1,101 @@ +# Editor-Only Reproducibility Crosswalk + +Do not include this file in the anonymous reviewer archive. It contains +searchable protocol identifiers and immutable provenance. + +## Active Authority + +| Object | Path | +|---|---| +| Claim registry | `docs/research/active_claims_2026-07-14.md` | +| Executable claim ledger | `configs/ijds_claim_ledger.yaml` | +| Source registry | `configs/ijds_active_evidence_sources.yaml` | +| Evidence manifest | `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json` | +| Body | `paper/CRPTO_ijds.qmd` | +| Supplement | `paper/supplement_ijds.qmd` | +| Generated TeX | `paper/submission/CRPTO_ijds_submission.tex` | + +## Immutable Lineages + +| Lineage | Outcome-free freeze | Endpoint-corrected evaluation | +|---|---|---| +| Binary geometry / exact support | `ijds-binary-geometry-frontier-v4-2026-07-12-v1` | `ijds-binary-geometry-frontier-v4-2026-07-15-v5` | +| Protocol tags | `protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1` | `protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5` | +| Protocol commits | `2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd` | `e2bba580a0b07c145bd64ff61440973d6e31349b` | +| Freeze SHA-256 | `c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c` | See source registry and execution receipt | +| Two-ruler diagnostic | `ijds-normalized-objective-frontier-2026-07-13-v1c` | `ijds-normalized-objective-frontier-2026-07-15-v5` | +| Protocol tags | `protocol/ijds-normalized-objective-frontier-2026-07-13-v1c` | `protocol/ijds-normalized-objective-frontier-2026-07-15-v5` | +| Protocol commits | `46f4df915d38eb5a6cc144484c6e6fe56d8ed397` | `e2bba580a0b07c145bd64ff61440973d6e31349b` | +| Freeze SHA-256 | `7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185` | See source registry and verified manifest | +| Credit controls | `ijds-credit-risk-controls-2026-07-13-v1b` | `ijds-credit-risk-controls-2026-07-15-v5` | +| Protocol tags | `protocol/ijds-credit-risk-controls-2026-07-13-v1b` | `protocol/ijds-credit-risk-controls-2026-07-15-v5` | +| Protocol commits | `1776cbf8b201ae5b92756e5ea397a403d6cc7c9f` | `e2bba580a0b07c145bd64ff61440973d6e31349b` | +| Freeze SHA-256 | `da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35` | See source registry and execution receipt | + +The raw-data audit is `ijds-raw-data-contract-2026-07-14-v2`; the reporting-lag +sensitivity is `ijds-label-lag-sensitivity-2026-07-14-v1`; and the evaluated-cap +tie audit is `ijds-policy-support-tie-audit-2026-07-12-v1`. Their descriptors +are hash-locked in `configs/ijds_active_evidence_sources.yaml`. + +The complete evaluation-endpoint availability sensitivity is +`ijds-endpoint-availability-sensitivity-2026-07-14-v1`, protocol-locked at +commit `8865f1cfbd387576bdf805f3e52f030261e4b717`. It reports lags 0, 3, 6, 8, +and 12 without selecting an endpoint; its six-month slice reconciles exactly +to the active evaluations. It is distinct from conformal-fit label timing. + +The complete portfolio-structure sensitivity is +`ijds-portfolio-structure-sensitivity-2026-07-15-v6`, protocol-locked at commit +`490c653a43e2003d83184f47e1277bd2d4390c43`. Its outcome-free freeze reports +all 36 budget--purpose-cap--LGD scenarios; its separate evaluation selects no +scenario and reconciles the baseline exactly to the active evaluation. +Structural V1--V4 remain stopped provenance; V5 is a data-only replay +dependency for V6 and is not paper-facing evidence. + +The later-origin recurrence is +`ijds-rolling-origin-2017-2026-07-15-v4`, protocol-locked at commit +`e2bba580a0b07c145bd64ff61440973d6e31349b`. The missingness recurrence is +`ijds-missingness-sensitivity-2026-07-15-v3`, protocol-locked at commit +`199afb083da37af6a51d5ba9e3c4d6280b952fe9`. Both report their declared grids +without selecting a model, encoding, or origin. + +The fit-label completion sensitivity is +`ijds-fit-label-completion-sensitivity-2026-07-16-v2`, protocol-locked at +commit `fbcafcf84645024b9753aba2f04a4263b8e76236`. The allocation-granularity +sensitivity is `ijds-allocation-granularity-sensitivity-2026-07-16-v3`, +protocol-locked at commit `fb1a7b1837d1f8ab2b81239533f51c996f41671c`. +Both record scientific `uv.lock` SHA-256 +`25cefb168506538c22b86a348c42869ea7fda64338815f2adea3fe7e07608f93`; +their freezes, summaries, and DVC roots are verified by the source registry. + +## DVC Capsule + +The 31 pointers are listed once in `configs/ijds_active_evidence_sources.yaml`. +They comprise data and model pointers for the active roots and sensitivities, +plus the data-only structural replay dependency. Pull with: + +```powershell +uv run python scripts/manage_ijds_dvc_capsule.py pull +``` + +Machine-local DVC credentials belong in `.dvc/config.local` and are never +committed. + +## Replay + +Publication replay uses the current checked-out builder to verify immutable +evidence and regenerate reviewer artifacts. A scientific rerun instead checks +out the protocol commit and environment-lock hash recorded for that lineage. +These are complementary contracts; current source code is not treated as a +substitute for a historical scientific environment. + +```powershell +uv sync --frozen --extra dev +just submission-build +just ijds-active-check +uv run python scripts/manage_ijds_dvc_capsule.py status +``` + +The paper-facing builder fails closed on source-hash, cardinality, endpoint, +selection-boundary, and solver-audit drift. Reviewer surfaces intentionally omit +the identifiers above. Protected historical champion stages and +`EXTRACTION_MANIFEST.json` are not modified or reproduced by this workflow. diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md deleted file mode 100644 index 9e77443..0000000 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ /dev/null @@ -1,70 +0,0 @@ -# IJDS Submission Roadmap - Target 2026-08-10 - -The date is an internal quality gate; IJDS submissions are rolling. - -Official sources to recheck in the submission week: - -- -- -- -- - -## Submission Thesis - -| IJDS dimension | CRPTO answer | -|---|---| -| Data | Temporal Lending Club panel with conformal fit, November selection, December audit, and OOT evaluation. | -| Method | Exact 90% conformal replay, deterministic endpoint cap, and one midpoint portfolio guardrail. | -| Decision | Allocate `$1M` under capital, concentration, and effective-PD constraints. | -| Evidence | Split nine-cell selector/audit, matched point-PD decision, temporal reversals, and month-cluster bootstrap. | -| Implication | An inspectable price of uncertainty, including cases where the static guardrail should be rejected. | - -## Completed Scientific Refactor - -- Retired approximate cross-alpha headline values. -- Replayed conformal quantiles exactly at every sensitivity alpha. -- Selected the conventional 90% reference level; documented endpoint - saturation at tighter levels. -- Replaced nonlinear/tail policy families with `q=(p+u)/2`. -- Separated point-PD economics from conformal feasibility. -- Reduced policy selection to a round-number `3x3` calibration grid. -- Replaced the Markov-based selector screen with deterministic `B_u<=0.28` and - documented the exact cap-stability interval. -- Isolated outcomes from a 12-column selector frame; November selects and an - outcome-free December replay checks policy identity. -- Added the independent December decision audit, including the funded-set - coverage miss, and a 31-month cluster bootstrap. -- Added matched point-PD and 75% blend comparators. -- Promoted temporal reversals and limitations to the body. -- Rebuilt A35--A40 and active claim-sync tests. - -## Remaining Submission Work - -| Window | Deliverable | Exit condition | -|---|---|---| -| Jul 9--12 | Code and claim gates | Ruff, mypy, ty, focused tests, smoke, manifest, and drift gate green. | -| Jul 12--18 | PDF editorial QA | Official body and supplement render; no undefined citations; body within 25-page rule; visual QA complete. | -| Jul 18--24 | Reproducibility archive | Sanitized commands, source notes, run tags, hashes, and A35--A40 bundle staged. | -| Jul 25--31 | Anonymous package | Body, supplement, title page, cover letter, and disclosure form separated correctly. | -| Aug 1--8 | Cold review | Read only the generated PDFs; fix clarity, table, and citation defects. | -| Aug 9--10 | ScholarOne freeze | Upload, inspect ScholarOne proof, and submit only after go/no-go checklist. | - -## Acceptance Risks - -| Risk | Mitigation in current draft | -|---|---| -| Applied pipeline rather than method | One explicit objective/constraint contract and exact selector protocol. | -| Broad binary conformal intervals | A35 reports width and endpoint saturation; no 99% headline. | -| Adaptive funded-set validity | December directly demonstrates the coverage miss; deterministic accounting is separated from conditional Markov language. | -| Historical OOT reuse | "Retrospective lockbox replay" stated in abstract, design, limitations, supplement, and cover letter. | -| Baseline cherry-picking | Same candidates, budget, concentration, LGD, solver, and `tau`; temporal failures are shown. | -| Too many methods | A1--A34 demoted to diagnostics; A35--A40 support one midpoint policy. | -| Reproducibility mistaken for novelty | Decision method and managerial trade-off lead; tooling supports auditability. | -| Page and template risk | Official `informs4` build and visual QA are blocking gates. | - -## Freeze Rule - -After the scientific and PDF gates pass, do not reopen the policy for marginal -OOT gains. Reopen only for a concrete reviewer request, a simpler calibration- -only rule that matches the active result, or a formally stronger prospective or -selection-valid protocol. diff --git a/paper/submission/RAW_DATA_SOURCE_NOTES.md b/paper/submission/RAW_DATA_SOURCE_NOTES.md deleted file mode 100644 index 1c6d9e7..0000000 --- a/paper/submission/RAW_DATA_SOURCE_NOTES.md +++ /dev/null @@ -1,46 +0,0 @@ -# Raw Data Source Notes - -These notes support the IJDS Data and Code Disclosure Form and the accepted-paper -reproducibility package. They are not part of the anonymous reviewer manuscript. - -Last checked for the submission package: 2026-07-06 UTC. - -## Source Inventory - -| Source | Paper role | Public access note | In-package handling | -|---|---|---|---| -| Lending Club Loan Data 2007-2020Q3 | Main static credit-risk panel and promoted Lending Club funded-set certificate. | Kaggle mirrors and Figshare mirrors exist for `Loan_status_2007-2020Q3.csv`; the project uses a local raw CSV under `data/raw/`, which is ignored by Git and tracked through DVC metadata. | Provide acquisition instructions, schema/cleaning code, DVC pointers for processed artifacts when allowed, and manifest hashes. Do not commit or rehost the 1.7 GB raw CSV in Git. | -| Prosper loan-level data | Frozen external marketplace-loan economic replication. | Prosper documents loan-level data access through its investor/API data path: . | Provide source notes, curated summary CSVs, and generated A25-A34 evidence. Do not treat Prosper as a new exact funded-set certificate. | -| Freddie Mac Single-Family Loan-Level Dataset | Source ecosystem for the Freddie/Mendeley mortgage-credit replication. | Freddie Mac documents the Single-Family Loan-Level Dataset at . | Provide source notes and external replication summaries; do not redistribute raw Freddie files unless the journal workflow and source terms permit it. | -| Freddie/Mendeley processed mortgage windows | Processed FM24/FM36/FM48/FM60 windows used for the external replication audit. | Mendeley dataset page used by the paper: . | Provide the exact window definition used in the scripts and the generated A25-A34 artifacts. | -| Home Credit Default Risk | Audited but not promoted. | Kaggle competition data page: . | Mention only as archived context: it lacks the clean `exposure + return` investment contract required for the external economic claim. | - -## Source Permanence Caveat - -The raw Lending Club retail-loan file is a historical public-source dataset, not -a source with an active issuer-maintained permanence guarantee for this paper. -Lending Club ended retail-loan originations after the 2020 window used here, and -the `Loan_status_2007-2020Q3.csv` copies available through Kaggle/Figshare-style -mirrors are community or repository mirrors rather than journal-controlled -archives. The reproducibility package should therefore avoid depending on a -single raw-data URL. It should disclose source pages, schema/cleaning code, DVC -pointers or processed artifacts where source terms permit, and manifest hashes -that let a reviewer verify the exact paper-facing artifact chain. - -## Rebuild Boundary - -The accepted-paper package should let a reader rebuild or audit the manuscript -artifacts without relying on author-local paths: - -1. Obtain raw data from the public source pages above when redistribution is not - allowed. -2. Recreate processed artifacts through the project scripts or retrieve - processed/model artifacts through the declared DVC remote if access is - provided by the journal workflow. -3. Verify frozen outputs with `EXTRACTION_MANIFEST.json` and - `just validate-champion`. -4. Rebuild safe paper surfaces with `just tables`, `just figures`, - `just evidence`, `just journal-package`, and `just paper-submission-pdf`. - -The protected search and champion stages are not routine reproduction steps. -They require a new run tag and explicit revalidation plan. diff --git a/paper/submission/README.md b/paper/submission/README.md index 7a9d1e5..8011075 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -1,133 +1,101 @@ # IJDS Submission Package -This directory contains the official-template handoff and editor-facing -submission materials. The synchronized scientific sources are: - -- `paper/CRPTO_ijds.qmd`: anonymous body. -- `paper/supplement_ijds.qmd`: anonymous online supplement. -- `paper/submission/CRPTO_ijds_submission.tex`: manually compacted - `informs4` handoff. -- `paper/submission/CLAIM_AUDIT_MATRIX.md`: active claim/evidence map. -- `paper/submission/REPRODUCIBILITY_PACKAGE.md`: data/code package plan. - -The active manuscript has one policy: exact 90% conformal replay, -`q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under -`B_u<=0.28`. An outcome-free December replay and post-selection audit are part -of A36; the audit deliberately records that stable policy identity does not -imply selected-set coverage. A35--A40 are the active evidence bundle. Keep -body, supplement, TeX, and governance numerically aligned with -`tests/test_ijds_active_claim_sync.py`. - -## Preview +This directory is the pre-freeze anonymous IJDS handoff. The official TeX is +generated from `paper/CRPTO_ijds.qmd` and must not be edited directly. + +## Authorities + +- body: `paper/CRPTO_ijds.qmd`; +- supplement: `paper/supplement_ijds.qmd`; +- claim registry: `docs/research/active_claims_2026-07-14.md`; +- executable claim ledger: `configs/ijds_claim_ledger.yaml`; +- evidence registry: `configs/ijds_active_evidence_sources.yaml`; +- evidence manifest: `reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json`; +- TeX generator: `scripts/build_ijds_submission_tex.py`; +- compiler: `scripts/compile_ijds_submission.py`. + +The active study scans 2,925,493 raw rows and uses the exhaustive 640,543-loan +eligible design. It retains 376,890 primary OOT candidates, reconstructs +364,814 outcomes as observable by the cutoff, and bounds 12,076 unresolved +outcomes. Five retrospectively protocol-locked learner specifications support the coverage audit; +only CatBoost enters optimization. The decision audit reports five gamma +values, two rulers, three coordinates, exact declared point-cap supports, and +no winner. Evaluation-endpoint lags 0, 3, 6, 8, and 12 are reported without +selection; conformal-fit label timing is a separate sensitivity. An +observed-only fit and three declared stress rules vary the 215 labels that were +unavailable at their fitting cutoffs, without claiming sharp bounds over all +possible assignments. A USD 25 floor-with-residual-cash diagnostic checks the +numerical adequacy of the continuous allocation without claiming integer +optimality. + +## IJDS Requirements + +- IJDS-template PDF; +- at most 25 pages excluding references and appendices; +- separate online supplement; +- double-anonymous review; +- abstract no longer than 300 words; +- 1--10 keywords; and +- data/code disclosure at submission. + +Recheck the official guidelines during submission week. Current links are in +`configs/crpto_publication_targets.yaml`. + +## Build ```powershell -just ijds-evidence -just paper-submission -just paper-submission-pdf -``` - -HTML and browser-print PDFs are writing and visual-QA previews. The upload PDF -must come from `CRPTO_ijds_submission.tex` with the official INFORMS class and -the `dblanonrev` option. - -## Official Sources - -- INFORMS style files: -- IJDS submission guidelines: -- IJDS data/code policy: -- IJDS reviewer guidelines: -- IJDS Overleaf template: - -Local copies of `informs4.cls`, `informs2014.bst`, and related publisher files -are used for compilation and remain gitignored. Recheck official sources during -the final submission week. - -## Official Build - -Run from `paper/submission`: - -```powershell -if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } -latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +just submission-build +just submission-check ``` -`latexmk` is preferred because it automates convergence. The repository build -resolves TinyTeX's `latexmk.pl` and launches it with Perl on Windows, bypassing -the defective `runscript.tlu` executable wrapper. If that payload is unavailable -or fails, the robust fallback is: +The first command writes active evidence and document outputs in causal order; +the second verifies them without replaying scientific evidence. +`build_ijds_submission_tex.py --check` rejects stale generated TeX. +`informs_style_assets.json` pins the local publisher kit. The compiler attempts +`latexmk`; when the Windows TinyTeX wrapper is unavailable it runs: ```text pdflatex -> bibtex -> pdflatex -> pdflatex ``` -```powershell -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -bibtex CRPTO_ijds_submission -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -``` - -The three `pdflatex` calls are intentional: - -1. The first pass writes `.aux`, including citation keys and unresolved labels. -2. BibTeX reads `.aux` and writes the formatted `.bbl`. -3. The second LaTeX pass imports `.bbl` and resolves citations and references. -4. The final pass stabilizes labels, float positions, and pagination changed by - the bibliography and cross-references. - -After a TeX Live update, an `expl3` format mismatch can be repaired once with: - -```powershell -if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } -fmtutil-sys --byfmt pdflatex -``` - -The repository wrapper runs the working `latexmk` payload first and falls back -automatically: - -```powershell -just paper-submission-official -``` - -## Build Acceptance - -The official-template PDF is acceptable only when: - -- `.blg` has no bibliography warnings; -- `.log` has no undefined citations or references; -- the body is at most 25 pages under the IJDS counting rule; -- tables do not overflow or become unreadably small; -- the PDF remains double-anonymous; -- visual inspection confirms that figures, equations, and references render. - -Current verified build (2026-07-09): 13 pages total; References begin on page -11, so the main text occupies pages 1--10 and remains well within the 25-page -limit. The bibliography is clean, and visual inspection of all 13 pages found -no clipping, overlap, missing glyphs, or unreadable tables. - -Do not keep a page-count statement in this README without rebuilding the -current TeX. The final compile wrapper records the current count and warning -scan. - -## Anonymity - -- QMD metadata uses `author: "Anonymous"`. -- TeX uses `\documentclass[ijds,dblanonrev]{informs4}`. -- Author names, acknowledgements, repository ownership, personal URLs, local - usernames, and private remotes stay out of the reviewer packet. -- Cover-letter and data/code language lives in - `COVER_LETTER_AND_DISCLOSURE.md` and related editor-facing files. -- `TITLE_PAGE_DRAFT.md` is uploaded separately when ScholarOne requests it. - -## Final Gate - -```powershell -just submission-check -uv run dvc status --no-updates -git status --short -``` - -Use `SCHOLARONE_FINAL_CHECKLIST.md` for upload order and proof review. Preserve -the active midpoint narrative; OCE/CVaR, SPO+, external replications, and other -historical diagnostics remain supplement context rather than additional active -methods. +The first `pdflatex` creates `.aux`, BibTeX creates `.bbl`, the second LaTeX +pass resolves citations and cross-references, and the final pass stabilizes +labels, floats, and pagination. + +`just paper-pdf-audit` then verifies the three generated reviewer PDFs: Letter +page size, no blank pages, no identity or artifact fingerprints, a one-paragraph +abstract of at most 300 words, and no more than 25 official pages before the +References heading. Visual inspection remains required for clipping, overlap, +and table or figure legibility. + +## QA Record + +The pre-freeze closeout on 2026-07-16 produced the following current record: + +- official PDF pages: `29`; +- pre-reference pages: `25` (references begin on page 26); +- body preview pages: `21`; +- supplement preview pages: `31`; +- abstract words: `267`; +- `.blg` warnings: `0`; +- undefined citations/references: `0`; +- page-level visual inspection: `81/81` pages across the official, body, and + supplement PDFs; transient page renders were discarded after inspection; +- automated PDF inspection: one Letter page size per document, no blank page, + no reviewer identity or artifact fingerprint in text or metadata, and the + IJDS page and abstract limits satisfied. + +## Acceptance Criteria + +- the evidence source registry and manifest verify by hash; +- generated TeX is current with QMD; +- `.blg` has no warnings and `.log` has no undefined citations or labels; +- the pre-reference body is within the IJDS limit; +- all tables and figures are legible and inside margins; +- reviewer files contain no identity, local path, commit, tag, or hash; +- the abstract stays below 300 words; +- no retired endpoint or favorable `.25` claim returns; and +- scientific, drift, publication, compilation, and visual gates pass. + +This is still pre-freeze. Final tagging and ScholarOne proof comparison require +an explicit later freeze decision. diff --git a/paper/submission/REPRODUCIBILITY_PACKAGE.md b/paper/submission/REPRODUCIBILITY_PACKAGE.md index b3570b0..d064359 100644 --- a/paper/submission/REPRODUCIBILITY_PACKAGE.md +++ b/paper/submission/REPRODUCIBILITY_PACKAGE.md @@ -1,129 +1,93 @@ -# IJDS Reproducibility Package Plan +# IJDS Reproducibility Package -Official policy: +This editor-facing plan is the single source for capsule contents, raw-data +instructions, and replay commands. -CRPTO can support accepted-paper reproduction without exposing credentials, -local paths, or author identity during double-anonymous review. +## Release Stages -## Disclosure Timing - -| Stage | Disclose | Withhold | +| Stage | Provide | Exclude | |---|---|---| -| Initial submission | Neutral package description, source-data availability, and release timing. | Repository ownership, personal URLs, local paths, secrets. | -| Editor-requested verification | Anonymized source, A35--A40, tests, and sanitized artifact metadata. | Credentials, private remotes, non-anonymous provenance. | -| Acceptance | Public source, environment lock, data instructions, artifact pointers/hashes, and final outputs. | Secrets and data prohibited from redistribution. | - -## Package Contents +| Initial submission | Anonymous body, supplement, title-page form, disclosure form | Identity and searchable repository metadata in reviewer files | +| Editor verification | Sanitized active capsule and archive-local checksums | Credentials, machine paths, unrelated Git history | +| Acceptance | Code, lock, data instructions, active artifacts, final outputs | Secrets and source files prohibited from redistribution | -| Component | Files | Purpose | -|---|---|---| -| Environment | `pyproject.toml`, `uv.lock`, `justfile`. | Recreate the Windows-first toolchain. | -| Method source | `src/models/conformal_alpha_grid.py`, `src/optimization/`, active experiment scripts. | Replay exact intervals and solve declared policies. | -| Active config | `configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml`. | Fix alpha, split 3x3 selector/audit, endpoint cap, and solver settings. | -| Active evidence | A35--A40 CSV/TeX files and `ijds_policy_governance.json`. | Tie every paper claim to generated evidence. | -| Manuscript | body QMD, supplement QMD, official TeX. | Reproduce reviewer-facing surfaces. | -| Data pointers | `dvc.yaml`, `dvc.lock`, `.dvc/`, raw-data notes. | Retrieve large artifacts where terms permit. | -| Guardrails | active claim sync, publication integrity, manifest regression. | Detect narrative or historical artifact drift. | +## Minimal Active Capsule -## Active Artifact Contract - -| Evidence | Path | +| Component | Contents | |---|---| -| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | -| Calibration selector | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | -| Calibration audit grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_audit_grid.parquet` | -| Calibration decision audit | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_holdout_audit.csv` | -| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | -| Funded rows | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | -| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | -| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | - -Active run: -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. - -Exact-alpha run: -`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. - -The active policy evidence is intentionally separate from the manifest-protected -historical bundle. `tests/test_ijds_active_claim_sync.py` guards the submitted -claim; `tests/test_manifest_regression.py` guards frozen provenance. The -manifest is not rewritten to make a manuscript update look historical. - -## Reproduction Commands - -Paper-facing reproduction from frozen experiment outputs: +| Environment | `pyproject.toml`, `uv.lock`, `justfile` | +| Authority | Active claim registry, executable claim ledger, source registry, publication targets | +| Method | Complete `src` package, active runners, evidence and paper builders | +| Runs | Thirty-one DVC pointers for active roots, sensitivities, and replay dependencies | +| Evidence | One manifest, eighteen tables, three figures in PDF/PNG | +| Manuscript | QMD body/supplement, generated INFORMS TeX, bibliography | +| Gates | Scientific tests, lint, typing, drift, claim sync, anonymity, PDF QA | + +Historical selected-policy, compact-v7, pool93, external-transfer, and A1--A40 +materials are excluded from this capsule. + +## Raw Data Contract + +The active raw source is `Loan_status_2007-2020Q3.csv`, expected size +1,773,470,505 bytes and SHA-256 +`5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f`. +It is ignored by Git and referenced by DVC metadata. Public community or +repository mirrors have existed, but no single issuer-maintained permanent URL +is guaranteed. The package therefore supplies file identity, schema and +cleaning code, full-file audit artifacts, and reconstruction instructions +rather than depending on one URL or rehosting the raw CSV. + +The code scans 2,925,493 rows and 142 columns. The active design uses every +eligible 36-month loan under the declared chronology and schema. The archive is +not a verified point-in-time snapshot; endpoint availability is reconstructed +from servicing dates. Prosper, Freddie/Mendeley, and Home Credit files are +historical diagnostics and are not required. + +## Standard Reproduction ```powershell -just setup-base -just ijds-evidence -uv run pytest tests/test_ijds_active_claim_sync.py -q -just paper-submission -just paper-submission-official -just validate-champion +uv sync --frozen --extra dev +uv run python scripts/manage_ijds_dvc_capsule.py pull +just submission-build +just ijds-active-check +uv run python scripts/manage_ijds_dvc_capsule.py status ``` -Full isolated methodology replay: +The DVC pull requires machine-local credentials. The official PDF additionally +requires the pinned INFORMS style kit in `paper/submission`. -```powershell -just ijds-active-replay -``` +`just ijds-active-check` verifies active evidence without executing protected +historical stages. The maintainer-only submission closeout may validate +historical artifacts already present with `just submission-check`, but it does +not reproduce them. -The full replay recomputes the exact alpha grid, solves the nine policies on -November and December, opens December outcomes only for the independent audit, -evaluates the frozen selected policy, and rebuilds A35--A40. It writes only to -versioned experiment paths and does not overwrite the frozen PD model, -calibrator, historical intervals, or manifest. +## Full Replay Boundary -Official-template compilation is automated by: +All outcome-free roots are immutable. A new methodology replay requires a new +protocol tag, run tag, and fresh output paths. It must retain all declared cells, +may not overwrite an active or historical root, and cannot silently replace the +paper contract after inspecting outcomes. -```powershell -just paper-submission-official -``` - -Manual Windows fallback: +## Official PDF Build ```powershell -cd paper/submission -if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -bibtex CRPTO_ijds_submission -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +just paper-tex +just paper-official ``` -## Data and Artifact Boundary - -- Lending Club, Prosper, and Freddie/Mendeley raw data are distributed through - their original sources, not copied into Git. -- Large processed parquet and model binaries use DVC or journal-approved - artifact delivery when source terms permit. -- `EXTRACTION_MANIFEST.json` verifies the historical upstream bundle. -- Review-stage copies of DVC/configuration metadata must remove repository - ownership, remote URLs, credentials, and absolute local paths. -- If a remote is unavailable, provide journal-approved processed artifacts plus - hashes, schema/source notes, and the commands above. - -## Non-Routine Stages - -The active reproduction does not run protected upstream stages: - -```text -crpto.pd.champion -crpto.conformal.intervals -crpto.conformal.validation -crpto.portfolio.optimization -crpto.portfolio.bound_exact_eval -``` +The compiler attempts `latexmk` and falls back to +`pdflatex -> bibtex -> pdflatex -> pdflatex`. The passes create the auxiliary +graph, bibliography, cross-references, and stable pagination in that order. -Those stages would retrain, rewrite frozen artifacts, or reopen historical -search. Any such run requires a distinct tag and drift report. +## Acceptance QA -## Acceptance Checklist +1. Reproduce from a fresh clone and the 31 DVC pointers. +2. Confirm evidence and QMD-to-TeX builders are byte-idempotent. +3. Run scientific, lint, type, drift, publication, and protected-artifact checks. +4. Compile and inspect body, supplement, and official PDF page by page. +5. Confirm identity, path, tag, commit, and hash sanitization for reviewers. +6. Publish data acquisition, dictionary, environment, and artifact hashes. +7. Document platform-level numerical differences without retuning evidence. -1. Build in the locked `uv` environment. -2. Rebuild A35--A40 and run active claim sync. -3. Validate historical manifest hashes. -4. Compile and visually inspect the official PDF and supplement. -5. Sanitize author identity and local paths in the review archive. -6. Publish source-data acquisition instructions and artifact hashes. -7. Record any unavoidable platform-level numerical differences explicitly. +Exact immutable identifiers live only in +`EDITOR_ONLY_REPRODUCIBILITY_CROSSWALK.md`. diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index d257269..81a5dd4 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -1,79 +1,61 @@ # ScholarOne Final Checklist -Use only after the scientific content and official PDFs are frozen. +Use only after an explicit submission-freeze decision. Until then this is a +living closeout checklist, not evidence that the package is final. + +## Scientific Lock + +- [ ] Active claim registry is `docs/research/active_claims_2026-07-14.md`. +- [ ] Evidence manifest, executable claim ledger, and source registry verify by hash. +- [ ] Census is 376,890 candidates, 364,814 resolved, and 12,076 unresolved. +- [ ] The five endpoint reasons sum exactly to the candidate, resolved, and unresolved totals. +- [ ] All 40 five-model coverage upper bounds are below 0.90; largest 0.897726. +- [ ] Objective-matched .25 is described as crossing zero, never as favorable. +- [ ] Two-ruler, exact-support, fit-label-lag, endpoint-availability, and solver-stability limits are stated. +- [ ] Endpoint lags 0/3/6/8/12 are complete, unselected, and the six-month slice reconciles to the active evaluation. +- [ ] Fit-label and evaluation-endpoint timing are not described as a joint factorial sensitivity. +- [ ] Missingness encodings and the second origin are bounded recurrences, not winners or independent validation. +- [ ] Identification-width statements match the exact unresolved-row identity and the six reported tracks. +- [ ] No learner, window, gamma, ruler, coordinate, cap, comparator, or policy is selected. +- [ ] No selected-set, causal, prospective, confirmatory, deployment, Markov, + point-in-time-snapshot, cash-flow-return, or fair-lending claim appears. ## Files -| File | Reviewer-facing | Local status | -|---|:---:|---| -| Anonymous manuscript PDF from `CRPTO_ijds_submission.tex` | Yes | Rebuild and recheck after every body edit. | -| Anonymous supplement PDF | Yes | Render and visually inspect. | -| Separate title page | No | Complete from `TITLE_PAGE_DRAFT.md`. | -| Data and Code Disclosure Form | Editor/system | Finalize from the draft. | -| Cover letter | Editor | Finalize from `COVER_LETTER_AND_DISCLOSURE.md`. | -| Reproducibility note/archive | Editor/system | Sanitize identity, paths, and remotes. | - -## Official Build - -```powershell -just paper-submission-official -``` - -The wrapper uses the direct `latexmk.pl` payload on Windows and falls back to the verified -`pdflatex -> bibtex -> pdflatex -> pdflatex` loop. Accept only when: - -- `.blg` has zero warnings; -- `.log` has no undefined citation/reference warnings; -- body page count satisfies the IJDS 25-page rule; -- figures and tables fit; -- PDF metadata and visible content remain anonymous. - -Current local build (2026-07-09): 13 pages total, with References beginning on -page 11; citation/reference scans are clean. Recount after every substantive -TeX edit. - -## Local Gates - -```powershell -just ijds-evidence -uv run pytest tests/test_ijds_active_claim_sync.py -q -just publication-integrity -just lint -just type-check -just type-advisory-full -just smoke -just validate-champion -just paper-submission -just paper-submission-official -uv run dvc status --no-updates -``` - -`dvc status` is a report, not permission to rerun protected stages. Do not -repair paper-stage drift by overwriting the frozen upstream chain. - -## Anonymous PDF QA - -- No author names, affiliations, acknowledgements, repository ownership, - personal URLs, local usernames, or private remotes. -- Correct title, abstract, keywords, section order, and supplement designation. -- References use the official INFORMS bibliography style. -- No missing glyphs, clipped figures, overflow tables, orphan headings, or - unreadably small text. -- Active numbers match A35--A40 and governance. -- Temporal reversals and retrospective-design caveat remain visible. -- OCE/CVaR, SPO+, and external datasets remain diagnostics, not active methods. - -## ScholarOne Proof Go/No-Go - -Open the ScholarOne-generated proof, not only the local files. Submission is -**NO-GO** if any of these occur: - -- title page or author identity leaks into reviewer files; -- body/supplement order is wrong; -- figure, equation, table, or bibliography is missing or clipped; -- data/code answers differ from the cover letter; -- page count or anonymous-review option is wrong; -- uploaded PDF differs from the locally validated build. - -Repair locally, rerun the gates, re-upload, and inspect the new proof before -final submission. +- [ ] Anonymous official IJDS PDF. +- [ ] Anonymous online supplement PDF. +- [ ] Separate title-page metadata form. +- [ ] Optional cover letter and generative-AI disclosure, if uploaded. +- [ ] Data and code disclosure form. +- [ ] Editor-only reproducibility crosswalk, if requested. +- [ ] Sanitized reproducibility capsule, if requested. + +## Build and Numerical QA + +- [ ] `just ijds-active-check` passes. +- [ ] `just validate-champion-strict`, `just type-check`, and + `just type-check-fast` pass. +- [ ] Full active tests, Ruff, mypy, and ty pass. +- [ ] Generated TeX is current with QMD. +- [ ] `.blg` has no warnings. +- [ ] `.log` has no undefined citations, labels, or rerun requests. +- [ ] Pre-reference body is within the IJDS page limit. +- [ ] Abstract is at most 300 words and keywords are within 1--10. +- [ ] Every official, body-preview, and supplement page is visually inspected. +- [ ] No clipping, overlap, blank page, broken table, missing glyph, or tiny figure text. + +## Anonymity and Availability + +- [ ] Reviewer files contain no author name, email, local path, repository URL, + protocol tag, commit, hash, DVC coordinate, or acknowledgments. +- [ ] Public/searchable code does not create an identity leak in reviewer files. +- [ ] Raw-data acquisition and hash instructions are accurate. +- [ ] No secrets or `.dvc/config.local` contents are included. + +## Final ScholarOne Proof + +- [ ] Submitting-author ORCID iD is present in ScholarOne. +- [ ] Compare uploaded proof page by page with the validated local PDFs. +- [ ] Confirm title, abstract, keywords, equations, references, and supplement links. +- [ ] Record final page counts and artifact hashes. +- [ ] Create the immutable submission tag only after user approval. diff --git a/paper/submission/TITLE_PAGE_DRAFT.md b/paper/submission/TITLE_PAGE_DRAFT.md index eacede6..a2bfdd0 100644 --- a/paper/submission/TITLE_PAGE_DRAFT.md +++ b/paper/submission/TITLE_PAGE_DRAFT.md @@ -1,63 +1,55 @@ # IJDS Title Page Draft -**Do not include this file in the anonymous reviewer packet.** It contains -author identity and contact fields by design and belongs only in the separate -ScholarOne/INFORMS title-page surface. +Upload separately from the double-anonymous manuscript. -Upload it only as the separate title page requested by ScholarOne/INFORMS. +## Title -## Manuscript Title +CRPTO: An Identification Audit of Binary Conformal Credit Portfolio Optimization -CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios +## Running Title + +Binary Conformal Portfolio Audit ## Keywords -conformal prediction; predict-then-optimize; credit risk; portfolio -optimization; calibration; reproducible data science +conformal prediction; predict-then-optimize; comparator design; credit risk; +portfolio selection; temporal transport; partial identification ## Author -Carlos Alfredo Vergara Rojas +**Author input required in the private ScholarOne copy.** Do not place author +identity in this tracked template. ## Affiliation -Not listed in this repository. Complete in ScholarOne before upload, using the -current institutional affiliation exactly as it should appear in the submission -system. +**Author input required:** enter the current institutional affiliation exactly +as it should appear in the publication record. Do not infer it from an old +profile or thesis record. ## Corresponding Author -Carlos Alfredo Vergara Rojas - -Email: cavr94@gmail.com - -ORCID: Not listed in this repository; complete in ScholarOne if applicable. - -## Acknowledgements - -None for the initial double-anonymous submission. Any acknowledgements should -remain outside the anonymous manuscript and supplement. - -## Funding - -The author received no specific funding for this work. - -## Conflicts of Interest - -The author declares no conflicts of interest. - -## Data and Code Disclosure Summary - -The manuscript includes numerical and computational work. The author will -complete the IJDS Data and Code Disclosure Form at submission and, if the paper -is accepted, provide the code, manuscript sources, reproducibility commands, -artifact metadata, and data-access instructions described in -`REPRODUCIBILITY_PACKAGE.md` and `RAW_DATA_SOURCE_NOTES.md`. - -## Double-Anonymous Handling - -- Do not include this file in reviewer-facing materials. -- Keep public repository, DVC, MLflow, DagsHub, personal, affiliation, and - acknowledgement details out of `CRPTO_ijds_submission.tex`, - `CRPTO_ijds.qmd`, and `supplement_ijds.qmd`. -- Use third-person citations for any author-identifying prior work, if added. +**Author input required in the private ScholarOne copy:** full name, current +email, postal address if requested, and ORCID. The submitting author must +provide an ORCID under the current IJDS instructions. + +## Declarations + +- Funding: The author received no specific funding for this work. +- Conflicts of interest: The author declares no conflicts of interest. +- Acknowledgements: None in the initial submission unless a non-anonymous + acknowledgement is required by the editor. +- Data and code: The official IJDS disclosure form will describe the staged + release of source code, environment locks, DVC metadata, derived evidence, + and raw-data acquisition instructions. +- Generative AI: OpenAI Codex assisted with code review and refactoring, test + scaffolding, literature triage and summarization, + consistency checks, and language editing. It was not treated as evidence or + an author. The author verified all primary sources, analyses, citations, + numbers, code, claims, and prose and accepts full responsibility. + +## Handling + +Create `TITLE_PAGE_PRIVATE.md` locally for the completed upload; that filename +is ignored by Git. Do not merge the private title page into the reviewer-facing +PDF. Reconcile title, running title, keywords, author spelling, email, ORCID, +and declarations against the final ScholarOne proof before submission. diff --git a/paper/submission/informs-pandoc-template.tex b/paper/submission/informs-pandoc-template.tex new file mode 100644 index 0000000..940f59e --- /dev/null +++ b/paper/submission/informs-pandoc-template.tex @@ -0,0 +1,84 @@ +%% Generated manuscript template. Edit paper/CRPTO_ijds.qmd, not the output TeX. +\documentclass[ijds,dblanonrev]{informs4} + +\usepackage{eqndefns-left} +\RequirePackage{tgtermes} +\RequirePackage{newtxtext} +\RequirePackage{newtxmath} +\RequirePackage{bm} +\RequirePackage{endnotes} +\OneAndAHalfSpacedXII +\TheoremsNumberedThrough +\EquationsNumberedThrough +\ECRepeatTheorems +\MANUSCRIPTNO{} + +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{amsmath,amssymb,mathtools} +\usepackage{booktabs,longtable,array,calc} +\usepackage{graphicx} +\usepackage{placeins} +\usepackage{float} +\usepackage{tabularx} +\usepackage{xcolor} +\usepackage{hyperref} +\hypersetup{hidelinks} +\graphicspath{{../../reports/crpto/figures/}} +\floatplacement{figure}{H} +\setlength{\emergencystretch}{2em} +\usepackage{natbib} +\bibpunct[, ]{(}{)}{,}{a}{}{,} +\def\bibfont{\small} +\def\bibsep{\smallskipamount} +\def\bibhang{24pt} +\def\newblock{\ } +\def\BIBand{and} + +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\providecommand{\passthrough}[1]{#1} +\newsavebox\pandocbox +\newcommand*\pandocbounded[1]{% + \sbox\pandocbox{#1}% + \ifdim\wd\pandocbox>\linewidth + \resizebox{\linewidth}{!}{\usebox\pandocbox}% + \else + \usebox\pandocbox + \fi} + +$if(highlighting-macros)$ +$highlighting-macros$ +$endif$ + +\begin{document} + +\RUNAUTHOR{Anonymous} +\RUNTITLE{Binary Conformal Geometry Audit} + +\TITLE{$title$} + +\ARTICLEAUTHORS{% + \AUTHOR{} + \AFF{} +} + +$if(abstract)$ +\ABSTRACT{% +$abstract$% +} +$endif$ + +$if(keywords)$ +\KEYWORDS{$for(keywords)$$keywords$$sep$; $endfor$} +$endif$ + +\maketitle + +$body$ + +\clearpage +\bibliographystyle{informs2014} +\bibliography{../references} + +\end{document} diff --git a/paper/submission/informs_style_assets.json b/paper/submission/informs_style_assets.json new file mode 100644 index 0000000..5d476eb --- /dev/null +++ b/paper/submission/informs_style_assets.json @@ -0,0 +1,24 @@ +{ + "schema_version": "1.0", + "source_url": "https://pubsonline.informs.org/authorportal/latex-style-files", + "verified_on": "2026-07-11", + "informs4_class_version": "2024/06/03 v1.02", + "assets": { + "eqndefns-left.sty": { + "bytes": 16576, + "sha256": "98914dc259f4bfab181631123f0fba994d888212f174e322ba267c8c01035b25" + }, + "informs2014.bst": { + "bytes": 31531, + "sha256": "73c34c8f36abd9088a9b7462e4601419360cdc9afaa227a98cd07f4e8a60c366" + }, + "informs4.cls": { + "bytes": 101135, + "sha256": "82bc064ab2540b5bd11ad8804330fcd9ffc5b420fe6ebe9dfa9ab6b29f8349aa" + }, + "informs_Logo.pdf": { + "bytes": 14342, + "sha256": "fb3afb44b470f0d54463e894a990d2a375cd67a70997d98c1dd35a1df094aa0c" + } + } +} diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 35aebb3..1bff238 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -1,401 +1,1284 @@ --- -title: "Online Supplement for CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" +title: "Online Supplement for CRPTO: An Identification Audit of Binary Conformal Credit Portfolio Optimization" author: "Anonymous" -date: today lang: en -bibliography: ../book/references.bib -csl: ../book/apa.csl +bibliography: references.bib +csl: apa.csl link-citations: true format: html: toc: true - number-sections: true + number-sections: false css: ijds.css pdf: documentclass: article - number-sections: true + number-sections: false geometry: - - margin=1in + - margin=0.85in execute: freeze: auto echo: false warning: false --- -# Scope and Evidence Map +# Scope and Evidence Contract + +This supplement documents one retrospective CRPTO specification audit. The +archive had been inspected during earlier project iterations. The active +protocol was committed before its complete run, but it is not a prospective +trial, preregistration, or pristine lockbox. The inferential object is the +finite Lending Club archive under the declared information, outcome, policy, +and comparator contracts. + +The audit reports every eligible consecutive six-month residual window and five +protocol-locked coverage specifications. The primary detailed audit retains CatBoost and +numeric logistic, while monotonic CatBoost and two WOE/IV scorecards broaden +model-class coverage checks; only the primary CatBoost enters optimization. The +missingness-encoding and second-origin families are complete, bounded +sensitivities rather than selection grids. The +decision audit reports the complete five-point gamma path, two rulers constructed +without policy-development or OOT evaluation outcomes, three interior coordinates, +and every comparator in each named support. *Evaluation-outcome-blind* means that +earlier model and conformal fitting may use historical labels, while policy +construction reads neither development nor OOT evaluation outcomes. The +nine fixed-cap policies remain a supporting exact-frontier diagnostic, not a +closed family of promotion candidates. No out-of-time (OOT) outcome selects a +learner, window, taxonomy, gamma, ruler, coordinate, policy, cap, or comparator. +Nested supports, overlapping windows, and repeated allocations are +specification paths, not independent replications or votes. + +| Reader question | Location | Audit answer | +|---|---|---| +| Which loans could enter a decision? | Appendix A | Status-independent monthly menus retain unresolved outcomes | +| What does the conformal object predict? | Appendix B | The observed binary terminal label, not latent individual PD | +| Why can interval geometry change abruptly? | Appendix B | The binary residual order statistic changes around prevalence $\pi=\alpha$ | +| What is held fixed in portfolio comparisons? | Appendix C | Menu, budget, payoff, loan bounds, purpose cap, solver, and one declared ruler | +| What is exact? | Appendix D | Score-ruler algebra, C2 feasibility, binary-outcome bounds, and basis-frontier envelopes | +| Which empirical signs survive? | Appendix E | The endpoint sign changes by ruler/coordinate and no broad-support sign survives | +| Can the evidence be replayed? | Appendix F | Yes, from the hash-linked active source registry and full raw-data audit | +| What may the paper claim? | Appendix G | An identification audit, not superiority, causality, or selected-set validity | +| How does CRPTO differ from its closest methods? | Appendix H | By its frozen-score audit target, not by claiming each component as new | + +: Reader map for the Online Supplement. {#tbl-s-reader-map} + +# Appendix A: Data, Timing, and Outcome Observability + +## A.1 Single status-independent universe + + + + +The source is the Lending Club 2007--2020Q3 public research archive. A +chunked full-file audit reads all 2,925,493 rows and 142 columns; 2,925,492 are +valid dated loans. The archive contains 2,060,077 36-month and 865,415 60-month +contracts. The single active design retains all 640,543 loans eligible under +deterministic schema, date, term, and origination-feature checks. It is an +exhaustive design population, not a sample or row cap. Candidate membership +never depends on whether the reconstructed endpoint is resolved. + +The decision unit is issue month. Every OOT month receives a fresh USD 1 +million budget, and a policy can allocate only to loans issued in that month. +The primary panel contains 376,890 candidates in 15 menus from April 2016 +through June 2017. A July--September 2017 extension contains 88,227 candidates +and is retained only as a more heavily censored diagnostic. + +| Block | Issue dates | Rows | Labels available by cutoff | Role | +|---|---:|---:|---:|---| +| PD development | 2007-06--2010-12 | 17,433 | 17,392 | model fit and temporal validation | +| Platt calibration and taxonomy | 2011-01--2011-12 | 14,101 | 14,077 | separately fitted score map and fixed edges | +| Residual pool | 2012-01--2013-01 | 49,007 | 48,857 | all eight six-month recipes | +| Policy development | 2013-02--2013-12 | 94,885 | not read | outcome-free comparator support | +| Primary OOT | 2016-04--2017-06 | 376,890 | read only after freeze | complete locked evaluation | +| Censored extension | 2017-07--2017-09 | 88,227 | read only after freeze | secondary censoring stress | + +: S1 chronology and label observability. Blocks do not overlap except that the eight residual windows overlap by design. {#tbl-s-chronology} + +The information cutoff for model and residual labels is March 31, 2016. The +evaluation endpoint cutoff is September 30, 2020. The distributed file is not +a verified point-in-time snapshot: 36,485 last-payment dates and 40,214 +last-credit-pull dates occur after that cutoff. A charged-off residual +label is treated as available at the month-end of `last_pymnt_d` plus six +calendar months; every included residual month retains more than 99% of +terminal labels under this +rule. The minimum monthly retention is 0.992956. + +The complete audit explains why the remaining archive is not exchangeable +extra training data. Forty-eight variables have negligible early support and +near-complete later support, so admitting them would change the temporal +feature contract. Label availability by the cutoff is 59,910/162,570 (36.85%) +for 2014, 28,878/283,173 (10.20%) for 2015, and 1,110/96,120 (1.15%) for +2016Q1, with zero observed bads in the last group. A resolved-only fit would +therefore select on duration. Sixty-month contracts define another horizon; +using censored cohorts coherently would require a survival or competing-risk +estimand rather than a larger binary sample. + +For primary OOT, requested and funded amount differ by only USD 18,000 across +376,890 loans. The funded-to-requested ratio is 0.999996 and only about two +loans are partially funded. The optimizer nevertheless names listed loan +amount as its exposure contract rather than treating the two fields as +interchangeable by assumption. + +## A.2 Endpoint taxonomy and censoring + + + +Statuses containing `Charged Off` are positive ($Y=1$), and statuses containing +`Fully Paid` are negative ($Y=0$), only when reconstructed endpoint availability +is no later than the cutoff. Exact `Default`, every nonterminal status, and +terminal statuses becoming available later remain unresolved. The primary +panel has 364,814 resolved and 12,076 unresolved candidates. Fully Paid is +available at the month-end of `last_pymnt_d`; Charged Off is available at that +month-end plus six calendar months. The charge-off delay is a modeling +assumption, not the known operational charge-off date. The exact reason census +is: + +| Reconstructed endpoint reason | Candidates | Resolved | Unresolved | +|---|---:|---:|---:| +| Fully Paid by cutoff | 307,842 | 307,842 | 0 | +| Charged Off by cutoff | 56,972 | 56,972 | 0 | +| Nonterminal or otherwise unresolved | 11,551 | 0 | 11,551 | +| Terminal after reconstructed cutoff | 47 | 0 | 47 | +| Terminal availability date missing | 478 | 0 | 478 | + +: Endpoint-reason partition of all 376,890 primary candidates. The last two rows sum to 525 but represent different observability failures. {#tbl-s-endpoint-reasons} + +This endpoint is a cutoff-specific terminal classification, not a lifetime hazard. +The analysis does not assume that unresolved outcomes are missing at random. +Instead, allocations are frozen first and each unresolved binary outcome is +allowed to take either value in sharp fixed-allocation and paired-policy +bounds. This preserves the historical menu while exposing the uncertainty +that outcome filtering would conceal. + +## A.3 Physical information boundary + +Prediction, residual-recipe construction, policy development, comparator +support, point-frontier enumeration, and allocation freezing run without OOT +status, status-indexed payoff, miscoverage, or another outcome-derived field. The +outcome-free phase persists 51,117 solve records and 5,001,617 funded rows. It +contains 1,872 guardrail solves: eight windows by nine policies by 26 monthly +menus, of which 11 are development and 15 are primary OOT. It also contains +1,080 OOT C2 solves: eight windows by nine policies by 15 months. + +The separate two-ruler freeze contains 6,240 solves, 622,455 funded rows, 720 +endpoint comparisons, 1,440 reversed-ID endpoint reruns, 288 separate GLOP +checks, and 26 objective-optimum basis diagnostics. Its dimensions are eight +windows by 26 months by five gammas by three coordinates by two rulers. It +loads no outcome column. + +The separate credit-control freeze contains five score vectors for all 640,543 +eligible rows, 160 residual recipes, 3,638,660 fit-audit rows, and 3,520 +outcome-free geometry cells. It reproduces the CatBoost and numeric-logistic +score columns used in the detailed geometry audit exactly, and freezes +monotonic CatBoost plus both WOE/IV scorecards before any primary OOT outcome +join. No control enters a portfolio solve. + +Each evaluator verifies its freeze identity and every artifact descriptor +before a keyed outcome join. Exact ID alignment is required; partial or +duplicated joins fail. The resulting evidence therefore separates allocation +construction from outcomes and prevents an outcome-dependent interpretation +from changing the frozen decision rule. + +# Appendix B: Prediction and Binary Conformal Geometry + +## B.1 Primary learner and four protocol-locked coverage controls + +The primary score is a CatBoost classifier with seed 42, 29 numeric and nine +categorical origination-time features. A Platt map fitted on 2011 raw margins +produces $p_i$. Five score strata, fixed from all status-independent 2011 +scores, form the canonical Mondrian taxonomy. One-, two-, and ten-stratum +recipes are closed coverage diagnostics rather than a selection grid. + +A numeric logistic regression provides the first coverage-only +control. It uses median imputation, standard scaling, class balancing, and its +own 2011 Platt map and fixed taxonomy. Three further controls use the same +temporal roles and all eligible rows: + +- **Monotonic CatBoost.** Positive constraints apply to interest rate, DTI, + loan-to-income and its square, installment burden, revolving utilization, + revolving-balance-to-income, public records, recent inquiries, delinquency + severity, and five adverse-history flags. Negative constraints apply to + annual and log income, FICO, credit age, and delinquency recency. The inherited + `recent_chargeoff` flag is constant in PD development and Platt calibration, + so its declared positive constraint is inert in fitting. +- **Platform-signal WOE/IV.** Twenty-six fields include loan amount, interest, + installment, income and burden ratios, revolving and bureau measures, grade, + subgrade, home ownership, purpose, verification, interest bucket, and the + interest-bucket-by-grade interaction. +- **Pricing-excluded application WOE/IV.** Nineteen fields retain loan amount, income, DTI, + loan-to-income, revolving utilization and balances, FICO, credit age, + employment length, account counts, public records, inquiries, delinquency, + home ownership, purpose, and verification. Grade, subgrade, interest, + installment, pricing buckets, and their interaction are excluded. + +Each OptBinning process is fit on PD development only, with automatic +monotonic trend, two to eight bins, and a 5% minimum bin share +[@navaspalencia2020]. Each learner then receives its own 2011 Platt map and +taxonomy. None of the four controls enters the portfolio optimizer or is chosen +from OOT outcomes. They ask whether coverage transport is specific to model +class, unconstrained shape, or platform pricing; they are not a model-selection +contest. + +The raw audit labels 30 origination fields as temporally eligible, whereas the +frozen model consumes engineered predictors rather than every eligible raw +column. Four eligible fields are deliberately absent. `initial_list_status` +and `policy_code` are constant in PD development. `acc_now_delinq` and +`delinq_amnt` have only a handful of nonzero early observations and are not +added beside the more stable delinquency history and severity features. This +is a prespecified representation choice, not a claim that the omitted fields +are unavailable or useless in another design. + +The raw-feature contract records two exceptions to the 95% coverage threshold. +`mths_since_last_delinq` is structurally nullable: minimum fitting coverage is +0.336075 and primary OOT coverage is 0.528555; the frozen transformation maps +missing recency to 999. `pub_rec_bankruptcies` has minimum fitting coverage +0.921700 and complete primary OOT coverage; missing values map to no recorded +bankruptcy. Feature engineering emits observed-value indicators for audit, but +the active portfolio learner does not add them post hoc. Appendix B.7 reports +a dedicated feature-semantics-preserving encoding sensitivity. It does not +identify a missingness mechanism or select an encoding. + +## B.2 Credit-control prediction, IV, and temporal shift + +All five scores use the same 376,890 primary candidates, of which 364,814 are +resolved and 12,076 are unresolved. All 30 model-by-role calibration +intercept/slope fits converge in 8--16 iterations. Primary OOT default prevalence +is 0.156167, versus 0.104781 on the 2011 Platt block and 0.196438 among resolved +extension loans. + +| Learner | OOT AUC | Brier | ECE10 | Mean cal. error | Slope | Min lower | Max upper | Upper $<.90$ | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| CatBoost | 0.640605 | 0.129878 | 0.047215 | -0.047109 | 0.795427 | 0.842485 | 0.882597 | 8 | +| Numeric logistic | 0.642045 | 0.128846 | 0.031149 | -0.028923 | 0.543210 | 0.850031 | 0.896222 | 8 | +| Monotonic CatBoost | 0.651954 | 0.128613 | 0.041563 | -0.041264 | 0.835149 | 0.848396 | 0.886489 | 8 | +| Platform-signal WOE scorecard | 0.633066 | 0.129485 | 0.035968 | -0.035510 | 0.918655 | 0.848908 | 0.894908 | 8 | +| Pricing-excluded application WOE scorecard | 0.612939 | 0.130190 | 0.031769 | -0.029752 | 0.709836 | 0.852013 | 0.897726 | 8 | + +: S2 five-model primary OOT prediction and coverage audit. Min lower and max upper form the across-window all-candidate coverage hull, not a confidence interval. {#tbl-s-credit-controls} + +Every mean calibration error is negative and every slope is below one, +so all specifications underpredict later default on average and change +calibration shape. Monotonic CatBoost has descriptively higher OOT AUC than the +active score by 0.011348 and lower Brier by 0.001265. Platform WOE exceeds the +pricing-excluded application scorecard by 0.020127 AUC and has 0.000705 lower +Brier. These complete reported +contrasts diagnose model and platform-signal dependence; because they use the +audited OOT outcomes, they cannot promote a learner. + +All 45 OptBinning problems report `OPTIMAL`. The most informative platform +fields are incumbent pricing and grade signals, whereas FICO and inquiries lead +the pricing-excluded application scorecard: + +| Scorecard | Feature | IV | Primary OOT feature PSI | +|---|---|---:|---:| +| Platform | int_rate_bucket__grade | 0.337569 | 0.962804 | +| Platform | sub_grade | 0.319325 | 0.104275 | +| Platform | grade | 0.299544 | 0.093703 | +| Platform | int_rate | 0.278429 | 0.180056 | +| Platform | int_rate_bucket | 0.244858 | 0.146043 | +| Pricing excluded | fico_score | 0.213574 | 0.227297 | +| Pricing excluded | inq_last_6mths | 0.170864 | 0.390443 | +| Pricing excluded | purpose | 0.088878 | 0.187985 | +| Pricing excluded | rev_utilization | 0.073903 | 0.086944 | +| Pricing excluded | delinq_recency | 0.049288 | 0.130864 | + +: S3 leading development IV values and development-to-primary-OOT feature PSI. IV ranks the frozen development bins; PSI is descriptive shift, not a validity test. {#tbl-s-woe-iv-psi} + +| Score | Development-to-primary-OOT PSI | +|---|---:| +| CatBoost | 0.139757 | +| Numeric logistic | 0.005168 | +| Monotonic CatBoost | 0.093706 | +| Platform-signal WOE | 0.148867 | +| Pricing-excluded application WOE | 0.072332 | + +: S4 score PSI under common development-decile bins. {#tbl-s-score-psi} + +The largest feature PSI is 0.962804 for the price-grade interaction, followed +by verification status (0.608133), DTI (0.552263), and recent inquiries +(0.390443). Yet numeric logistic has score PSI 0.005168 and still excludes 0.90 +coverage in all eight windows. PSI can reveal marginal shift, but neither low +nor high PSI is a conformal transport certificate. + +## B.3 Complete residual-window specification + + + +For each learner, window, and fixed score stratum $g$, define residuals -This supplement supports one active method: an exact 90% conformal replay, -the midpoint decision score $q=(p+u)/2$, risk tolerance $\tau=0.17$, and a -nine-cell calibration selector. A35--A40 are the active evidence bundle. -Earlier A1--A34 analyses remain useful as model, robustness, comparison, or -external-transfer diagnostics, but none selects or redefines the submitted -policy. +$$ +R_i=|Y_i-p_i|. +$$ -| Evidence | Role in the submission | Claim boundary | -|---|---|---| -| A35 | Exact alpha replay, coverage, width, and endpoint saturation. | Other alpha rows are sensitivities, not additional policies. | -| A36 | All nine calibration selector cells. | No OOT outcome-derived ranking columns. | -| A37 | Full-OOT and fixed-policy temporal evaluation. | Retrospective stress evidence, not prospective performance. | -| A38 | Funded exposure and outcomes by letter grade. | Business composition, not legal fair-lending certification. | -| A39 | Fixed-allocation month-cluster bootstrap plus loan-level sensitivity. | Does not resample model, conformal recipe, selector, or solver. | -| A40 | Selected, more-conservative, and matched point-PD decisions. | Retrospective contrasts, not causal effects or universal dominance. | +At target miscoverage $\alpha=0.10$, the finite-sample split rank is -The active governance file is -`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. -The final tagged ranking code does not read OOT defaults or returns. Earlier -project development did inspect the static OOT corpus, so the evaluation is a -retrospective lockbox replay rather than a pristine prospective trial. +$$ +k_g=\left\lceil(n_g+1)(1-\alpha)\right\rceil, +$$ -# Appendix A: Exact Conformal and Accounting Details +and $c_g$ is the corresponding ordered residual. The clipped interval is -## A.1 Frozen Recipe Replay +$$ +[\ell_i,u_i]=[\max(0,p_i-c_g),\min(1,p_i+c_g)]. +$$ -Let $p_j$ be calibrated PD and $Y_j\in\{0,1\}$ the calibration outcome. The -active recipe uses the scaled nonconformity score +It predicts the observed binary outcome. It is not a confidence interval for +latent individual PD and not the convex hull of its intersection with the +binary outcome space. The latter intersection is reported separately as +empty, $\{0\}$, $\{1\}$, or $\{0,1\}$. + +| Window | Residual dates | Fit rows | CatBoost fit coverage | +|---|---:|---:|---:| +| W1 | 2012-01--2012-06 | 14,948 | 0.900388 | +| W2 | 2012-02--2012-07 | 16,674 | 0.900444 | +| W3 | 2012-03--2012-08 | 19,049 | 0.900362 | +| W4 | 2012-04--2012-09 | 21,767 | 0.900262 | +| W5 | 2012-05--2012-10 | 24,270 | 0.900247 | +| W6 | 2012-06--2012-11 | 26,685 | 0.900244 | +| W7 | 2012-07--2012-12 | 28,411 | 0.900215 | +| W8 | 2012-08--2013-01 | 30,129 | 0.900229 | + +: S5 complete six-month residual specification. Fit rows are the sums across the five fixed CatBoost strata. {#tbl-s-windows} + +All eight windows are reported. The windows overlap, and their growing sample +sizes partly reflect platform growth. They are not independent temporal +replications. + +## B.4 Candidate coverage with unresolved outcomes + +Let $M_i=1\{Y_i\notin[\ell_i,u_i]\}$. Resolved coverage is computed on labeled +candidates. For each unresolved row, the evaluator computes the two attainable +values $M_i(0)$ and $M_i(1)$. The sharp all-candidate coverage interval is +$[1-n^{-1}\sum_i\max_y M_i(y),\;1-n^{-1}\sum_i\min_y M_i(y)]$. This matters +for empty, singleton, and $\{0,1\}$ sets: an unresolved outcome cannot be forced +to fail when both binary endpoints are covered, nor forced to succeed when the +set is empty. + +| Window | CatBoost resolved | CatBoost all-candidate bound | Logistic resolved | Logistic all-candidate bound | CatBoost mean width | +|---|---:|---:|---:|---:|---:| +| W1 | 0.877047 | [0.854714, 0.880522] | 0.893058 | [0.871819, 0.896222] | 0.664992 | +| W2 | 0.879166 | [0.857025, 0.882597] | 0.892101 | [0.870851, 0.895277] | 0.665341 | +| W3 | 0.877735 | [0.855653, 0.881143] | 0.891819 | [0.870641, 0.894969] | 0.665320 | +| W4 | 0.874873 | [0.852570, 0.878341] | 0.888157 | [0.866664, 0.891374] | 0.665031 | +| W5 | 0.872513 | [0.849932, 0.876033] | 0.885692 | [0.863663, 0.889013] | 0.664708 | +| W6 | 0.872113 | [0.849426, 0.875669] | 0.883045 | [0.860561, 0.886468] | 0.664569 | +| W7 | 0.869481 | [0.846518, 0.873093] | 0.877428 | [0.854517, 0.880976] | 0.664154 | +| W8 | 0.865855 | [0.842485, 0.869564] | 0.873234 | [0.850031, 0.876863] | 0.500569 | + +: S6 primary coverage over 376,890 candidates, including 12,076 unresolved outcomes. {#tbl-s-coverage} + +Every upper endpoint is below 0.90 for both detailed learners. Section B.2 +shows the same eight-of-eight result for all five learner specifications. This +is a temporal transport result for the declared archive; it does not contradict +split conformal validity under exchangeability. + +## B.5 Binary threshold discontinuity + + + +For a constant score $0\le p<1/2$ and $Y\sim\mathrm{Bernoulli}(\pi)$, the +population absolute residual is $p$ with probability $1-\pi$ and $1-p$ with +probability $\pi$. Its $(1-\alpha)$ quantile is therefore $$ -s_j=\frac{|Y_j-p_j|}{\sigma(p_j)},\qquad -\sigma(p)=\sqrt{\max\{p(1-p),10^{-6}\}}. +c(p,\pi)= +\begin{cases} +p, & \pi\le\alpha,\\ +1-p, & \pi>\alpha. +\end{cases} $$ -Calibration scores are partitioned into five score-quantile Mondrian cells. -For a cell with $n_g$ calibration rows, the finite-sample quantile level is +The discrete prediction set changes from $\{0\}$ to $\{0,1\}$ at the +prevalence threshold. This is a population proposition under a constant score, +not a proof for varying scores inside an empirical stratum. + +CatBoost stratum 2 displays the corresponding diagnostic pattern: + +| Window | Fit rows | Fit prevalence | Residual quantile | OOT mean width | Empty | $\{0\}$ | $\{0,1\}$ | OOT coverage bound | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| W1 | 3,249 | 0.116651 | 0.891974 | 0.987580 | 0.0000 | 0.8844 | 0.1156 | [0.847091, 0.875593] | +| W2 | 3,621 | 0.116542 | 0.892012 | 0.987614 | 0.0000 | 0.8832 | 0.1168 | [0.847331, 0.875808] | +| W3 | 4,018 | 0.113987 | 0.891609 | 0.987255 | 0.0000 | 0.8954 | 0.1046 | [0.845092, 0.873885] | +| W4 | 4,511 | 0.109953 | 0.890281 | 0.986041 | 0.0000 | 0.9347 | 0.0653 | [0.837223, 0.867357] | +| W5 | 5,001 | 0.106579 | 0.889415 | 0.985220 | 0.0000 | 0.9606 | 0.0394 | [0.832087, 0.863094] | +| W6 | 5,502 | 0.105234 | 0.889252 | 0.985063 | 0.0000 | 0.9660 | 0.0340 | [0.831189, 0.862360] | +| W7 | 5,929 | 0.101703 | 0.888435 | 0.984263 | 0.0000 | 0.9904 | 0.0096 | [0.826483, 0.858375] | +| W8 | 6,238 | 0.097147 | 0.111801 | 0.207631 | 0.0026 | 0.9974 | 0.0000 | [0.822536, 0.854707] | + +: S7 CatBoost stratum-2 geometry. Set columns report candidate shares. {#tbl-s-phase} + +The W7--W8 prevalence change crosses $\alpha=0.10$ and coincides with the +quantile and width change. The narrower W8 interval does not repair transport; +its stratum-2 all-candidate upper bound is 0.854707. + +For any binary outcome, interval miscoverage has the exact identity $$ -\min\left\{1, -\frac{\lceil(n_g+1)(1-\widetilde\alpha)\rceil}{n_g} -\right\}, +M_i=1\{Y_i=0,\ell_i>0\}+1\{Y_i=1,u_i<1\}. $$ -with `higher` interpolation and frozen used alpha -$\widetilde\alpha=0.095$ for target $\alpha=0.10$. Sparse cells fall back to -the recorded global recipe, and frozen group/temporal factors may widen but -never narrow an interval. For evaluation loan $i$, +Endpoint saturation can therefore increase coverage without improving score +ranking, and a width reduction can increase missed rare positives. + +## B.6 Label-lag sensitivity + + + +The primary recipes date a charged-off label six calendar months after the +month-end of last payment. Because the archive and six-month recipe had already +been inspected, the retrospective sensitivity protocol locked the complete +0-, 3-, 6-, 8-, and 12-month grid before its corresponding run; this is not +preregistration. The inherited rule requires more than 99% retention in every fitting month. +Scores, strata, OOT candidates, and outcomes are unchanged; only fitting-label +availability and the resulting stratum-2 residual quantile vary. + +| Charged-off lag (months) | Minimum monthly retention | Passes >99% rule | W7 prevalence | W7 quantile | W8 prevalence | W8 quantile | +|---:|---:|:---:|---:|---:|---:|---:| +| 0 | 0.997116 | Yes | 0.103216 | 0.888691 | 0.099024 | 0.111870 | +| 3 | 0.996669 | Yes | 0.102914 | 0.888675 | 0.098736 | 0.111862 | +| 6 | 0.992956 | Yes | 0.101703 | 0.888435 | 0.097147 | 0.111801 | +| 8 | 0.986816 | No | 0.099730 | 0.111883 | 0.094097 | 0.111712 | +| 12 | 0.974535 | No | 0.093138 | 0.111690 | 0.087196 | 0.111504 | + +: S7A retrospectively protocol-locked fit-label sensitivity. The W7--W8 threshold crossing persists for every lag satisfying the retention stop rule. {#tbl-s-label-lag} + +The crossing disappears at 8 and 12 months because W7 is already below the +threshold, but those specifications fail the locked retention criterion. This +separates stability across lags satisfying the strict retention rule from sensitivity +under more aggressive label delay. It does not establish that prevalence +caused the empirical geometry change. + +This family changes labels used to fit the conformal recipes. It does not vary +availability of primary OOT outcomes at the evaluation cutoff. Appendix E.2 +reports that separate endpoint-availability family; the two timing axes were +not crossed factorially. + +## B.7 Missingness-encoding sensitivity + + + +The outcome-free sensitivity freezes three CatBoost specifications before the +endpoint join: the active sentinel convention, the same transformed features +plus explicit missing indicators, and native nullable numeric values. The +bankruptcy feature remains binary in every arm: observed positive counts map to +one, observed zeros to zero, and only unavailable values become null in the +native arm. Thus the comparison changes encoding, not feature semantics. + +| Encoding | OOT AUC | Brier | Lowest lower | Highest upper | Windows upper $<.90$ | +|---|---:|---:|---:|---:|---:| +| Active sentinels | 0.640605 | 0.129878 | 0.842485 | 0.882597 | 8/8 | +| Explicit missing indicators | 0.643632 | 0.129652 | 0.845507 | 0.884332 | 8/8 | +| Native nullable features | 0.644262 | 0.129533 | 0.845284 | 0.880037 | 8/8 | + +: Complete missingness-encoding sensitivity. OOT outcomes select neither the encoding nor the learner. {#tbl-s-missingness} + +The coverage finding recurs in all 24 model-window cells. Small descriptive +differences in AUC or Brier do not promote an encoding. The exercise neither +distinguishes missing-at-random from informative missingness nor evaluates +portfolio contrasts under the alternative encodings. + +## B.8 Second temporal origin + + + +One later-origin protocol shifts the same CatBoost construction from the 2016 +decision origin to 2017, using eight six-month residual windows beginning in +2013. The active and later-origin families contain 376,890 and 77,105 +candidates, respectively; 12,076 and 11,014 outcomes remain unrestricted. + +| Window | Primary-origin bound | Later-origin bound | +|---|---:|---:| +| W1 | [0.854714, 0.880522] | [0.753557, 0.874768] | +| W2 | [0.857025, 0.882597] | [0.752429, 0.874340] | +| W3 | [0.855653, 0.881143] | [0.750159, 0.872537] | +| W4 | [0.852570, 0.878341] | [0.750626, 0.873199] | +| W5 | [0.849932, 0.876033] | [0.750820, 0.873406] | +| W6 | [0.849426, 0.875669] | [0.751741, 0.873990] | +| W7 | [0.846518, 0.873093] | [0.750976, 0.873886] | +| W8 | [0.842485, 0.869564] | [0.749899, 0.873523] | + +: Complete two-origin CatBoost coverage census. The later family is a retrospective recurrence, not an independent replication. {#tbl-s-second-origin} + +All 16 upper bounds lie below 0.90. The substantially lower later-origin lower +bounds reflect its larger unresolved fraction; they should not be read as a +more severe resolved-case coverage failure. No origin is selected, and two +origins cannot establish temporal invariance or prospective performance. + +## B.9 Declared fit-label scenario sensitivity + + + + +The active fit uses only labels observable at each fitting cutoff. There are +215 unavailable labels: 41 in PD development, 24 in Platt calibration, and 150 +in conformal fitting. A separately frozen sensitivity refits the complete +CatBoost--Platt--residual pipeline under the observed-only fit and three +declared stress rules. The +evaluation outcome panel remains physically isolated until all scores and +recipes have been written. + +| Completion scenario | Lowest lower | Highest upper | Windows upper $<.90$ | W7 prevalence / quantile | W8 prevalence / quantile | W7--W8 crossing | +|---|---:|---:|---:|---:|---:|:---:| +| All unavailable default | 0.849662 | 0.884669 | 8/8 | 0.105974 / 0.889212 | 0.100287 / 0.887805 | No | +| All unavailable nondefault | 0.842466 | 0.883252 | 8/8 | 0.101993 / 0.889341 | 0.093092 / 0.111002 | Yes | +| Hindsight terminal | 0.848980 | 0.884579 | 8/8 | 0.105467 / 0.889103 | 0.099809 / 0.112260 | Yes | +| Observed only | 0.842485 | 0.882594 | 8/8 | 0.101703 / 0.888435 | 0.097147 / 0.111801 | Yes | + +: S7B declared fit-label scenario sensitivity. Coverage columns summarize the eight overall candidate bounds; prevalence and quantile columns refer to stratum 2. {#tbl-s-fit-completion} + +All 32 overall scenario-window upper bounds remain below 0.90. The coverage +transport conclusion is therefore unchanged across these four scenarios. The +geometric account is narrower: the all-default scenario leaves both W7 and W8 +above the 0.10 prevalence threshold and removes the quantile crossing. Since +learner and calibration refits are nonlinear, these four scenarios are +declared stresses rather than sharp bounds over every one of the $2^{215}$ label +assignments. No scenario is selected, and the later terminal scenario is +hindsight sensitivity rather than an information-safe deployment recipe. + +# Appendix C: Portfolio and Comparator Protocol + +## C.1 Coherent plug-in objective and status-indexed endpoint + +For candidate $i$ with listed amount $L_i$, return rate $r_i$, point score +$p_i$, and LGD $\lambda=0.45$, the optimizer uses $$ -[\ell_i,u_i] -=\left[ -\max\{0,p_i-\widehat q_{g(i)}\sigma(p_i)\}, -\min\{1,p_i+\widehat q_{g(i)}\sigma(p_i)\} -\right] +v_i=(1-p_i)r_i-p_i\lambda. $$ -before the recorded widening factors. +The status-indexed standardized payoff proxy uses the algebraically coherent endpoint + +$$ +z_i(Y_i)=(1-Y_i)r_i-Y_i\lambda. +$$ -The replay reads all settings from the frozen conformal result payload. At the -90% reference level, maximum absolute differences against the stored artifact -are `4.44e-16` for point predictions, `3.33e-16` for lower endpoints, and -`6.67e-16` for upper endpoints. The active claim therefore uses recomputed -finite-sample quantiles rather than average-width scaling. +Thus $v_i$ is the model-implied expectation of $z_i$ if $p_i$ were the true +conditional event probability. The audit does not assert that condition. The +endpoint is not investor return, IRR, NPV, welfare, or a cash-flow model; it +omits amortization, prepayment timing, recovery timing, fees, discounting, and +capital costs. -## A.2 Policy Algebra +## C.2 Monthly linear program -For a generic linear blend -$q_i=p_i+\gamma(u_i-p_i)$ and funded-exposure weights $w_i$, +For score $s_i$ and cap $\tau$, each monthly policy solves $$ -\Gamma_{\mathrm{CP}}=\sum_iw_i(u_i-p_i), +\begin{aligned} +\max_a\quad & \sum_i a_i v_i \\ +\text{s.t.}\quad +& \sum_i a_i=B,\\ +& 0\le a_i\le L_i,\\ +& \sum_i a_i s_i\le \tau B,\\ +& \sum_{i:g(i)=h}a_i\le0.25B\quad\text{for each purpose }h, +\end{aligned} $$ +with $B=\$1{,}000{,}000$. The point policy uses $s_i=p_i$. The guardrail uses + $$ -\Gamma_{\mathrm{int}}=\gamma\Gamma_{\mathrm{CP}},\qquad -\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}. +q_i(\gamma)=(1-\gamma)p_i+\gamma u_i. $$ -At $\gamma=0.5$, the conformal premium is split equally. The endpoint budget is +The complete score path is +$\gamma\in\{0,0.25,0.50,0.75,1\}$. The frozen empirical contrast is +$\gamma=1$ minus $\gamma=0$; the three interior gamma values diagnose the path +and cannot be selected as winners. + +## C.3 Two common-frontier rulers + +Let $\mathcal A_t$ contain the nonrisk constraints above, let +$s(\gamma)=q(\gamma)$, and define $$ -B_u=\sum_iw_iu_i -=\sum_iw_iq_i+\Gamma_{\mathrm{res}}. +m_{\gamma t}=B^{-1}\min_{a\in\mathcal A_t}s(\gamma)^\top a, +\quad z_t^*=\max_{a\in\mathcal A_t}v^\top a, +\quad o_{\gamma t}=B^{-1}s(\gamma)^\top a_t^*. $$ -The optimizer constrains $\sum_iw_iq_i\le\tau$. It does not constrain $B_u$ -directly, which is why the residual premium remains visible in every audit. +For the primary **objective-matched ruler**, let +$z^{\min}_{\gamma t}$ denote the plug-in objective at a minimum-score +portfolio and $z_t^L=\max_\gamma z^{\min}_{\gamma t}$. At +$\rho\in\{.25,.50,.75\}$ every score minimizes funded score subject to the +same objective floor -## A.3 Deterministic Identity +$$ +v^\top a\ge z_t^L+\rho(z_t^*-z_t^L). +$$ -**Proposition A.1 (funded-set accounting).** For any fixed allocation with -nonnegative normalized weights, outcomes $Y_i\in[0,1]$, upper endpoints -$u_i\in[0,1]$, and -$Z_i=\mathbf1\{Y_i>u_i\}$, +This matches model-implied objective sacrifice, not true expected return. For +the secondary **normalized-score ruler**, define $$ -\sum_iw_iY_i\le B_u+V, -\qquad -B_u=\sum_iw_iu_i, -\quad -V=\sum_iw_iZ_i. +c_{\gamma t}(\eta)=m_{\gamma t} + +\eta(o_{\gamma t}-m_{\gamma t}),\qquad +\eta\in\{.25,.50,.75\}, +$$ + +and maximize the plug-in objective under +$s(\gamma)^\top a\le Bc_{\gamma t}(\eta)$. This coordinate is invariant to +positive affine score units, but it does not match objective sacrifice, +default risk, or shadow price. Coordinate one reaches the common verified +plug-in optimum for every gamma and is therefore a structural null for the +endpoint allocation contrast. The evaluated three-coordinate interior grid is +finite; it is not a continuous two-ruler frontier. + +## C.4 Supporting named comparators and exact support + +The earlier fixed-cap grid supplies supporting comparator diagnostics: + +| Policy | $\tau$ | $\gamma$ | Policy | $\tau$ | $\gamma$ | Policy | $\tau$ | $\gamma$ | +|---|---:|---:|---|---:|---:|---|---:|---:| +| P1 | 0.15 | 0.25 | P4 | 0.17 | 0.25 | P7 | 0.19 | 0.25 | +| P2 | 0.15 | 0.50 | P5 | 0.17 | 0.50 | P8 | 0.19 | 0.50 | +| P3 | 0.15 | 0.75 | P6 | 0.17 | 0.75 | P9 | 0.19 | 0.75 | + +: S8 supporting fixed-cap grid. It is not a closed policy family and no policy is selected or promoted. {#tbl-s-policies} + +The score and its cap jointly define a feasible decision problem. The audit +therefore declares four point-score comparison objects. + +**C0: same numeric cap.** The point policy copies the guardrail's $\tau$. Since +$q_i\ge p_i$, this is a positive control for mechanical feasible-set nesting, +not a neutral baseline. + +**C1: development mean.** For each window-policy pair, the point cap is the +capital-weighted mean of the guardrail allocation's point-score moments over +the eleven common February--December 2013 development menus. + +**C2: contemporaneous funded-moment match.** After a guardrail allocation +$a^q$ is frozen on an OOT menu, the point cap is + $$ +\tau^{C2}=\frac{\sum_i a_i^q p_i}{B}. +$$ + +This is outcome-free and menu-adaptive. It is useful for decomposition but is +not a deployable ex ante comparator because its cap is defined from the +guardrail allocation it evaluates. -*Proof.* If $Y_i\le u_i$, then $Y_i\le u_i+Z_i=u_i$. If $Y_i>u_i$, then -$Z_i=1$ and $Y_i\le1\le u_i+1$. Multiply by $w_i$, sum, and use -$w_i\ge0$. $\square$ +**Exact cap frontier.** The development-admissible support for each +window-policy pair is the closed minimum--maximum interval of its eleven +monthly development point-score moments. Its global hull is +$[0.055573,0.099997]$. A secondary broad-stress support is $[0.05,0.12]$. +HiGHS basis-ranging endpoints and every named/support endpoint produce 3,067 +distinct point caps. The supports are fixed without outcomes. -The proposition is deterministic and remains true after adaptive allocation. -It is deliberately weak: $V$ is observed only after outcomes become available. +## C.5 Allocation granularity diagnostic -## A.4 Assumption-Conditional Markov Corollary +The LP permits fractional dollar exposure. To diagnose whether that relaxation +drives the reported rates, an outcome-free transformation floors each positive +baseline exposure to the nearest USD 25 lot and leaves the monthly residual as +cash. It does not redistribute cash, resolve a mixed-integer program, change a +ruler, or select a track. Both rounded and continuous outcomes are normalized +by the same committed capital, so cash remains visible rather than silently +renormalizing the rounded portfolio upward. -**Assumption A.1 (weighted funded-set validity).** For the random -calibration/evaluation draw and frozen allocation rule, -$\mathbb E[V]\le\alpha$. +# Appendix D: Exact Statements and Proofs -This assumption is stronger than marginal or Mondrian coverage. The optimizer -uses $p_i$ and $u_i$ to choose $w_i$, so per-cell conformal validity does not -automatically transfer to funded-dollar weights. +## D.1 Positive affine cap equivalence and same-cap nesting -**Corollary A.1.** Under Assumption A.1, +Suppose a score is globally $s_i=\kappa p_i+b$ with $\kappa>0$ and every feasible +portfolio fills budget $B$. Then $$ -\Pr(V\ge t)\le\frac{\alpha}{t},\quad t>0, +\sum_i a_i s_i\le\tau B +\quad\Longleftrightarrow\quad +\sum_i a_i p_i\le\frac{\tau-b}{\kappa}B. $$ -and therefore +Thus a positive affine score admits an exactly translated point cap. The +empirical guardrail is not globally affine because clipping and stratum-specific +residual quantiles change its slope and intercept. + +For the qualified converse, work on the full-budget affine hull +$H=\{a:\mathbf 1^\top a=B\}$. If the two cap constraints are proper and +nonredundant and their boundaries agree on a relatively open subset of $H$, +their projected normals must be positively collinear. Equivalently, +$s=\kappa p+b\mathbf 1$ on that affine hull for some $\kappa>0$. This converse +does not follow from equality over a lower-dimensional feasible polytope or +from two formulations returning the same optimizer at one cap. + +Because $u_i\ge p_i$ and $\gamma\in[0,1]$, $q_i\ge p_i$. At the same numerical +cap, every guardrail-feasible allocation is point-feasible. The point optimum +therefore weakly dominates in the shared plug-in objective. Neither statement +orders status-indexed outcomes. + +## D.2 Positive-affine invariance of normalized coordinates + +Let $\widetilde s=\kappa s+b$ for $\kappa>0$. The score minimum and score of the common +plug-in optimum transform as +$m_{\widetilde s}=\kappa m_s+b$ and $o_{\widetilde s}=\kappa o_s+b$. Hence $$ -\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt\alpha\right) -\le\sqrt\alpha. +c_{\widetilde s}(\eta)=\kappa c_s(\eta)+b. $$ -*Proof.* Apply Markov's inequality to nonnegative $V$, set -$t=\sqrt\alpha$, and combine with Proposition A.1. $\square$ +Using the full-budget equality, +$\widetilde s^\top a\le Bc_{\widetilde s}(\eta)$ if and only if +$s^\top a\le Bc_s(\eta)$. The normalized ruler therefore defines the same +feasible set after a positive affine change of score units. The result does not +extend to the non-affine point-versus-upper score map and does not equalize +plug-in opportunity cost. + +## D.3 C2 plug-in objective dominance -At active $\alpha=0.10$, the probability bound is `0.316228`. Because this is -loose and assumption-conditional, the paper treats it as sensitivity language, -not as the primary contribution or a deterministic default cap. Formal -selected-set validity would require a dedicated protocol -[@hegazy2025valid_selection_conformal_sets]. +Let $a^q$ be a full-budget guardrail allocation and set +$c=p^\top a^q/B$. Then $p^\top a^q=cB$, so $a^q$ is feasible for the point LP +with cap $c$. Every nonrisk constraint and the plug-in objective are unchanged. +Consequently, -# Appendix B: Active Evidence A35--A40 +$$ +V_p^*(c)\ge v^\top a^q. +$$ -## A35. Exact Alpha Replay and Saturation +Across all 1,080 C2 cells, the maximum absolute funded-moment residual is +$8.33\times10^{-17}$. The minimum point-minus-guardrail plug-in objective is +$-1.46\times10^{-10}$ dollars, numerical zero within solver tolerance. This +reconciles the theorem; it does not establish status-indexed payoff dominance. -| Selected | Target $\alpha$ | Used $\alpha$ | Coverage | Avg. width | Min partition coverage | Min grade coverage | $u=1$ rate | -|:---:|---:|---:|---:|---:|---:|---:|---:| -| no | `0.01` | `0.0095` | `0.996720` | `0.988215` | `0.989629` | `0.972270` | `0.935424` | -| no | `0.03` | `0.0285` | `0.988478` | `0.969779` | `0.969799` | `0.966493` | `0.832314` | -| no | `0.05` | `0.0475` | `0.972825` | `0.953580` | `0.950583` | `0.952575` | `0.718156` | -| no | `0.07` | `0.0665` | `0.955271` | `0.795666` | `0.934782` | `0.938443` | `0.613550` | -| yes | `0.10` | `0.0950` | `0.934836` | `0.788879` | `0.926310` | `0.926797` | `0.517873` | -| no | `0.12` | `0.1140` | `0.918265` | `0.783174` | `0.906875` | `0.905738` | `0.454511` | -| no | `0.15` | `0.1425` | `0.886098` | `0.646176` | `0.863562` | `0.863101` | `0.334859` | -| no | `0.20` | `0.1900` | `0.849380` | `0.636585` | `0.829376` | `0.828007` | `0.244668` | +## D.4 Sharp common-outcome bounds -: A35 exact alpha sensitivity. Source: `crpto_tableA35_exact_alpha_grid.csv`. + -The 99% row is nearly saturated and is not the active policy level. Rows other -than 90% apply the same frozen widening recipe as sensitivity; they are not -separately tuned conformal winners. +For fixed exposure weights $w_i$ and an additive binary endpoint +$f_i(Y_i)$, each unresolved outcome may be zero or one. The fixed-policy lower +and upper bounds are -## A36. Calibration Policy Selector +$$ +\sum_{i\in R}w_i f_i(Y_i)+ +\sum_{i\in U}\min_{y\in\{0,1\}}w_i f_i(y), +\qquad +\sum_{i\in R}w_i f_i(Y_i)+ +\sum_{i\in U}\max_{y\in\{0,1\}}w_i f_i(y). +$$ -| Nov. status | Candidate | $\tau$ | $\gamma$ | Nov. objective | Nov. $B_u$ | Dec. $B_u$ | -|---|---|---:|---:|---:|---:|---:| -| **Selected** | `linear-005` | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | -| Eligible | `linear-009` | `0.19` | `0.75` | `$96,446.16` | `0.226723` | `0.228577` | -| Eligible | `linear-002` | `0.15` | `0.50` | `$95,995.22` | `0.226088` | `0.227695` | -| Eligible | `linear-006` | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | -| Eligible | `linear-003` | `0.15` | `0.75` | `$90,893.16` | `0.177614` | `0.178203` | -| Ineligible | `linear-007` | `0.19` | `0.25` | `$113,986.45` | `0.422203` | `0.446378` | -| Ineligible | `linear-004` | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | -| Ineligible | `linear-001` | `0.15` | `0.25` | `$105,398.23` | `0.328413` | `0.343502` | -| Ineligible | `linear-008` | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | +For candidate coverage, $f_i=1-M_i$ gives the equivalent and operationally +clear form -: A36 complete selector. Source: `crpto_tableA36_calibration_policy_selector.csv`. +$$ +C_L=1-\frac{1}{n}\sum_i\max_y M_i(y),\qquad +C_U=1-\frac{1}{n}\sum_i\min_y M_i(y). +$$ -The selector uses `14,943` November rows and the deterministic screen -$B_u\le0.28$. The loader keeps outcomes separate from its 12-column input -frame, which contains no outcome or assumption-conditional quantity. The -selected row remains optimal for caps in -`[0.259036, 0.290491)`. Replaying the same outcome-free rule on `20,695` -December rows selects the same policy. Only then are December outcomes opened: -the midpoint funds 193 loans, returns `$53,313.05`, has weighted default -`0.145650`, weighted miscoverage `0.124925`, and endpoint `0.262082`. The -miscoverage miss is direct evidence that selector stability is not selected-set -coverage validity. +These are not obtained by adding or subtracting the unresolved fraction from +resolved coverage; the attainable miss values depend on each prediction set. -## A37. Temporal Fixed-Policy Evaluation +For a paired contrast, one common unresolved $Y_i$ is optimized over the union +of loans funded by either policy. This avoids the invalid operation of assigning +different outcomes to the same unresolved loan under two policies. The bounds +are sharp for the finite archive under unrestricted binary completion. They are +not sampling confidence intervals. -| Period | Policy | Return | Default | Miscoverage | Endpoint | Threshold | -|---|---|---:|---:|---:|---:|---:| -| Full OOT | Selected 50/50 | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | -| Full OOT | Conservative 75% | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | -| Full OOT | Point PD | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | -| 2018H1 | Selected 50/50 | `$92,530.73` | `0.106703` | `0.097428` | `0.259170` | `0.575398` | -| 2018H1 | Point PD | `$118,101.99` | `0.190825` | `0.076425` | `0.938912` | `1.255140` | -| 2018H2 | Selected 50/50 | `$156,185.51` | `0.026725` | `0.026725` | `0.262441` | `0.578669` | -| 2018H2 | Point PD | `$95,603.58` | `0.236728` | `0.113175` | `0.915546` | `1.231774` | -| 2019H1 | Selected 50/50 | `$123,590.69` | `0.077325` | `0.061700` | `0.261872` | `0.578100` | -| 2019H1 | Point PD | `$144,281.46` | `0.170275` | `0.038275` | `0.884838` | `1.201065` | -| 2019H2 | Selected 50/50 | `$110,251.95` | `0.103250` | `0.103250` | `0.261503` | `0.577731` | -| 2019H2 | Point PD | `$256,966.20` | `0.023775` | `0.007900` | `0.898805` | `1.215033` | -| 2020+ | Selected 50/50 | `$99,689.54` | `0.083775` | `0.083775` | `0.264462` | `0.580690` | -| 2020+ | Point PD | `$218,629.14` | `0.016900` | `0.000000` | `0.861425` | `1.177653` | + -: A37 selected temporal rows and matched point-PD rows. Source: -`crpto_tableA37_calibration_selected_temporal_evaluation.csv`. +For any additive paired contrast with unresolved contribution $d_i(Y_i)$, +separability gives the exact width -Each temporal policy receives a fresh `$1M` budget. The table is intentionally -unfavorable to a universal-dominance reading: point PD is much stronger in -2019H2 and 2020+, whereas CRPTO is much stronger in 2018H2. +$$ +U-L=\sum_{i\in U}|d_i(1)-d_i(0)|. +$$ -## A38. Funded Letter-Grade Audit +Let $\delta_i=a_i^A-a_i^B$. For declared positive capital normalizers +$N_A,N_B$, define $\delta_i^w=a_i^A/N_A-a_i^B/N_B$. The reported widths are +therefore -| Grade | Funded | Exposure share | Default | Miscoverage | Point PD | Midpoint $q$ | Endpoint | Return | -|---|---:|---:|---:|---:|---:|---:|---:|---:| -| B | `1` | `0.0040` | `0.000000` | `0.000000` | `0.062075` | `0.101472` | `0.140869` | `$643.20` | -| C | `91` | `0.3136` | `0.028622` | `0.028622` | `0.067801` | `0.108598` | `0.149395` | `$47,303.75` | -| D | `170` | `0.5868` | `0.045839` | `0.045839` | `0.088328` | `0.194926` | `0.301523` | `$109,618.08` | -| E | `33` | `0.0806` | `0.000000` | `0.000000` | `0.085242` | `0.179551` | `0.273860` | `$19,879.71` | -| F | `11` | `0.0130` | `0.269231` | `0.076923` | `0.110970` | `0.425687` | `0.740404` | `$1,266.55` | -| G | `2` | `0.0020` | `0.000000` | `0.000000` | `0.146801` | `0.573401` | `1.000000` | `$616.30` | +$$ +\begin{aligned} +W_{\mathrm{payoff}}&=\sum_{i\in U}|\delta_i|(r_i+\lambda),\\ +W_{\mathrm{payoff\ rate}}&=\sum_{i\in U}|\delta_i^w|(r_i+\lambda),\\ +W_{\mathrm{default}}&=\sum_{i\in U}|\delta_i^w|,\\ +W_{\mathrm{miss}}&=\sum_{i\in U}|\delta_i^w|\,|m_i(1)-m_i(0)|. +\end{aligned} +$$ -: A38 selected funded-set composition. Source: -`crpto_tableA38_calibration_selected_grade_audit.csv`. +The active fully invested policies set $N_A=N_B=B$. The floor-with-cash +diagnostic retains the same committed-capital denominator $B$ so that cash is +not removed by renormalization. Resolved loans contribute zero width. +Unresolved loans funded identically by both policies also contribute zero; for +miscoverage, so do unresolved loans whose prediction set gives the same miss +indicator at both binary outcomes. +Identification loss is consequently governed by exposure disagreement and +binary set geometry, not by the unresolved count alone. -The table uses letter grade recovered from `sub_grade`; the aligned conformal -group is stored separately. This prevents the score-quantile partition labels -from being misreported as credit grades. +## D.5 Basis-endpoint sufficiency -## A39. Fixed-Allocation Bootstrap + -| Resampling unit | Metric | Observed | 2.5% | 97.5% | -|---|---|---:|---:|---:| -| Month clusters | Realized return | `$179,327.59` | `$163,421.14` | `$193,551.65` | -| Month clusters | Weighted default | `0.039375` | `0.020517` | `0.062622` | -| Month clusters | Weighted miscoverage | `0.036875` | `0.018338` | `0.060084` | -| Month clusters | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.132145` | `0.230815` | -| Month clusters | Endpoint budget | `0.258051` | `0.211996` | `0.314870` | -| Funded loans | Realized return | `$179,327.59` | `$162,706.17` | `$193,924.74` | -| Funded loans | Weighted default | `0.039375` | `0.020869` | `0.061742` | -| Funded loans | Weighted miscoverage | `0.036875` | `0.019358` | `0.058812` | -| Funded loans | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.137159` | `0.224308` | -| Funded loans | Endpoint budget | `0.258051` | `0.217308` | `0.308571` | - -: A39, 5,000 fixed-allocation resamples per scheme with seed `20260709`. -Month clusters are primary; funded-loan resampling is an independence -sensitivity. Source: `crpto_tableA39_calibration_selected_bootstrap.csv`. The -CSV also records bootstrap means and medians. - -## A40. Matched Decision Audit - -| Policy | Expected objective | Realized return | Default | Miscoverage | Endpoint | Cond. threshold | +With the full-budget equality, the point-score cap appears in the LP right-hand +side. Within one optimal basis, the allocation is affine in the cap. Against a +fixed guardrail, every resolved-outcome contrast is therefore affine. For one +unresolved outcome, the sharp lower contribution is the minimum of two affine +functions and the upper contribution their maximum. The lower bound is concave +and the upper bound convex within the basis range, so adverse extrema occur at +its endpoints. + +Evaluating each support endpoint and every HiGHS basis-ranging endpoint is +therefore sufficient for the exact sharp envelope over the declared support, +up to solver tolerance. Fixed-grid interpolation is not used for an exactness +claim. + +For support $\mathcal C$, if $[L(c),U(c)]$ is the paired sharp contrast at cap +$c$, the declared comparator envelope is + +$$ +\left[\min_{c\in\mathcal C}L(c), + \max_{c\in\mathcal C}U(c)\right]. +$$ + +This reports whether a sign survives one named outcome-free comparator support. +It is not a causal identified set, a confidence interval, or quantification +over all conceivable comparators. + +# Appendix E: Empirical Comparator Audit + +## E.1 Complete two-ruler endpoint census + + + + + +Each row below summarizes one protocol-locked ruler-coordinate track over the same +eight overlapping residual recipes and fifteen OOT menus. Contrasts are +$\gamma=1$ minus $\gamma=0$. The objective-matched ruler is primary; the +normalized-score ruler is a secondary scale-invariance stress test. + +| Ruler | Coordinate | Active months/window | Plug-in objective difference (USD) | Payoff bound hull (USD) | Default bound hull (pp) | Miscoverage bound hull (pp) | |---|---:|---:|---:|---:|---:|---:| -| Selected 50/50 CRPTO | `$168,271.56` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | -| More-conservative 75% blend | `$160,690.13` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | -| Point-PD matched-$\tau$ | `$214,019.15` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | - -: A40 matched full-OOT audit. Source: -`crpto_tableA40_calibration_selected_point_baseline.csv`. - -Selected CRPTO pays `$17,041.55` (`8.678%`) relative to point PD. It reduces -weighted default by `7.9025` percentage points, weighted miscoverage by -`0.5025` percentage points, and the conditional threshold by `66.3266` -percentage points. -Its exact active funded-set quantities are -$\Gamma_{\mathrm{CP}}=0.176102$, -$\Gamma_{\mathrm{res}}=0.088051$, -$B_u=0.258051$, -$B_u+V=0.294926$, and -$B_u+\sqrt{0.10}=0.574279$. - -# Appendix C: Supporting Diagnostics A1--A34 - -A1--A34 were generated before the final simplification. They remain useful for -reviewer questions but are not coequal methods or selectors. - -| Range | Contents | Current role | +| Objective matched | .25 | 4 | [0.00, 0.00] | [-9,134.34, 5,603.66] | [-0.0068, 0.1265] | [-0.0068, 0.1265] | +| Objective matched | .50 | 15 | [0.00, 0.00] | [-82,616.17, -27,958.37] | [0.4572, 1.0973] | [1.0154, 1.9321] | +| Objective matched | .75 | 15 | [0.00, 0.00] | [-179,484.66, 92,558.18] | [-0.4352, 2.4948] | [1.3252, 4.1848] | +| Normalized score | .25 | 15 | [425,196.46, 557,294.01] | [-626,374.61, -195,967.63] | [8.4829, 13.4246] | [8.3910, 13.7536] | +| Normalized score | .50 | 15 | [152,030.99, 226,847.97] | [-259,658.18, -54,025.82] | [3.2214, 6.5637] | [2.1070, 5.2142] | +| Normalized score | .75 | 15 | [28,263.40, 51,201.89] | [-135,781.22, 9,812.59] | [1.4392, 2.3807] | [0.3447, 1.6991] | + +: S9 complete finite-grid endpoint contrast. Hulls are minimum lower and maximum upper bounds across windows, not sampling intervals. {#tbl-s-two-ruler} + +| Ruler / coordinate | Payoff-proxy width (USD) | Default width (pp) | Miscoverage width (pp) | +|---|---:|---:|---:| +| Objective matched .25 | 14,738--14,738 | 0.1333--0.1333 | 0.1333--0.1333 | +| Objective matched .50 | 53,530--54,370 | 0.6282--0.6375 | 0.6282--0.6375 | +| Objective matched .75 | 109,550--271,562 | 1.2006--2.9246 | 1.1340--2.6459 | +| Normalized score .25 | 340,845--373,705 | 3.7599--4.1019 | 3.6399--3.9432 | +| Normalized score .50 | 146,499--198,657 | 1.5346--2.0383 | 1.5346--1.9183 | +| Normalized score .75 | 83,833--97,897 | 0.8041--0.9416 | 0.7256--0.9416 | + +: Exact identification-width ranges across the eight windows. Each width equals the sum of unresolved loan-wise contribution ranges in Appendix D.4. {#tbl-s-identification-width} + +The active evaluator changes only outcome observability and the resulting sharp +bounds; scores, residual recipes, rulers, coordinates, and allocations remain +frozen. Its five-row endpoint-reason partition appears in +@tbl-s-endpoint-reasons. The sharp interval width is governed by the exposure +disagreement on those unresolved loans, as shown in Appendix D.4, rather than +by treating all 12,076 cases as an undifferentiated timing count. + +Objective-matched .25 is one repeated allocation contrast: every window has +the same four active months and eleven structural zeros, with allocations +identical across windows to cents. The guardrail-minus-point allocation changes 44 loan-month positions +and USD 155,937.27 of one-way turnover across USD 15 million. Under the +reconstructed endpoint, all three sharp intervals cross zero in all eight +windows. The earlier point values are attainable upper or lower endpoints, not +identified contrasts. This is one repeated allocation, not eight independent +confirmations. + +At objective-matched .50, payoff is lower and default/miscoverage higher in all +eight window bounds. At .75, payoff and default cross zero in seven windows and +are adverse in W8; miscoverage is higher in all eight. Normalized .25 and .50 +are adverse in all eight windows. At normalized .75, default and miscoverage +are adverse in all eight, while payoff is adverse in seven and crosses zero in +one. The plug-in objective differences show that this ruler does not equalize +opportunity cost. Across the full 48-cell census, payoff is lower in 32 and +crosses zero in 16; default is higher in 33 and crosses zero in 15; +miscoverage is higher in 40 and crosses zero in 8. These are specification +counts, not votes. + +## E.2 Evaluation-endpoint availability sensitivity + +The endpoint-availability sensitivity holds every score, fitted residual +recipe, ruler, coordinate, support, and allocation fixed. It changes only the +administrative lag used to decide whether a Charged Off outcome is observable +at September 30, 2020. All five retrospectively protocol-locked lags are +reported; neither coverage nor portfolio outcomes select an endpoint. + +| Lag (months) | Resolved / unresolved | Coverage upper $<.90$ | Maximum upper | Payoff lower / cross | Default higher / cross | Miscoverage higher / cross | +|---:|---:|---:|---:|---:|---:|---:| +| 0 | 364,861 / 12,029 | 40 / 40 | 0.897641 | 32 / 16 | 33 / 15 | 40 / 8 | +| 3 | 364,861 / 12,029 | 40 / 40 | 0.897641 | 32 / 16 | 33 / 15 | 40 / 8 | +| 6 | 364,814 / 12,076 | 40 / 40 | 0.897726 | 32 / 16 | 33 / 15 | 40 / 8 | +| 8 | 364,570 / 12,320 | 40 / 40 | 0.898151 | 32 / 16 | 33 / 15 | 40 / 8 | +| 12 | 363,288 / 13,602 | 39 / 40 | 0.900411 | 31 / 17 | 32 / 16 | 40 / 8 | + +: S9B complete nonselective evaluation-endpoint lag grid. Direction counts summarize all 48 two-ruler cells for $\gamma=1-\gamma=0$; omitted opposite and exact-zero counts are zero. {#tbl-s-endpoint-lag} + +The six-month row reconciles exactly to the active evaluation: after removing the lag +column, all 120 coverage cells, 48 two-ruler contrasts, and 648 exact-support +envelopes are value-identical. Lags 0, 3, 6, and 8 retain the active 40/40 coverage result +and the same two-ruler direction census despite different endpoint counts. At +12 months, the pricing-excluded WOE/IV scorecard in W2 has upper coverage +0.900411, leaving 39/40 cells below target. One payoff and one default cell also +move from a one-sided adverse direction to crossing zero; no opposite +one-sided direction appears. + +| Lag (months) | Development payoff lower / cross | Development default cross | Development miscoverage higher / cross | Broad-support cross | +|---:|---:|---:|---:|---:| +| 0 | 6 / 66 | 72 | 27 / 45 | 216 | +| 3 | 6 / 66 | 72 | 27 / 45 | 216 | +| 6 | 6 / 66 | 72 | 27 / 45 | 216 | +| 8 | 6 / 66 | 72 | 27 / 45 | 216 | +| 12 | 0 / 72 | 72 | 26 / 46 | 216 | + +: S9C complete exact-support endpoint sensitivity. Each row contains all 72 development-support cells and all 216 broad-stress cells; omitted opposite and exact-zero counts are zero. {#tbl-s-endpoint-support-lag} + +Thus the broad-support nonidentification result survives every endpoint lag, +while the all-model/all-window coverage statement is not universal over the +complete lag grid. The active six-month result remains the declared endpoint +contract, not a lag selected from these outcomes. This family must not be +combined with Appendix B.6 as if all 25 fit-label-by-endpoint combinations had +been evaluated. + +## E.3 Portfolio-structure sensitivity + + + +The structural sensitivity changes three declared portfolio assumptions while +preserving the frozen scores, residual recipes, rulers, coordinates, windows, +monthly menus, and endpoint contract. It reports every element of +$\{0.5,1,2\}$ million in budget $\times\{0.20,0.25,0.30,1.00\}$ in maximum +purpose share $\times\{0.25,0.45,0.65\}$ in LGD. Allocations were constructed +without outcome columns and hash-frozen before the endpoint join; no scenario +or result was selected. + +| Metric | Adverse | Favorable | Crosses zero | Exact zero | Total | +|---|---:|---:|---:|---:|---:| +| Status-indexed payoff proxy | 740 | 223 | 709 | 56 | 1,728 | +| Terminal default | 1,072 | 133 | 467 | 56 | 1,728 | +| Funded miscoverage | 1,205 | 131 | 336 | 56 | 1,728 | + +: S9D complete structural direction census. Adverse means lower payoff or higher default/miscoverage; favorable reverses those signs. Counts cover 36 scenarios, six ruler-coordinate tracks, and eight windows. {#tbl-s-structure-aggregate} + +Every scenario retains at least 17 adverse default cells and 21 adverse +miscoverage cells. Zero scenarios are favorable on all three metrics in all 48 +cells, and zero are adverse on all three metrics in all 48 cells. Conversely, +26 scenarios contain at least one favorable payoff cell and 20 contain at +least one favorable default or miscoverage cell. These facts rule out both a +universal favorable ordering and its universal-adversity mirror. + +The complete scenario census follows. Within each metric, entries are +adverse/favorable/crossing/exact-zero counts and sum to 48. `B` is monthly +budget in USD millions; `Cap bind` is the share of the 1,440 portfolios in which +the purpose constraint binds. + +| B | Purpose cap | LGD | Pay A/F/C/Z | Default A/F/C/Z | Miss A/F/C/Z | Cap bind | +|---:|---:|---:|---:|---:|---:|---:| +| 0.5 | 0.20 | 0.25 | 16/1/31/0 | 39/1/8/0 | 39/0/9/0 | 100% | +| 0.5 | 0.20 | 0.45 | 32/8/8/0 | 32/1/15/0 | 39/1/8/0 | 100% | +| 0.5 | 0.20 | 0.65 | 31/8/1/8 | 32/8/0/8 | 32/3/5/8 | 100% | +| 0.5 | 0.25 | 0.25 | 16/8/24/0 | 24/8/16/0 | 39/0/9/0 | 100% | +| 0.5 | 0.25 | 0.45 | 30/2/8/8 | 31/1/8/8 | 23/1/16/8 | 100% | +| 0.5 | 0.25 | 0.65 | 32/0/8/8 | 32/0/8/8 | 32/0/8/8 | 100% | +| 0.5 | 0.30 | 0.25 | 17/8/23/0 | 40/1/7/0 | 40/6/2/0 | 100% | +| 0.5 | 0.30 | 0.45 | 24/1/15/8 | 31/1/8/8 | 24/8/8/8 | 100% | +| 0.5 | 0.30 | 0.65 | 31/0/9/8 | 31/0/9/8 | 32/0/8/8 | 100% | +| 0.5 | 1.00 | 0.25 | 15/8/25/0 | 25/0/23/0 | 24/8/16/0 | 0% | +| 0.5 | 1.00 | 0.45 | 30/1/9/8 | 31/1/8/8 | 30/0/10/8 | 0% | +| 0.5 | 1.00 | 0.65 | 24/8/8/8 | 24/8/8/8 | 32/0/8/8 | 0% | +| 1.0 | 0.20 | 0.25 | 1/17/30/0 | 32/15/1/0 | 32/16/0/0 | 100% | +| 1.0 | 0.20 | 0.45 | 11/8/29/0 | 25/8/15/0 | 34/8/6/0 | 100% | +| 1.0 | 0.20 | 0.65 | 33/0/15/0 | 33/0/15/0 | 40/0/8/0 | 100% | +| 1.0 | 0.25 | 0.25 | 14/15/19/0 | 24/8/16/0 | 32/15/1/0 | 100% | +| 1.0 | 0.25 | 0.45 | 32/0/16/0 | 33/0/15/0 | 40/0/8/0 | 100% | +| 1.0 | 0.25 | 0.65 | 25/0/23/0 | 38/0/10/0 | 40/0/8/0 | 100% | +| 1.0 | 0.30 | 0.25 | 28/9/11/0 | 32/8/8/0 | 32/14/2/0 | 100% | +| 1.0 | 0.30 | 0.45 | 18/8/22/0 | 32/8/8/0 | 33/8/7/0 | 100% | +| 1.0 | 0.30 | 0.65 | 31/8/9/0 | 26/8/14/0 | 32/4/12/0 | 100% | +| 1.0 | 1.00 | 0.25 | 18/8/22/0 | 24/8/16/0 | 40/8/0/0 | 0% | +| 1.0 | 1.00 | 0.45 | 16/16/16/0 | 31/16/1/0 | 24/10/14/0 | 0% | +| 1.0 | 1.00 | 0.65 | 31/8/9/0 | 24/8/16/0 | 24/3/21/0 | 0% | +| 2.0 | 0.20 | 0.25 | 0/0/48/0 | 17/0/31/0 | 26/1/21/0 | 100% | +| 2.0 | 0.20 | 0.45 | 2/0/46/0 | 22/0/26/0 | 33/0/15/0 | 100% | +| 2.0 | 0.20 | 0.65 | 24/8/16/0 | 24/0/24/0 | 32/0/16/0 | 100% | +| 2.0 | 0.25 | 0.25 | 0/9/39/0 | 32/0/16/0 | 21/1/26/0 | 100% | +| 2.0 | 0.25 | 0.45 | 9/16/23/0 | 17/0/31/0 | 32/5/11/0 | 100% | +| 2.0 | 0.25 | 0.65 | 20/0/28/0 | 32/0/16/0 | 40/0/8/0 | 100% | +| 2.0 | 0.30 | 0.25 | 16/16/16/0 | 33/8/7/0 | 32/3/13/0 | 100% | +| 2.0 | 0.30 | 0.45 | 9/8/31/0 | 31/0/17/0 | 37/0/11/0 | 100% | +| 2.0 | 0.30 | 0.65 | 31/0/17/0 | 33/0/15/0 | 48/0/0/0 | 100% | +| 2.0 | 1.00 | 0.25 | 8/8/32/0 | 32/8/8/0 | 35/8/5/0 | 0% | +| 2.0 | 1.00 | 0.45 | 32/0/16/0 | 40/0/8/0 | 40/0/8/0 | 0% | +| 2.0 | 1.00 | 0.65 | 33/8/7/0 | 33/0/15/0 | 40/0/8/0 | 0% | + +: S9E complete 36-scenario structural sensitivity. Exact zeros occur when the two endpoint allocations coincide under the declared ruler-coordinate-window cell. {#tbl-s-structure-scenarios} + +Purpose caps 0.20--0.30 bind in every portfolio, whereas the 1.00 stress cap +never binds. The common-frontier constraint binds in every portfolio. Maximum +single-loan weight rounds to 0.08, 0.04, and 0.02 as the budget rises from USD +0.5 to 1 to 2 million. The grid therefore changes funded concentration rather +than relabeling an inactive parameter. Its baseline scenario reproduces the +active two-ruler bounds exactly. + +## E.4 Allocation granularity diagnostic + + + +The USD 25 floor-with-cash rule covers the complete baseline continuous family: +1,440 monthly portfolios, 143,175 positive source exposures, and all 96 +ruler-coordinate-window endpoint tracks. It changes 2,985 exposures; eight +rounded exposures become zero. Monthly residual cash ranges from USD 25 to USD +75, averaging USD 26.01. + +| Diagnostic | Maximum absolute value | +|---|---:| +| Monthly cash share | 0.0000750 | +| Fifteen-month track cash share | 0.0000333 | +| Status-indexed payoff-rate perturbation | 0.000003448 | +| Default-rate perturbation | 0.000012843 | +| Miscoverage-rate perturbation | 0.000011902 | + +: S9F deterministic USD 25 allocation-granularity diagnostic. Rate differences are proportions; multiplying by 100 gives percentage points. {#tbl-s-allocation-granularity} + +The largest rate change is therefore 0.001284 percentage points. This is small +relative to the reported endpoint hulls and supports use of the continuous +relaxation for this archive and lot size. It does not establish integer +optimality, robustness to another rounding rule, or the outcome of a discrete +reoptimization. + +## E.5 Named-comparator census + +The table counts the sign of the guardrail-minus-point sharp contrast over 72 +window-policy cells per rule. Lower and higher refer to the guardrail. A +crossing contains zero. + +| Comparator | Metric | Lower | Crosses zero | Higher | +|---|---|---:|---:|---:| +| C0 | Payoff | 0 | 54 | 18 | +| C0 | Default | 72 | 0 | 0 | +| C0 | Miscoverage | 15 | 52 | 5 | +| C1 | Payoff | 28 | 40 | 4 | +| C1 | Default | 3 | 35 | 34 | +| C1 | Miscoverage | 3 | 24 | 45 | +| C2 | Payoff | 35 | 31 | 6 | +| C2 | Default | 7 | 58 | 7 | +| C2 | Miscoverage | 5 | 20 | 47 | + +: S10 named-comparator direction census. These descriptive counts are not votes because the comparators define different feasible sets. {#tbl-s-named} + +C0's uniform lower-default direction follows the same-threshold nesting design, +not a neutral economic comparison. C2 satisfies plug-in dominance but still +produces both status-indexed payoff signs and many unresolved contrasts. The +difference is precisely why a comparator label cannot replace an explicit +support. + +## E.6 Exact support envelopes + + + +Over the broad stress support $[0.05,0.12]$, every one of the 216 +window-policy-metric envelopes crosses zero. Within the narrower +development-admissible support: + +| Metric | Guardrail lower | Crosses zero | Guardrail higher | Total | +|---|---:|---:|---:|---:| +| Status-indexed payoff proxy | 6 | 66 | 0 | 72 | +| Terminal default | 0 | 72 | 0 | 72 | +| Funded miscoverage | 0 | 45 | 27 | 72 | + +: S11 exact direction counts within development-admissible support. {#tbl-s-support} + +![Direction of the exact development-admissible comparator envelope. A minus or plus denotes a sign-robust guardrail-minus-point envelope; zero denotes an envelope containing zero. Default contains zero in all 72 cells.](../reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png){#fig-s-envelopes width=96%} + +The six lower-payoff cells occur before W8; in W8, all 27 policy-metric +envelopes cross zero. The W8 geometry change therefore removes the directions +that survive some earlier development supports. It does not prove W8 is +correct or earlier windows are false; it shows the direction is not invariant +over the complete residual specification. + +Default crosses zero in all 72 development-support cells. This is stronger +than reporting disagreement between C0, C1, and C2 because it evaluates the +continuous cap support exactly. It remains scoped to the declared finite +support. + +The 3,067 count refers to distinct scalar cap values; 7,297 refers to +period-specific evaluated cap rows. + +The solver audit evaluates 7,297 point-cap rows across 15 periods, including +2,204 named unique cap-months. The smallest absolute nonbasic reduced cost is +$3.88\times10^{-4}$, so no evaluated basis is classified as near-tied. Of +2,941 primal-degenerate bases, reversed-order reruns produce zero tie-sensitive +rows; maximum allocation distance is $1.45\times10^{-14}$ and maximum absolute +objective difference is $1.72\times10^{-9}$. This establishes deterministic +stability at the evaluated caps, not continuous-frontier uniqueness. + +## E.7 Why no policy is promoted + +The empirical record does not support a winner for three separate reasons. +First, candidate coverage misses its 0.90 target before optimization for both +learners and every residual window. Second, the endpoint sign changes across +the two rulers and three coordinates. Third, named comparators disagree and +exact envelopes eliminate all broad-support directions and all W8 +development-support directions. Selecting .25, one ruler, an earlier window, +one comparator, or one of the six payoff cells after seeing these results would +convert a specification audit into post-result selection. + +# Appendix F: Reproducibility and Active Lineage + +## F.1 Two-phase execution + +The active audit uses two immutable phases. Phase 1 produces learner scores, +residual recipes, label-availability audits, outcome-free geometry, LP solve +records, allocations, and comparator supports. Before writing a freeze it +verifies full budgets, solver status, split-conformal ranks, label retention, +C2 moment matching, exact IDs, and the absence of outcome columns. + +Phase 2 verifies the freeze descriptors, reconstructs the endpoint panel, and +performs exact keyed joins without changing a score, recipe, support, ruler, +coordinate, or allocation. The active primary evaluation contains 1,408 +coverage rows, 5,112 monthly evaluations, 360 aggregates, 221,040 paired sharp +contrasts, and 648 comparator envelopes. Separate complete families cover five +credit learners, endpoint lag, portfolio structure, missingness encoding, and +a second temporal origin. The evidence builder checks hashes, contracts, and +cardinalities before producing 18 tables, three figures in PNG and PDF, and one +paper-facing manifest. Additional outcome-isolated families cover joint +fit-label completion and deterministic allocation granularity. + +## F.2 Replay contract + +The replay contract distinguishes two operations. **Publication replay** runs +the current builder against immutable DVC artifacts, verifies every descriptor, +and regenerates paper tables, figures, and the manifest transactionally. +**Scientific replay** checks out the recorded protocol commit for a lineage and +uses the `uv.lock` hash stored with that lineage; current source code is not +silently substituted for historical scientific code. + +The shared contract requires: + +1. a lineage-specific environment locked by `uv`; +2. immutable, hash-described outcome-free and evaluated roots tracked by 31 + DVC pointers across the primary audit, two-ruler diagnostic, credit controls, + endpoint, structure, missingness, second-origin, fit-label completion, and + allocation-granularity families; +3. a separate freeze and evaluate command with no combined shortcut; +4. exact ID alignment at the outcome join; +5. byte-identical evidence builds; +6. claim-sync tests across the manifest, body, supplement, and generated TeX; +7. full tests, lint, static typing, protected-artifact validation, and an + active implementation/evidence drift gate; and +8. compilation and page-level visual inspection of the body, supplement, and + official IJDS submission PDF. + +Protected historical stages and their extraction manifest are not required to +replay the active evidence and are not overwritten. + +## F.3 Anonymous-review boundary + +Reviewer-facing files omit exact commits, tags, hashes, repository links, and +remote coordinates because they are searchable author fingerprints. An +editor-only claim registry preserves that crosswalk. At acceptance, the code, +environment lock, evidence manifest, immutable pointers, and raw-data +reconstruction instructions can be released under the journal's data and code +policy. + +# Appendix G: Claim Boundary and Limitations + + + + + + +| Evidence object | Permitted conclusion | Not established | |---|---|---| -| A1--A2 | Predictive benchmarks and the historical robustness frontier. | Upstream model and provenance context. | -| A3--A11 | Nested/temporal holdouts, sensitivity, synthetic shift, funded-set loan audit, and exact finalist checks. | Robustness diagnostics; no active hyperparameter selection. | -| A12--A18 | OCE/CVaR, satisficing, dependence, leave-period-out, bootstrap, budget/LGD, and policy-family audits. | Historical mechanism checks. | -| A19 | Synthetic regret-auditability comparison with two-stage and SPO+. | Comparator only; not the `$1M` funded portfolio. | -| A20--A24 | Tail, concentration, multi-distribution, and online-style diagnostics. | Stress evidence; no new conformal guarantee. | -| A25--A34 | Prosper and Freddie/Mendeley external analyses. | Static transfer evidence; not active Lending Club certificates. | - -The external applications are useful because they cover marketplace personal -loans and single-family mortgages. They use older frozen replication contracts, -not the final nine-cell selector. They therefore support plausibility of the -general PD-to-conformal-to-allocation workflow but cannot be quoted as direct -replications of the active midpoint policy. - -Similarly, the synthetic SPO+ experiment answers a different question. It -shows that decision-focused training reduces regret on its own synthetic -optimization task [@elmachtoub2022], whereas the active CRPTO result evaluates -an uncertainty-constrained real-dollar allocation. Combining those values into -one leaderboard would be misleading. - -# Appendix D: Reproducibility Protocol - -## D.1 Active Commands - -The paper-facing evidence is regenerated from isolated experiment outputs: - -```powershell -just ijds-evidence -uv run pytest tests/test_ijds_active_claim_sync.py -q -just paper-submission -just paper-submission-official -just validate-champion -``` - -The stronger methodology replay is explicit and writes only to versioned -experiment paths: - -```powershell -just ijds-active-replay -``` - -It is not part of ordinary manuscript rendering because it solves portfolios -and recomputes exact interval grids. The frozen upstream model, calibrator, -historical intervals, and manifest are never overwritten. - -## D.2 Artifact Lineage - -| Object | Active source | -|---|---| -| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | -| Selector grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | -| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | -| Funded allocations | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | -| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | -| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | - -The active run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. -The exact-alpha run is -`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. - -## D.3 Claim-Sync Contract - -`tests/test_ijds_active_claim_sync.py` verifies: - -1. policy mode, $\tau$, $\gamma$, alpha, selector size, and forbidden columns; -2. agreement between governance and A35/A40; -3. presence of every A35--A40 CSV and TeX file; -4. numeric anchors across body, supplement, and official submission TeX; -5. absence of retired headline values from active manuscript surfaces. - -Historical manifest tests remain separate. This separation lets the submitted -claim evolve without rewriting or silently reinterpreting frozen provenance. - -# Appendix E: Limitations and Claim Crosswalk - -| Claim | Direct evidence | What is not claimed | +| Full raw-file audit | The 640,543 active rows exhaust the declared eligible population | That all 2.9 million rows share one horizon, schema, or estimand | +| Reconstructed endpoint | 364,814 primary outcomes are resolved by the declared cutoff and 12,076 are bounded as unresolved | That the distributed file is a verified point-in-time snapshot | +| Endpoint-reason partition | The five reasons sum exactly to all 376,890 candidates, including 47 terminal-after-cutoff and 478 missing-date cases | An identified missingness mechanism or observed operational event dates | +| Eight windows and five learners | Under the active six-month endpoint, candidate coverage upper bounds are below 0.90 in all 40 declared cells | Universal conformal invalidity, endpoint-lag invariance, an OOT-selected winner, or future performance | +| WOE/IV, monotonic, and PSI controls | The transport result is not unique to active CatBoost or platform-pricing features | Scorecard superiority, external validity, or PSI-based coverage certification | +| Three missingness encodings | All 24 coverage upper bounds are below 0.90 | A missingness mechanism, encoding winner, or portfolio robustness | +| Second temporal origin | All eight later-origin CatBoost upper bounds are below 0.90 | Independent replication, temporal invariance, or prospective validation | +| Four fit-label scenarios | All 32 overall coverage upper bounds are below 0.90; the W7--W8 crossing remains in three scenarios | Sharp bounds over all label assignments, geometry invariance, or preregistration | +| Constant-score proposition and stratum-2 path | Binary residual geometry is prevalence-sensitive | Exact finite-sample explanation for varying-score strata or a universal transition | +| Fit-label sensitivity | The W7--W8 crossing persists at every lag passing the 99% retention rule | Invariance to arbitrarily delayed fit labels or a causal explanation | +| Evaluation-endpoint sensitivity | Coverage is below 0.90 in 40/40 cells at lags 0, 3, 6, and 8 and in 39/40 at lag 12; all broad-support envelopes cross at every lag | Lag-universal coverage failure or an endpoint selected from outcomes | +| Timing design | Fit-label and evaluation-endpoint lags are separate one-factor sensitivities | Joint or full-factorial lag robustness | +| Two rulers and three interior coordinates | Endpoint direction is not ruler- or coordinate-invariant on the declared grid | A continuous-frontier ordering or preferred gamma/ruler/coordinate | +| Complete portfolio-structure grid | Every scenario retains adverse default and miscoverage cells; zero scenarios are uniformly favorable or adverse | A scenario winner, structural invariance, universal harm, or deployment guidance | +| USD 25 floor-with-cash diagnostic | Rate perturbations are at most 0.001284 percentage points in the complete baseline family | Integer optimality, discrete reoptimization, or another lot rule | +| C0 nesting and 1,080 C2 reconciliations | Plug-in objective dominance under their stated cap constructions | Status-indexed payoff, default, or miscoverage dominance | +| Exact declared supports | Portfolio signs depend on comparator support and residual window | Universal quantification over all comparators | +| Evaluated-cap solver audit | Allocations are deterministic to tolerance at the 7,297 evaluated rows | Continuous joint-frontier uniqueness | +| Sharp unresolved-outcome bounds | Finite-archive binary completion uncertainty is exposed | Sampling uncertainty or missing-at-random identification | +| Identification-width identity | Bound width decomposes exactly over unresolved exposure disagreement | Sampling uncertainty, causal uncertainty, or a missingness model | + +: S12 active claim boundary. {#tbl-s-claims} + +The study uses one discontinued consumer-credit platform and accepted 36-month +loans only. It observes no counterfactual outcomes for rejected applicants. +The terminal label compresses payment paths and competing events into one +binary endpoint. The status-indexed payoff proxy omits timing, recovery, fees, +discounting, +and capital costs. The temporal gap does not restore exchangeability. Numeric +logistic, monotonic CatBoost, and both WOE/IV learners are coverage controls, +not four additional portfolio systems. + +The raw archive also contains 60-month contracts, duration-censored cohorts, +and 48 late-schema fields. Including them would change the target or temporal +information set. The controls share one platform and their OOT differences +cannot support promotion because the same outcomes define this audit. + +The archive is not a verified historical snapshot. Servicing dates after the +cutoff require a conservative reconstruction that cannot recover the missing +point-in-time extract. The six-month Charged Off delay is a modeling assumption +rather than an observed operational date. The retrospectively locked fit-label +analysis supports only lags satisfying strict greater-than-99% retention; the +separate endpoint analysis has one upper-coverage exception at 12 months. They +were not crossed factorially. Delinquency recency is structurally nullable and +bankruptcy count has partial early schema support; deterministic frozen +mappings and two alternative encodings are evaluated, but the missingness +mechanism is not identified. The second origin remains a retrospective +recurrence from the same platform, not prospective validation. + +The observed-only fit and three declared stress rules retain the coverage +conclusion but not one common geometric path. The all-default scenario removes +the W7--W8 crossing, and the four nonlinear refits are not exhaustive over +every missing-label assignment. Continuous allocation is numerically stable +under the declared USD +25 floor rule, but the study does not solve or evaluate a mixed-integer policy. + +Candidate-level coverage is not selected-set coverage. C2 matches one funded +point-score moment but neither equalizes feasible sets nor defines a deployable +counterfactual. Purpose categories and score strata are not protected classes, +so the audit is not a fair-lending assessment. The overlapping windows are not +independent evidence, and the constant-score proposition is only a mechanism +for empirical strata whose scores vary. + +The two-ruler diagnostic contains three interior coordinates, not a continuous +joint frontier. Coordinate one is a structural null. Objective matching uses a +model-implied payoff, while normalized-score matching does not equalize its +opportunity cost. Eleven common development menus and fifteen OOT decisions do +not satisfy the decision-context requirements needed to import a CROMS, +inverse conformal risk-control, or decision-calibration guarantee. + +The active conclusion is deliberately narrow. Binary absolute-residual +geometry can change discontinuously around the prevalence threshold, but that +path is not invariant to every fit-label completion. Declared candidate-level +coverage does not transport to the OOT archive for all five learner +specifications, and status-indexed portfolio direction changes with the declared +ruler, coordinate, residual recipe, and comparator support. No selected gamma, +ruler, coordinate, policy, causal effect, prospective guarantee, selected-set +validity, Markov certificate, universal guardrail direction, or deployment +recommendation is claimed. + +# Appendix H: Closest-Work Positioning + +CRPTO combines established ingredients but asks a different identification +question. The matrix below records the boundary that is summarized in the main +paper; it is positioning evidence, not an empirical result or novelty claim for +credit scoring, conformal prediction, or linear programming separately. + +| Literature family | Established contribution | Active CRPTO boundary | |---|---|---| -| Exact 90% replay | Reference endpoints match to `6.67e-16`; A35. | Independent tuning of every alpha sensitivity row. | -| Outcome-free, assumption-free policy ranking, conditional on the frozen recipe | A36 physically separated outcomes, deterministic endpoint cap, cap-stability interval, and December replay. | A label-free conformal recipe, historically untouched OOT corpus, or preregistration. | -| Independent calibration decision audit | December reselects the midpoint before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | Selected-set conformal validity or a prospective deployment guarantee. | -| Full-OOT return-risk trade-off | A37 and A40. | Causal effect or universal dominance. | -| Funded-set accounting | Proposition A.1 and exact row-level allocation. | Nominal selected-set conformal coverage. | -| Conditional Markov sensitivity | Corollary A.1 under weighted validity. | Deterministic risk cap or sharp tail guarantee. | -| Composition transparency | A38 letter-grade reconciliation. | Legal fair-lending certification. | -| Contribution stability | A39 month-cluster bootstrap with funded-loan sensitivity. | Full pipeline uncertainty or model-selection confidence interval. | -| Reproducibility | Commands, run tags, hashes, and sync tests. | Cross-machine bit-identical model retraining. | - -The active result is intentionally narrow. Optimized OCE/CVaR, online or -non-exchangeable conformal recalibration [@farinhas2024nonexchangeable_crc], -formal valid selection among conformal sets -[@hegazy2025valid_selection_conformal_sets], causal allocation, and live -multi-period deployment would require new protocols and are not hidden -acceptance criteria for this paper. +| Credit scoring and cost-aware calibration [@lessmann2015; @yang2025costaware; @das2023creditgraph] | Probability quality and richer predictive structure | Treats a Platt-scaled default score as an input; no AUC-leadership claim | +| P2P and robust credit portfolios [@serrano2016profitscoring; @chi2019p2p; @babaei2020p2p] | Economic loan selection under risk and uncertainty | Adds binary-set geometry, outcome isolation, and comparator identification | +| Conformal portfolio and contextual optimization [@kato2025; @sun2024ptc; @johnstone2021; @patel2024; @hu2026crc] | Coverage-backed uncertainty sets or calibrated decision guarantees for portfolio and contextual programs | Audits a frozen binary credit score, endpoint reconstruction, and matched comparators; no asset-return, contextual-LP, or selected-set guarantee is inherited | +| Decision-optimal and counterfactual prediction sets [@wang2026optimal_decision_prediction_sets; @zheng2026counterfactual_decision_sets] | Couple set construction or coverage to an explicit downstream loss or induced action | Audits an observational accepted-loan archive; no counterfactual funding effect or policy-coupled validity | +| Valid selection and decision-risk calibration [@hegazy2025valid_selection_conformal_sets; @bao2025croms; @yeh2025training; @zhou2026creme] | Procedures controlling model selection, operational loss, or robustness | Uses ruler sensitivity as a diagnostic; eleven development menus do not support those certificates | +| Decision-focused learning [@donti2017; @elmachtoub2022; @mandi2024] | Training predictions against downstream loss | Preserves the predictor for governance and audits a post-hoc decision layer | +| Baseline and ablation design | Holding the decision problem fixed while changing one ingredient | Shows that a shared numeric cap does not hold decision stringency fixed | + +: S13 closest-work boundary for the active CRPTO contribution. {#tbl-s-closest-work} diff --git a/pyproject.toml b/pyproject.toml index c4a24f9..33fd592 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "crpto" version = "0.1.0" -description = "CRPTO — Conformal Robust Predict-Then-Optimize: conformal prediction + robust portfolio optimization, applied to credit risk (Lending Club 2007-2020). Standalone research pipeline and Quarto book accompanying the paper." +description = "CRPTO - Conformal Risk-Aware Predict-Then-Optimize: an auditable study of binary conformal geometry, temporal transport, and portfolio comparator design in historical credit data." readme = "README.md" requires-python = ">=3.11,<3.13" license = { text = "MIT" } @@ -11,15 +11,12 @@ keywords = [ "probability-of-default", "conformal-prediction", "predict-then-optimize", - "robust-optimization", + "comparator-design", + "decision-audit", "portfolio-optimization", - "model-risk-management", - "fairness", "lending-club", "catboost", - "mapie", - "pyomo", - "quarto", + "partial-identification", ] classifiers = [ "Development Status :: 4 - Beta", @@ -37,71 +34,32 @@ classifiers = [ ] dependencies = [ - # Core data & numerics — floors aligned with uv.lock effective versions "pandas>=2.3,<4", "numpy>=2.0,<3", "pyarrow>=18", "scipy>=1.14,<2", "scikit-learn>=1.6,<2", - "joblib>=1.5", - "pandera>=0.25", - "tqdm>=4.66", - # ML / Conformal / Calibration - "catboost>=1.2,<1.3", # PIN — champion reproducibility - "optuna>=4.0", - "mapie>=1.4", - "fairlearn>=0.12", - "betacal>=1.1", - "venn-abers>=1.5", - "skops>=0.12", + "catboost>=1.2,<1.3", "optbinning>=0.21", - # Optimization "pyomo>=6.10", "highspy>=1.10", - "ortools>=9.10", - # Data orchestration & MLOps - "dvc[s3]>=3.60", - "mlflow>=3.10", - "dagshub>=0.7", + "ortools>=9.10,<9.12", # OptBinning 0.21 requires OR-Tools <9.12. "duckdb>=1.3", - "dbt-duckdb>=1.10", - # Visualization + "dvc[s3]>=3.60", "matplotlib>=3.10", - "seaborn>=0.13", - "plotly>=6.0", - # Notebooks / Jupyter - "nbformat>=5.10", - "nbclient>=0.10", - "ipykernel>=6.29", - # Runtime & config "pyyaml>=6.0", - "python-dotenv>=1.0", "loguru>=0.7", - # Testing baseline (also exposed in dev extra) - "pytest>=9.0", ] [project.optional-dependencies] dev = [ + "pytest>=9.0", "pytest-cov>=7.0", "ruff>=0.10", "mypy>=1.13", "hypothesis>=6.115", - "nbstripout>=0.9", - "jupytext>=1.19", - "jupyter>=1.1", "pre-commit>=4.0", - "optuna-integration>=4.0", -] -search = [ - "optuna-integration>=4.0", -] -spo = [ - "pyepo>=1.0", - "torch>=2.5", -] -docs = [ - "jupyterlab>=4.5", + "pypdf>=6,<7", ] [project.urls] @@ -109,14 +67,13 @@ Homepage = "https://github.com/EigenCharlie/Paper_CRPTO" Repository = "https://github.com/EigenCharlie/Paper_CRPTO" Issues = "https://github.com/EigenCharlie/Paper_CRPTO/issues" Documentation = "https://github.com/EigenCharlie/Paper_CRPTO#readme" -Changelog = "https://github.com/EigenCharlie/Paper_CRPTO/blob/main/CHANGELOG.md" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["src", "crpto"] +packages = ["src"] # --------------------------------------------------------------------------- # Ruff @@ -125,13 +82,11 @@ packages = ["src", "crpto"] target-version = "py311" line-length = 100 extend-exclude = [ - "book/_book", - "book/_freeze", - "book/.quarto", ".venv", "data", "models", "reports", + "scripts/search", ] [tool.ruff.lint] @@ -173,13 +128,10 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["T20", "B011", "PT011"] -"notebooks/**/*.py" = ["T20", "B018"] -"notebooks/**/*.ipynb" = ["T20", "B018"] "scripts/**/*.py" = ["T20"] -"book/_helpers/**" = ["T20"] [tool.ruff.lint.isort] -known-first-party = ["crpto", "src"] +known-first-party = ["src"] combine-as-imports = true [tool.ruff.format] @@ -199,7 +151,6 @@ markers = [ ] filterwarnings = [ "ignore::DeprecationWarning:catboost.*", - "ignore::DeprecationWarning:pandera.*", "ignore::PendingDeprecationWarning", ] @@ -209,7 +160,7 @@ filterwarnings = [ [tool.coverage.run] branch = true source = ["src"] -omit = ["tests/*", "scripts/*", "notebooks/*"] +omit = ["tests/*", "scripts/*"] [tool.coverage.report] exclude_lines = [ @@ -226,92 +177,128 @@ skip_covered = false # --------------------------------------------------------------------------- [tool.mypy] python_version = "3.11" -files = ["src", "scripts"] +files = ["src", "scripts", "tests"] +exclude = [ + "^scripts/analyze_crpto_evidence\\.py$", + "^scripts/backtest_conformal_coverage\\.py$", + "^scripts/build_crpto_journal_package\\.py$", + "^scripts/build_tail_satisficing_challenger_audit\\.py$", + "^scripts/export_crpto_tables\\.py$", + "^scripts/generate_conformal_intervals\\.py$", + "^scripts/generate_crpto_figures\\.py$", + "^scripts/materialize_feature_artifacts\\.py$", + "^scripts/optimize_portfolio\\.py$", + "^scripts/optimize_portfolio_tradeoff\\.py$", + "^scripts/run_crpto_vs_spo_stability\\.py$", + "^scripts/search/", + "^scripts/train_pd_model\\.py$", + "^scripts/validate_conformal_experiment\\.py$", + "^scripts/validate_conformal_policy\\.py$", + "^src/data/make_dataset\\.py$", + "^src/data/prepare_dataset\\.py$", + "^src/optimization/tail_satisficing_objective\\.py$", +] strict_optional = true warn_redundant_casts = true warn_unused_ignores = true warn_unreachable = true -disallow_untyped_defs = false # gradual — flip to true later +# Compatibility modules remain on typed-body checking; the active modules +# listed below are fully strict. +disallow_untyped_defs = false check_untyped_defs = true follow_imports = "silent" show_error_codes = true pretty = true -# Suppress import-untyped (mypy 1.x) globally: most third-party scientific libs -# (pandas, sklearn, mlflow, ...) don't ship stubs and we don't want each call -# site littered with `# type: ignore[import-untyped]`. +# Suppress import-untyped globally because several scientific dependencies do +# not ship complete stubs; runtime contracts cover their integration points. disable_error_code = ["import-untyped"] [[tool.mypy.overrides]] module = [ "catboost.*", - "mapie.*", - "venn_abers.*", - "betacal.*", "optbinning.*", - "skops.*", - "fairlearn.*", "pyomo.*", "highspy.*", "ortools.*", "dvc.*", - "dagshub.*", - "pyepo.*", - "shap.*", - "plotly.*", "pandas.*", - "pandera.*", - "mlflow.*", "duckdb.*", "loguru.*", - "torch", - "torch.*", - "tabpfn", - "tabpfn.*", ] ignore_missing_imports = true -[[tool.mypy.overrides]] -module = [ - "scripts.run_spo_comparison", - "scripts.run_spo_real", - "scripts.run_crpto_vs_spo_stability", - "scripts.experiments.run_tabpfn_tabprep_full", -] -disable_error_code = ["import-not-found"] - -# Strict mode for the fully-annotated modules. Legacy code stays under the -# global laxer config. Promote modules to this list once they pass cleanly. -# Modules in this list have been made fully typed enough for strict checks. +# Strict mode for the fully annotated current modules. Protocol-frozen sources +# not listed here still pass the global typed-body gate without source churn. [[tool.mypy.overrides]] module = [ "src.optimization.policy", + "src.optimization.policy_evaluation", + "src.optimization.policy_selection", "src.optimization.cuopt_adapter", "src.optimization.portfolio_model", - "src.optimization.robust_opt", - "src.optimization.tail_satisficing_objective", - "src.models.calibration", - "src.models.conformal", - "src.models.conformal._scores", - "src.models.conformal.classification", - "src.models.conformal.pd_intervals", - "src.models.conformal.regression", - "src.models.conformal_adapters", - "src.models.conformal_artifacts", - "src.models.conformal_diagnostics", - "src.models.conformal_tuning", - "src.models.optuna_tuning", - "src.models.pd_contract", - "src.models.pd_model", - "src.models.pd_pipeline", - "src.models.venn_abers", - "src.utils.pipeline_state", - "src.utils.pipeline_topology", - "src.utils.mlflow_tracing", - "src.utils.optuna_storage", - "src.utils.script_helpers", + "src.models.binary_conformal_guardrail", + "src.data.outcome_observability", + "src.ijds_audit.config", + "src.ijds_audit.allocation_granularity", + "src.ijds_audit.claim_ledger", + "src.ijds_audit.evaluation", + "src.ijds_audit.fit_label_sensitivity", + "src.ijds_audit.geometry", + "src.ijds_audit.grid_contracts", + "src.ijds_audit.policy_support", + "src.ijds_audit.portfolio", + "src.ijds_audit.publication_generation", + "src.ijds_audit.publication_sources", + "src.ijds_audit.protocol", + "src.ijds_audit.robustness_evidence", + "src.ijds_audit.sensitivity_evidence", + "src.ijds_audit.simulation", + "src.ijds_audit.structural_evidence", + "src.ijds_challengers.archive", + "src.ijds_challengers.config", + "src.ijds_challengers.evaluation", + "src.ijds_challengers.evaluation_config", + "src.ijds_challengers.frontier", + "src.ijds_challengers.normalized_frontier", + "src.evaluation.maturity_safe_portfolio", + "scripts.build_ijds_binary_geometry_frontier_v4_evidence", + "scripts.build_ijds_policy_support_tie_evidence", + "scripts.build_ijds_submission_tex", + "scripts.check_publication_integrity", + "scripts.compile_ijds_submission", + "scripts.inspect_ijds_pdfs", + "scripts.run_ty_advisory", + "scripts.experiments.run_ijds_binary_geometry_frontier_v4", + "scripts.experiments.run_ijds_allocation_granularity_sensitivity", + "scripts.experiments.run_ijds_fit_label_completion_sensitivity", + "scripts.experiments.run_ijds_normalized_objective_frontier", + "scripts.experiments.run_ijds_normalized_objective_frontier_v2", ] disallow_untyped_defs = true disallow_incomplete_defs = true disallow_untyped_decorators = true warn_return_any = true no_implicit_optional = true + +# V1c source is protocol-hashed. ty's pandas stubs infer the final Series.any() +# reduction as Series | bool, while mypy strict and runtime tests verify it. +[[tool.ty.overrides]] +include = ["src/ijds_challengers/evaluation.py"] + +[tool.ty.overrides.rules] +unresolved-attribute = "ignore" + +# These active sources are implementation-hashed in the publication evidence. +# Pandas stubs widen groupby scalar keys to Hashable and Timestamp.date() to +# date | NaTType; runtime contracts and strict mypy checks cover both paths. +[[tool.ty.overrides]] +include = ["scripts/experiments/run_ijds_label_lag_sensitivity.py"] + +[tool.ty.overrides.rules] +invalid-argument-type = "ignore" + +[[tool.ty.overrides]] +include = ["src/ijds_audit/raw_data_audit.py"] + +[tool.ty.overrides.rules] +unresolved-attribute = "ignore" diff --git a/reports/crpto/appendix/README.md b/reports/crpto/appendix/README.md deleted file mode 100644 index 655541e..0000000 --- a/reports/crpto/appendix/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# CRPTO Appendix Ledgers - -This folder stores static, curated CSV ledgers imported during the final parent -intake. They are appendix evidence only. - -Do not treat these files as paper-facing generated tables. The canonical table -surface remains `reports/crpto/tables/`, generated by -`scripts/export_crpto_tables.py` and protected by regression tests. diff --git a/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_conformal_group_diagnostics_2026-05-21.csv b/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_conformal_group_diagnostics_2026-05-21.csv deleted file mode 100644 index ad1be50..0000000 --- a/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_conformal_group_diagnostics_2026-05-21.csv +++ /dev/null @@ -1,8 +0,0 @@ -group,n,coverage_90,avg_width_90,median_width_90,coverage_95,avg_width_95,median_width_95,coverage_before,coverage_after,multiplier,adjusted,cov90_gap_to_0p90,cov95_gap_to_0p95,grade_gate,parent_action -A,72388,0.9094601315135106,0.23040531343709056,0.21159875239017956,0.945833563574073,0.7417439856402007,0.7347888140114475,0.9043393466601658,0.9116528522671867,1.2,True,0.00946013151351055,-0.004166436425926956,watch,retain_as_supporting_evidence -B,75311,0.9343654977360545,0.90836938568326,0.9886194314716429,0.9549999336086362,0.9445130278732093,1.0,0.9095217883418223,0.9233870967741935,1.05,True,0.03436549773605446,0.0049999336086362645,pass,retain_as_supporting_evidence -C,70568,0.9254903072214035,0.9422662507822753,1.0,0.9467464006348486,0.9629989925826444,1.0,0.9222002374356945,0.9222002374356945,1.0,False,0.02549030722140344,-0.0032535993651513495,watch,retain_as_supporting_evidence -D,45637,0.9280846681420777,0.9576071026976826,1.0,0.9497556806976795,0.9682007279133129,1.0,0.932987012987013,0.932987012987013,1.0,False,0.028084668142077662,-0.0002443193023204948,watch,retain_as_supporting_evidence -E,10828,0.8883450314000739,0.9461195935514686,1.0,0.9019209456963428,0.9530176817040606,1.0,0.93,0.93,1.0,False,-0.011654968599926097,-0.04807905430365711,watch,targeted_followup -F,1731,0.9399191218948585,0.9737148719870481,1.0,0.9399191218948585,0.9737148719870481,1.0,0.9487666034155597,0.9487666034155597,1.0,False,0.03991912189485847,-0.010080878105141466,watch,targeted_followup -G,406,0.9113300492610837,0.9697590588932684,1.0,0.9236453201970444,0.9741057219794047,1.0,0.9207317073170732,0.9207317073170732,1.0,False,0.011330049261083719,-0.026354679802955583,watch,targeted_followup diff --git a/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_pd_intake_2026-05-21.csv b/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_pd_intake_2026-05-21.csv deleted file mode 100644 index c5aff4a..0000000 --- a/reports/crpto/appendix/crpto_appendix_paper1_bound_improvement_pd_intake_2026-05-21.csv +++ /dev/null @@ -1,5 +0,0 @@ -role,rank,lane_id,feature_profile,monotonic_policy,auc_roc,brier_score,ece,hpo_best_validation_auc,delta_auc_vs_incumbent,delta_brier_vs_incumbent,delta_ece_vs_incumbent,parent_decision -incumbent_replay,0,incumbent__frozen_champion,incumbent,canonical_4,0.7126777845551742,0.1545907367602431,0.006152293607422043,0.7220014049345107,0.0,0.0,0.0,frozen_reference -challenger,1,full_challenger_woe__bureau_behavior_15,full_challenger_woe,bureau_behavior_15,0.7206787352232472,0.15316074834071589,0.007688869061412672,0.7280827433559496,0.008000950668073048,-0.0014299884195272117,0.001536575453990629,main_challenger -challenger,2,full_challenger__canonical_4,full_challenger,canonical_4,0.7206242320060079,0.1531819498978701,0.005916541211478011,0.7280258024895705,0.007946447450833727,-0.0014087868623730015,-0.00023575239594403156,sensitivity_baseline -challenger,3,full_challenger_woe__affordability_rate_5,full_challenger_woe,affordability_rate_5,0.7200520927122714,0.15327620018308713,0.007501719650255938,0.7282488050115092,0.007374308157097276,-0.001314536577155967,0.0013494260428338952,sensitivity_baseline diff --git a/reports/crpto/appendix/crpto_appendix_paper1_bound_pareto_decision_summary_2026-05-25.csv b/reports/crpto/appendix/crpto_appendix_paper1_bound_pareto_decision_summary_2026-05-25.csv deleted file mode 100644 index 3e048ce..0000000 --- a/reports/crpto/appendix/crpto_appendix_paper1_bound_pareto_decision_summary_2026-05-25.csv +++ /dev/null @@ -1,74 +0,0 @@ -candidate_family,run_label,tier,decision_read,candidate_rank,shortlist_bucket,policy_mode,risk_tolerance,gamma,delta_cap_quantile,tail_focus_quantile,uncertainty_aversion,realized_total_return,return_delta_vs_champion,alpha01_exact_pass,alpha01_weighted_miscoverage_V,alpha01_gamma_cp,alpha01_violation,alpha01_empirical_coverage_funded,n_funded -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_V_exact_pass;best_return_exact_pass,append_or_park_no_champion_case,188,forced_incumbent_neighbors,blended_uncertainty,0.19,0.375,1.0,1.0,0.1,146569.73552904688,-23894.80739941582,True,0.038528,0.251736,0.0,0.9651,252.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_V_exact_pass;best_return_exact_pass,append_or_park_no_champion_case,189,forced_incumbent_neighbors,capped_blended_uncertainty,0.19,0.375,1.0,1.0,0.1,146569.73552904688,-23894.80739941582,True,0.038528,0.251736,0.0,0.9651,252.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_V_exact_pass;best_return_exact_pass,append_or_park_no_champion_case,153,forced_incumbent_neighbors,blended_uncertainty,0.185,0.375,1.0,1.0,0.1,144310.75016481982,-26153.79276364288,True,0.048775,0.242933,0.0,0.9646,250.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,106,forced_incumbent_neighbors,blended_uncertainty,0.18,0.55,1.0,1.0,0.1,127092.17342211276,-43372.36950634995,True,0.060775,0.181286,0.0,0.9603,244.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,107,forced_incumbent_neighbors,capped_blended_uncertainty,0.18,0.55,1.0,1.0,0.1,127092.17342211276,-43372.36950634995,True,0.060775,0.181286,0.0,0.9603,244.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,38,forced_incumbent_neighbors,blended_uncertainty,0.17,0.5,1.0,1.0,0.1,125243.30910481798,-45221.233823644725,True,0.060775,0.180306,0.0,0.9602,243.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,217,incumbent_region,capped_blended_uncertainty,0.165,0.55,0.75,1.0,0.25,118364.30346496968,-52100.239463493024,True,0.059775,0.160508,0.0,0.9643,251.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,214,incumbent_region,capped_blended_uncertainty,0.165,0.55,0.9,1.0,0.25,118096.78227316157,-52367.76065530113,True,0.059775,0.160508,0.0,0.9643,251.0 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,211,incumbent_region,capped_blended_uncertainty,0.165,0.55,0.95,1.0,0.25,118046.38664266416,-52418.15628579854,True,0.059775,0.160508,0.0,0.9643,251.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,append_or_park_no_champion_case,30,return_global,segment_relative_tail_blended_uncertainty,0.16,0.325,1.0,0.95,0.25,166361.201522614,-4103.341405848711,True,0.0831,0.340404,0.0,0.928,245.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,append_or_park_no_champion_case,32,return_global,segment_relative_tail_blended_uncertainty,0.165,0.375,1.0,0.95,0.25,166208.00531537604,-4256.537613086664,True,0.08614,0.342942,0.0,0.919,248.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,append_or_park_no_champion_case,34,return_global,segment_relative_tail_blended_uncertainty,0.18,0.325,1.0,0.9,0.25,166087.2873129657,-4377.255615497008,True,0.06479,0.337588,0.0,0.938,240.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,append_or_park_no_champion_case,100,return_global,segment_relative_tail_blended_uncertainty,0.17,0.325,1.0,0.9,0.25,161931.01624283788,-8533.52668562482,True,0.056109,0.31568,0.0,0.9417,238.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,append_or_park_no_champion_case,213,forced_incumbent_neighbors,blended_uncertainty,0.165,0.375,1.0,1.0,0.1,131951.73566406107,-38512.807264401636,True,0.057775,0.210692,0.0,0.9634,250.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,append_or_park_no_champion_case,214,forced_incumbent_neighbors,capped_blended_uncertainty,0.165,0.375,1.0,1.0,0.1,131951.73566406107,-38512.807264401636,True,0.057775,0.210692,0.0,0.9634,250.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,238,forced_incumbent_neighbors,blended_uncertainty,0.17,0.5,1.0,1.0,0.1,125243.30910481798,-45221.233823644725,True,0.060775,0.180306,0.0,0.9602,243.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,239,forced_incumbent_neighbors,capped_blended_uncertainty,0.17,0.5,1.0,1.0,0.1,125243.30910481798,-45221.233823644725,True,0.060775,0.180306,0.0,0.9602,243.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,205,forced_incumbent_neighbors,blended_uncertainty,0.165,0.475,1.0,1.0,0.1,124514.20710513435,-45950.33582332835,True,0.060775,0.180035,0.0,0.9598,244.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,116,conservative_proxy,capped_blended_uncertainty,0.16,0.6,0.75,1.0,0.25,114470.3948542467,-55994.148074216006,True,0.065075,0.145439,0.0,0.9606,252.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,104,conservative_proxy,capped_blended_uncertainty,0.16,0.6,0.9,1.0,0.25,114160.31987412396,-56304.22305433874,True,0.065075,0.145439,0.0,0.9606,252.0 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,103,conservative_proxy,capped_blended_uncertainty,0.16,0.6,0.95,1.0,0.25,114136.28072668929,-56328.26220177341,True,0.065075,0.145439,0.0,0.9606,252.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,102,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.2,1.0,0.75,0.0,142285.958913793,-28178.5840146697,True,0.099125,0.315162,0.0,0.9014,205.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,78,incumbent_region,segment_relative_tail_blended_uncertainty,0.18,0.5,1.0,0.75,0.1,141035.29040580042,-29429.252522662282,True,0.094275,0.303643,0.0,0.9086,182.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,77,incumbent_region,segment_tail_blended_uncertainty,0.18,0.5,1.0,0.75,0.1,140794.0726014238,-29670.4703270389,True,0.058075,0.376037,0.0,0.9189,180.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,10,incumbent_region,segment_tail_blended_uncertainty,0.165,0.5,1.0,0.75,0.05,130910.71940319923,-39553.82352526348,True,0.053075,0.378561,0.0,0.9279,184.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,106,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.05,129909.14187443744,-40555.401054025264,True,0.053075,0.369598,0.0,0.9286,185.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,2,incumbent_region,segment_relative_tail_blended_uncertainty,0.165,0.5,1.0,0.75,0.2,128383.14671652296,-42081.39621193975,True,0.078575,0.268697,0.0,0.9246,190.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,85,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.1,128042.57760311202,-42421.965325350684,True,0.079261,0.269515,0.0,0.9231,186.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,36,incumbent_region,segment_tail_blended_uncertainty,0.17,0.5,1.0,0.75,0.05,126239.02096529321,-44225.52196316949,True,0.053075,0.385396,0.0,0.9279,182.0 -affordability_rate_5_segment_probe,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,82,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.2,126126.8791351281,-44337.66379333461,True,0.078575,0.2623,0.0,0.9235,189.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,74,incumbent_region,segment_relative_tail_blended_uncertainty,0.18,0.5,1.0,0.75,0.1,121607.06434274356,-48857.478585719145,True,0.095275,0.277982,0.0,0.9202,178.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_V_exact_pass;best_return_exact_pass,append_or_park_no_champion_case,5,incumbent_region,segment_relative_tail_blended_uncertainty,0.165,0.5,1.0,0.75,0.0,121605.80374969145,-48858.739178771255,True,0.082725,0.279439,0.0,0.9333,154.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,77,incumbent_region,segment_relative_tail_blended_uncertainty,0.18,0.5,1.0,0.75,0.0,120989.12725236984,-49475.41567609286,True,0.08575,0.304108,0.0,0.9293,168.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,94,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.0,117298.0892525555,-53166.45367590721,True,0.082725,0.271946,0.0,0.9322,158.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,52,incumbent_region,segment_relative_tail_blended_uncertainty,0.175,0.5,1.0,0.75,0.05,116108.95945221599,-54355.58347624671,True,0.082725,0.285166,0.0,0.9344,173.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,85,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.1,112524.2039773839,-57940.3389510788,True,0.08305,0.24469,0.0,0.9282,170.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,49,incumbent_region,segment_relative_tail_blended_uncertainty,0.175,0.5,1.0,0.75,0.2,93455.31449074653,-77009.22843771617,True,0.08345,0.249851,0.0,0.9267,181.0 -bureau_behavior_15_segment_probe,paper1_bound_expansion_2026_05_24_r1__bureau_behavior_15__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,25,incumbent_region,segment_relative_tail_blended_uncertainty,0.17,0.5,1.0,0.75,0.2,89905.8873782091,-80558.6555502536,True,0.096885,0.241321,0.0,0.9162,179.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_exact_pass;best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,118,forced_incumbent_neighbors,blended_uncertainty,0.18,0.375,1.0,1.0,0.1,160346.21021790063,-10118.33271056207,True,0.031125,0.233621,0.0,0.97,273.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_exact_pass;best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,119,forced_incumbent_neighbors,capped_blended_uncertainty,0.18,0.375,1.0,1.0,0.1,160346.21021790063,-10118.33271056207,True,0.031125,0.233621,0.0,0.97,273.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_exact_pass;best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,151,forced_incumbent_neighbors,blended_uncertainty,0.185,0.4,1.0,1.0,0.1,159963.87736327032,-10500.66556519238,True,0.031125,0.23025,0.0,0.9703,271.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_V_exact_pass,V_challenger_only_gamma_return_worse,48,forced_incumbent_neighbors,blended_uncertainty,0.17,0.375,1.0,1.0,0.1,156085.00705628333,-14379.535872179375,True,0.024,0.216095,0.0,0.9701,272.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_V_exact_pass,V_challenger_only_gamma_return_worse,81,forced_incumbent_neighbors,blended_uncertainty,0.175,0.4,1.0,1.0,0.1,155984.27198996386,-14480.270938498841,True,0.0237,0.215295,0.0,0.974,275.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_V_exact_pass,V_challenger_only_gamma_return_worse,82,forced_incumbent_neighbors,capped_blended_uncertainty,0.175,0.4,1.0,1.0,0.1,155984.27198996386,-14480.270938498841,True,0.0237,0.215295,0.0,0.974,275.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,141,forced_incumbent_neighbors,blended_uncertainty,0.185,0.55,1.0,1.0,0.1,148716.28704654615,-21748.255881916557,True,0.047275,0.185044,0.0,0.9699,269.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,142,forced_incumbent_neighbors,capped_blended_uncertainty,0.185,0.55,1.0,1.0,0.1,148716.28704654615,-21748.255881916557,True,0.047275,0.185044,0.0,0.9699,269.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,73,forced_incumbent_neighbors,blended_uncertainty,0.175,0.5,1.0,1.0,0.1,148564.4946689952,-21900.048259467498,True,0.047275,0.184103,0.0,0.97,267.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,1,forced_incumbent_neighbors,blended_uncertainty,0.165,0.55,1.0,1.0,0.1,142501.65715908472,-27962.885769377986,True,0.047275,0.16054,0.0,0.9699,269.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_Gamma_exact_pass,Gamma_challenger_only_V_return_worse,2,forced_incumbent_neighbors,capped_blended_uncertainty,0.165,0.55,1.0,1.0,0.1,142501.65715908472,-27962.885769377986,True,0.047275,0.16054,0.0,0.9699,269.0 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,best_Gamma_exact_pass,bound_challenger_only_return_worse,225,incumbent_region,segment_tail_blended_uncertainty,0.165,0.55,1.0,0.75,0.5,126934.17894554739,-43530.36398291531,True,0.0272,0.160967,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,return_challenger_only_bound_worse,1,return_global,capped_blended_uncertainty,0.19,0.325,0.75,1.0,0.15,170611.34163424745,146.79870578474947,True,0.058675,0.270366,0.0,0.959,270.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,append_or_park_no_champion_case,2,return_global,capped_blended_uncertainty,0.19,0.325,0.9,1.0,0.15,167551.91376347217,-2912.6291649905324,True,0.058675,0.270366,0.0,0.959,274.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_exact_pass,append_or_park_no_champion_case,3,return_global,blended_uncertainty,0.19,0.325,1.0,1.0,0.15,167481.12021210513,-2983.4227163575706,True,0.058675,0.270366,0.0,0.959,273.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,7,return_global,capped_blended_uncertainty,0.18,0.325,0.75,1.0,0.0,165109.8500223593,-5354.692906103417,True,0.032675,0.258856,0.0,0.9667,269.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,11,return_global,capped_blended_uncertainty,0.18,0.325,0.9,1.0,0.0,164626.4438320008,-5838.099096461898,True,0.032675,0.258856,0.0,0.9667,271.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_V_leq_champion,V_challenger_only_gamma_return_worse,12,return_global,capped_blended_uncertainty,0.19,0.375,0.75,1.0,0.05,164537.97976539787,-5926.563163064828,True,0.033675,0.249951,0.0,0.9635,272.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,240,forced_incumbent_neighbors,blended_uncertainty,0.17,0.475,1.0,1.0,0.1,148477.2524564737,-21987.290471988992,True,0.047275,0.183571,0.0,0.97,266.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,207,forced_incumbent_neighbors,blended_uncertainty,0.165,0.45,1.0,1.0,0.1,148317.61801947665,-22146.924908986053,True,0.047275,0.183033,0.0,0.9701,269.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_return_with_Gamma_leq_champion,Gamma_challenger_only_V_return_worse,208,forced_incumbent_neighbors,capped_blended_uncertainty,0.165,0.45,1.0,1.0,0.1,148317.61801947665,-22146.924908986053,True,0.047275,0.183033,0.0,0.9701,269.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,bound_challenger_only_return_worse,200,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.425,1.0,0.75,0.5,126937.19014928368,-43527.352779179026,True,0.0272,0.160967,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,bound_challenger_only_return_worse,198,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.45,1.0,0.75,0.5,126937.10519376643,-43527.43773469627,True,0.0272,0.160967,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_V_exact_pass,bound_challenger_only_return_worse,199,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.55,1.0,0.75,0.5,126936.73858904834,-43527.80433941436,True,0.0272,0.160967,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,bound_challenger_only_return_worse,177,conservative_proxy,blended_uncertainty,0.16,0.6,1.0,1.0,0.5,126929.36612165972,-43535.17680680298,True,0.0272,0.137742,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,bound_challenger_only_return_worse,178,conservative_proxy,capped_blended_uncertainty,0.16,0.6,1.0,1.0,0.5,126929.36612165972,-43535.17680680298,True,0.0272,0.137742,0.0,0.9705,263.0 -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,best_Gamma_exact_pass,bound_challenger_only_return_worse,176,conservative_proxy,capped_blended_uncertainty,0.16,0.6,0.95,1.0,0.5,126929.35475228775,-43535.188176174954,True,0.0272,0.137742,0.0,0.9705,263.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,100,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.2,1.0,0.75,0.1,157491.47340001757,-12973.069528445136,True,0.094402,0.369189,0.0,0.8963,213.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,92,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.2,1.0,0.75,0.2,156296.63858668334,-14167.904341779358,True,0.080525,0.352566,0.0,0.9056,206.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_return_exact_pass,append_or_park_no_champion_case,107,conservative_proxy,segment_tail_blended_uncertainty,0.16,0.2,1.0,0.75,0.05,155276.72947392872,-15187.813454533985,True,0.087467,0.389095,0.0,0.9076,218.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_V_exact_pass,append_or_park_no_champion_case,83,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.35,1.0,0.75,0.2,141899.36265525976,-28565.180273202946,True,0.05905,0.27019,0.0,0.9312,202.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_Gamma_exact_pass;best_V_exact_pass,append_or_park_no_champion_case,24,incumbent_region,segment_relative_tail_blended_uncertainty,0.17,0.5,1.0,0.75,0.2,141366.9665507856,-29097.57637767709,True,0.059075,0.265447,0.0,0.9292,205.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_Gamma_exact_pass;best_V_exact_pass,append_or_park_no_champion_case,1,incumbent_region,segment_relative_tail_blended_uncertainty,0.165,0.5,1.0,0.75,0.2,139871.55389083803,-30592.989037624677,True,0.06185,0.256325,0.0,0.9333,205.0 -canonical_4_segment_probe,paper1_bound_expansion_2026_05_24_r1__canonical_4__segment_probe_25k,best_Gamma_exact_pass,append_or_park_no_champion_case,81,conservative_proxy,segment_relative_tail_blended_uncertainty,0.16,0.5,1.0,0.75,0.2,134955.27887917944,-35509.26404928326,True,0.06185,0.250435,0.0,0.9327,203.0 -official_champion,official_champion_frozen,incumbent,official baseline: strongest balanced return/bound trade-off so far,0,official,blended_uncertainty,0.175,0.45,1.0,1.0,0.1,170464.5429284627,0.0,True,0.03645,0.18591,0.0,0.9433, diff --git a/reports/crpto/appendix/crpto_appendix_paper1_conformal_reopen_candidate_gap_diagnostics_2026-05-25.csv b/reports/crpto/appendix/crpto_appendix_paper1_conformal_reopen_candidate_gap_diagnostics_2026-05-25.csv deleted file mode 100644 index edd9558..0000000 --- a/reports/crpto/appendix/crpto_appendix_paper1_conformal_reopen_candidate_gap_diagnostics_2026-05-25.csv +++ /dev/null @@ -1,6 +0,0 @@ -candidate,namespace,coverage_90,min_group_coverage_90,avg_width_90,winkler_90,max_monthly_gap,min_monthly_coverage,worst_group,worst_group_coverage_90,worst_group_avg_width_90,intervals_path,min_group_gap_to_90,score -affordability_rate_5,paper1_champion_replacement_2026_05_23_r3__affordability_rate_5__conformal__phase2__final__rank-1,0.9443166262745197,0.9166467556405092,0.806269691132808,1.064707372293085,0.09999999999999998,0.9362752356977146,score_q00,0.9166467556405092,0.0766497497259854,data/processed/conformal_gap/paper1_champion_replacement_2026_05_23_r3__affordability_rate_5__conformal__phase2__final__rank-1/conformal_intervals_mondrian.parquet,-0.016646755640509214,1.2662747950762872 -bureau_behavior_15,paper1_conformal_reopen_bureau_behavior_15_2026_05_21__resume__2026_05_23_0040__phase2__final__rank-1,0.9199513127146773,0.8700591060214259,0.7496148085327241,1.2066647249842977,0.09999999999999998,0.9089151163566046,E,0.8700591060214259,0.9377190861454339,data/processed/conformal_gap/paper1_conformal_reopen_bureau_behavior_15_2026_05_21__resume__2026_05_23_0040__phase2__final__rank-1/conformal_intervals_mondrian.parquet,0.029940893978574135,1.69347736690322 -bureau_behavior_15,paper1_champion_replacement_2026_05_23_r3__bureau_behavior_15__conformal__phase2__final__rank-1,0.919936865449003,0.8700591060214259,0.7493568453144235,1.2068341251182069,0.09999999999999998,0.9089151163566046,E,0.8700591060214259,0.9376156465390247,data/processed/conformal_gap/paper1_champion_replacement_2026_05_23_r3__bureau_behavior_15__conformal__phase2__final__rank-1/conformal_intervals_mondrian.parquet,0.029940893978574135,1.693582276232554 -canonical_4,paper1_champion_replacement_2026_05_23_r3__canonical_4__conformal__phase2__final__rank-1,0.9318775305288782,0.9175819999239861,0.7907288458926857,1.0875521925324798,0.09999999999999998,0.9215163630978824,score_q04,0.9175819999239861,0.9905034492354011,data/processed/conformal_gap/paper1_champion_replacement_2026_05_23_r3__canonical_4__conformal__phase2__final__rank-1/conformal_intervals_mondrian.parquet,-0.017581999923986125,1.2852344040056511 -official_champion,conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1,0.9297140524941399,0.9189831313851271,0.7842303100675323,1.110742432294828,0.09999999999999998,0.9166790068611481,score_q03,0.9189831313851271,0.9869383918090624,data/processed/conformal_gap/conformal-reopen-2026-04-03-2149__resume__2026-04-05-1612__phase1__final__rank-1/conformal_intervals_mondrian.parquet,-0.018983131385127083,1.3068000098117112 diff --git a/reports/crpto/appendix/crpto_appendix_paper1_return_aware_rerank_summary_2026-05-25.csv b/reports/crpto/appendix/crpto_appendix_paper1_return_aware_rerank_summary_2026-05-25.csv deleted file mode 100644 index ca3bef9..0000000 --- a/reports/crpto/appendix/crpto_appendix_paper1_return_aware_rerank_summary_2026-05-25.csv +++ /dev/null @@ -1,5 +0,0 @@ -label,run,selection_reason,return,return_delta_vs_champion,alpha01_pass,V,V_delta_vs_champion,Gamma,Gamma_delta_vs_champion,violation,funded_coverage,policy_mode,risk,gamma,qcap,qtail,ua,bucket,candidate_rank -canonical_4_return_aware,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,selected_best_alpha01_exact_pass,170611.34163424745,146.79870578474947,True,0.058675,0.022224999999999995,0.270366,0.084456,0.0,0.959,capped_blended_uncertainty,0.19,0.325,0.75,1.0,0.15,return_global,1 -affordability_rate_5_return_aware,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k__return_aware_alpha01_2026_05_25_r5,selected_best_alpha01_exact_pass,166361.201522614,-4103.341405848711,True,0.0831,0.04664999999999999,0.340404,0.154494,0.0,0.928,segment_relative_tail_blended_uncertainty,0.16,0.325,1.0,0.95,0.25,return_global,30 -canonical_4_original_medium,paper1_bound_expansion_2026_05_24_r1__canonical_4__medium_triage_resume_75k,selected_best_alpha01_exact_pass,160346.21021790063,-10118.33271056207,True,0.031125,-0.005325000000000003,0.233621,0.047711,0.0,0.97,blended_uncertainty,0.18,0.375,1.0,1.0,0.1,forced_incumbent_neighbors,118 -affordability_rate_5_original_medium,paper1_bound_expansion_2026_05_24_r1__affordability_rate_5__medium_triage_resume_75k,selected_best_alpha01_exact_pass,146569.73552904688,-23894.80739941582,True,0.038528,0.0020779999999999965,0.251736,0.06582600000000002,0.0,0.9651,blended_uncertainty,0.19,0.375,1.0,1.0,0.1,forced_incumbent_neighbors,188 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/archive_inventory.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/archive_inventory.csv new file mode 100644 index 0000000..41141e2 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/archive_inventory.csv @@ -0,0 +1,2 @@ +raw_rows,distinct_nonblank_ids,blank_id_rows,valid_loan_rows,invalid_issue_date_rows,first_issue_date,last_issue_date,last_payment_date_max,last_credit_pull_date_max,last_payment_rows_after_cutoff,last_credit_pull_rows_after_cutoff,term36_rows,term60_rows,other_or_invalid_term_rows +2925493,2925493,0.0,2925492.0,1.0,2007-06-01,2020-09-01,2020-12-01,2020-10-01,36485.0,40214.0,2060077.0,865415.0,1.0 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cohort_inventory.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cohort_inventory.csv new file mode 100644 index 0000000..80c9911 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cohort_inventory.csv @@ -0,0 +1,9 @@ +cohort,rows +censored_extension,88227 +conformal_fit,49007 +maturity_gap,541863 +pd_development,17433 +policy_development,94885 +post_extension,877671 +primary_oot,376890 +probability_calibration,14101 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cutoff_label_availability.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cutoff_label_availability.csv new file mode 100644 index 0000000..47dba2e --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cutoff_label_availability.csv @@ -0,0 +1,4 @@ +issue_year,rows,terminal_at_snapshot,available_by_cutoff,available_rate,available_bad,available_good +2014,162570,162570.0,59910.0,0.36851817678538473,9317.0,50593.0 +2015,283173,283173.0,28878.0,0.10198006165842083,2163.0,26715.0 +2016,96120,96120.0,1110.0,0.011548064918851436,0.0,1110.0 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json new file mode 100644 index 0000000..f79fa15 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json @@ -0,0 +1,89 @@ +{ + "schema_version": "2026-07-14.1", + "status": "complete_full_archive_data_contract_audit", + "run_tag": "ijds-raw-data-contract-2026-07-14-v2", + "base_protocol": { + "path": "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml", + "bytes": 1232, + "sha256": "5123698bd6d21a0531f81ce7b26edc093dc49452e13aa531166c5ccc36400eca" + }, + "config": { + "path": "configs/experiments/ijds_raw_data_contract_2026-07-14_v2.yaml", + "bytes": 1268, + "sha256": "bb6f240caeb87999d7071590f851d5db886ba93baba5072c60e6f550c7d970e2" + }, + "raw_source": { + "path": "data/raw/Loan_status_2007-2020Q3.csv", + "bytes": 1773470505, + "sha256": "5878af2a088f8ab5214c9337289fb8b5eb6c6338fd3f417b6cdc18513dc6f35f", + "dvc_md5": "65adade308f21d60b7213088a88e684d" + }, + "results": { + "raw_rows": 2925493, + "valid_loan_rows": 2925492, + "distinct_nonblank_ids": 2925493, + "term36_rows_all_dates": 2060077, + "term60_rows_all_dates": 865415, + "raw_schema_columns": 142, + "last_payment_date_max": "2020-12-01", + "last_credit_pull_date_max": "2020-10-01", + "last_payment_rows_after_cutoff": 36485, + "last_credit_pull_rows_after_cutoff": 40214, + "term36_active_design_rows": 640543, + "eligible_raw_features": 30, + "declared_coverage_exceptions": 2, + "coverage_exceptions_requiring_sensitivity": 2, + "late_schema_features": 48, + "primary_oot_partial_funding_share": 5.306588129162355e-06, + "primary_oot_funded_ratio": 0.9999961757560076, + "primary_oot_total_requested_minus_funded": 18000.0 + }, + "artifacts": { + "archive_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/archive_inventory.csv", + "bytes": 400, + "sha256": "b6788da7d4d33c43f34b5bed5ca944a9ff4d349983df041bbc9eef8fd971d6e2" + }, + "status_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/status_inventory.csv", + "bytes": 470, + "sha256": "8d81783e3854efcf5ccd88f3342984cb44ed6d0cbef774b3bfab3382b4220845" + }, + "cohort_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cohort_inventory.csv", + "bytes": 194, + "sha256": "c5bd8d225a20c53d5865473a1550e2b0b506504f4d6e777b6d4ba28d3da645bb" + }, + "feature_coverage": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_coverage.csv", + "bytes": 67423, + "sha256": "c00b84d14e26cbba9b46f96fc22cba4f52565f3bc361b5ab24596b80cbc12e04" + }, + "feature_contract": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_contract.csv", + "bytes": 21744, + "sha256": "9444486e9139b5a2b5af53d0bd153f5153d618fceea95e9831f79f8d01477d97" + }, + "amount_alignment": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/loan_amount_alignment.csv", + "bytes": 729, + "sha256": "d0b2d8f6198e35b1d96739175c3d40f75584f090c67496a23295a853abbc4824" + }, + "cutoff_label_availability": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cutoff_label_availability.csv", + "bytes": 287, + "sha256": "a1463a347603fe4d6a72d69c797810e0ab72ae8ffe36258988215e8ffb35f689" + } + }, + "implementation_sha256": "21d3526d53f1329b5c08de8e3c3ea9e605975ea173e5ab24d963c2b5d861c1cd", + "git": { + "commit": "688f75dc4f285c75bc499c9e041dd30fb3acd70d", + "dirty": true, + "dirty_entries": 1, + "dirty_paths": [ + "?? reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/" + ] + }, + "protected_stages_run": [], + "protected_artifacts_written": [] +} diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/loan_amount_alignment.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/loan_amount_alignment.csv new file mode 100644 index 0000000..b160c38 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/loan_amount_alignment.csv @@ -0,0 +1,9 @@ +cohort,rows,total_gap,mean_gap,median_gap,max_gap,partial_share,funded_ratio +censored_extension,88227,0.0,0.0,0.0,0.0,0.0,1.0 +conformal_fit,49007,127925.0,2.610341379802885,0.0,12850.0,0.00042851021282673904,0.9997805051607024 +maturity_gap,541863,0.0,0.0,0.0,0.0,0.0,1.0 +pd_development,17433,3660025.0,209.94808696150977,0.0,21025.0,0.0447427293064877,0.9781837337301165 +policy_development,94885,1625.0,0.017125994625072458,0.0,1600.0,2.107814723085841e-05,0.9999986425245718 +post_extension,877671,0.0,0.0,0.0,0.0,0.0,1.0 +primary_oot,376890,18000.0,0.0477592931624612,0.0,10000.0,5.306588129162355e-06,0.9999961757560076 +probability_calibration,14101,316350.0,22.43457910786469,0.0,13175.0,0.00496418693709666,0.9976186972350617 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_contract.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_contract.csv new file mode 100644 index 0000000..ea9065c --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_contract.csv @@ -0,0 +1,143 @@ +feature,provenance_role,loaded_by_active_protocol,minimum_fitting_coverage,primary_oot_coverage,coverage_threshold,coverage_exception,missingness_semantics,requires_sensitivity,late_feature,eligible_for_current_temporal_model,decision,reason +acc_now_delinq,candidate_origination,False,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,requires temporal coverage and leakage review +annual_inc,active_protocol_input,True,0.9997705501061206,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +chargeoff_within_12_mths,active_protocol_input,True,0.9916824413468709,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +collections_12_mths_ex_med,active_protocol_input,True,0.9916824413468709,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +delinq_2yrs,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +delinq_amnt,candidate_origination,False,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,requires temporal coverage and leakage review +dti,active_protocol_input,True,1.0,0.9997426304757356,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +earliest_cr_line,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +emp_length,active_protocol_input,True,0.9606410892844479,0.927538539096288,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +fico_range_high,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +fico_range_low,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +grade,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +home_ownership,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +initial_list_status,candidate_origination,False,1.0,1.0,0.95,,,False,False,True,eligible,requires temporal coverage and leakage review +inq_last_6mths,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +installment,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +int_rate,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +loan_amnt,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +open_acc,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +policy_code,candidate_origination,False,1.0,1.0,0.95,,,False,False,True,eligible,requires temporal coverage and leakage review +pub_rec,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +purpose,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +revol_bal,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +revol_util,active_protocol_input,True,0.9954683645958814,0.9993181034254026,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +sub_grade,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +tax_liens,active_protocol_input,True,0.9939769402856651,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +total_acc,active_protocol_input,True,0.9983364882693742,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +verification_status,active_protocol_input,True,1.0,1.0,0.95,,,False,False,True,eligible,loaded by the active maturity-safe protocol +mths_since_last_delinq,active_protocol_input,True,0.33607545564144387,0.5285547507230226,0.95,structurally_nullable_credit_history_field,frozen feature engineering maps missing recency to 999,True,False,True,eligible_with_declared_coverage_exception,loaded by the active maturity-safe protocol +pub_rec_bankruptcies,active_protocol_input,True,0.9217002237136466,1.0,0.95,legacy_schema_partial_coverage,frozen feature engineering maps missing bankruptcy count to no recorded bankruptcy,True,False,True,eligible_with_declared_coverage_exception,loaded by the active maturity-safe protocol +addr_state,geography_proxy,False,1.0,1.0,0.95,,,False,False,False,exclude_by_role_or_coverage,available but excluded from the active model +annual_inc_joint,joint_application_origination,False,0.0,0.02665764546684709,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +application_type,joint_application_origination,False,1.0,1.0,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +column000,identifier_or_free_text,False,0.0,0.0,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +dti_joint,joint_application_origination,False,0.0,0.02665764546684709,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +emp_title,identifier_or_free_text,False,0.927593787674633,0.9266735652312346,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +funded_amnt,contemporaneous_funding,True,1.0,1.0,0.95,,,False,False,False,exclude_by_role_or_coverage,audited separately; not needed by the active portfolio because requested and funded amounts nearly coincide +funded_amnt_inv,contemporaneous_funding,False,1.0,1.0,0.95,,,False,False,False,exclude_by_role_or_coverage,audited separately; not needed by the active portfolio because requested and funded amounts nearly coincide +id,identifier_or_free_text,True,0.0,0.0,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +issue_d,protocol_metadata,True,0.0,0.0,0.95,,,False,False,False,exclude_by_role_or_coverage,used for chronology or term eligibility +mths_since_last_major_derog,candidate_origination,False,0.0,0.2895115285627106,0.95,,,False,False,False,exclude_by_role_or_coverage,requires temporal coverage and leakage review +mths_since_last_record,candidate_origination,False,0.04607505050298937,0.19810554803788905,0.95,,,False,False,False,exclude_by_role_or_coverage,requires temporal coverage and leakage review +mths_since_recent_bc_dlq,candidate_origination,False,0.0,0.2535779670460877,0.95,,,False,False,False,exclude_by_role_or_coverage,requires temporal coverage and leakage review +mths_since_recent_revol_delinq,candidate_origination,False,0.0,0.36314309214890284,0.95,,,False,False,False,exclude_by_role_or_coverage,requires temporal coverage and leakage review +revol_bal_joint,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_chargeoff_within_12_mths,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_collections_12_mths_ex_med,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_earliest_cr_line,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_fico_range_high,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_fico_range_low,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_inq_last_6mths,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_mort_acc,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_num_rev_accts,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_open_acc,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_open_act_il,joint_application_origination,False,0.0,0.0114038578895699,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +sec_app_revol_util,joint_application_origination,False,0.0,0.011236700363501287,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +term,protocol_metadata,True,0.0,0.0,0.95,,,False,False,False,exclude_by_role_or_coverage,used for chronology or term eligibility +title,identifier_or_free_text,False,0.9994837377387713,0.9774416938629308,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +url,identifier_or_free_text,False,1.0,1.0,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +verification_status_joint,joint_application_origination,False,0.0,0.02665764546684709,0.95,,,False,False,False,exclude_by_role_or_coverage,origination-time but introduced late +zip_code,identifier_or_free_text,False,1.0,0.9999973467059354,0.95,,,False,False,False,exclude_by_role_or_coverage,excluded from the scientific feature contract +acc_open_past_24mths,active_protocol_input,True,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,loaded by the active maturity-safe protocol +all_util,candidate_origination,False,0.0,0.9999389742365147,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +avg_cur_bal,candidate_origination,False,0.0,0.9999946934118709,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +bc_open_to_buy,candidate_origination,False,0.0,0.9877948473029265,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +bc_util,candidate_origination,False,0.0,0.9873464406060124,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +il_util,candidate_origination,False,0.0,0.861267213245244,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +inq_fi,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +inq_last_12m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +max_bal_bc,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mo_sin_old_il_acct,candidate_origination,False,0.0,0.9696489691952559,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mo_sin_old_rev_tl_op,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mo_sin_rcnt_rev_tl_op,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mo_sin_rcnt_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mort_acc,active_protocol_input,True,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,loaded by the active maturity-safe protocol +mths_since_rcnt_il,candidate_origination,False,0.0,0.9696489691952559,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mths_since_recent_bc,candidate_origination,False,0.0,0.9884183714081032,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +mths_since_recent_inq,candidate_origination,False,0.0,0.8870041656716814,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_accts_ever_120_pd,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_actv_bc_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_actv_rev_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_bc_sats,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_bc_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_il_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_op_rev_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_rev_accts,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_rev_tl_bal_gt_0,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_sats,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_tl_120dpd_2m,candidate_origination,False,0.0,0.9574199368516013,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_tl_30dpd,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_tl_90g_dpd_24m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +num_tl_op_past_12m,active_protocol_input,True,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,loaded by the active maturity-safe protocol +open_acc_6m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +open_act_il,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +open_il_12m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +open_il_24m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +open_rv_12m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +open_rv_24m,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +pct_tl_nvr_dlq,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +percent_bc_gt_75,candidate_origination,False,0.0,0.9877656610682162,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +tot_coll_amt,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +tot_cur_bal,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +tot_hi_cred_lim,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_bal_ex_mort,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_bal_il,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_bc_limit,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_cu_tl,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_il_high_credit_limit,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +total_rev_hi_lim,candidate_origination,False,0.0,1.0,0.95,,,False,True,False,exclude_late_schema,requires temporal coverage and leakage review +collection_recovery_fee,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +debt_settlement_flag,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +deferral_term,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_amount,post_outcome_or_servicing,False,0.0,0.018599591392714054,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_dpd,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_end_date,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_flag,post_outcome_or_servicing,False,1.0,0.9953408156225955,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_last_payment_amount,post_outcome_or_servicing,False,0.0,0.018599591392714054,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_length,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_loan_status,post_outcome_or_servicing,False,0.0,0.013935100427180344,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_payoff_balance_amount,post_outcome_or_servicing,False,0.0,0.018599591392714054,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_reason,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_start_date,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_status,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +hardship_type,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +last_credit_pull_d,post_outcome_or_servicing,False,0.9997705501061206,0.9999708137652896,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +last_fico_range_high,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +last_fico_range_low,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +last_pymnt_amnt,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +last_pymnt_d,post_outcome_or_servicing,True,0.9976481385877359,0.9987317254371302,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +loan_status,post_outcome_or_servicing,True,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +next_pymnt_d,post_outcome_or_servicing,False,0.0,0.03064819973997718,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +orig_projected_additional_accrued_interest,post_outcome_or_servicing,False,0.0,0.01644776990633872,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +out_prncp,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +out_prncp_inv,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +payment_plan_start_date,post_outcome_or_servicing,False,0.0,0.013940407015309507,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +pymnt_plan,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +recoveries,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +total_pymnt,post_outcome_or_servicing,True,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +total_pymnt_inv,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +total_rec_int,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +total_rec_late_fee,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature +total_rec_prncp,post_outcome_or_servicing,False,1.0,1.0,0.95,,,False,False,False,exclude_post_outcome,never eligible as a prediction feature diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_coverage.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_coverage.csv new file mode 100644 index 0000000..5e303c8 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_coverage.csv @@ -0,0 +1,1105 @@ +cohort,feature,rows,present_rows,coverage,first_seen +censored_extension,acc_now_delinq,88227,88227,1.0,2017-07-01 +conformal_fit,acc_now_delinq,49007,49007,1.0,2012-01-01 +maturity_gap,acc_now_delinq,541863,541863,1.0,2014-01-01 +pd_development,acc_now_delinq,17433,17404,0.9983364882693742,2007-06-01 +policy_development,acc_now_delinq,94885,94885,1.0,2013-02-01 +post_extension,acc_now_delinq,877671,877671,1.0,2017-10-01 +primary_oot,acc_now_delinq,376890,376890,1.0,2016-04-01 +probability_calibration,acc_now_delinq,14101,14101,1.0,2011-01-01 +censored_extension,acc_open_past_24mths,88227,88227,1.0,2017-07-01 +conformal_fit,acc_open_past_24mths,49007,42982,0.8770583794151856,2012-03-01 +maturity_gap,acc_open_past_24mths,541863,541863,1.0,2014-01-01 +pd_development,acc_open_past_24mths,17433,0,0.0, +policy_development,acc_open_past_24mths,94885,94885,1.0,2013-02-01 +post_extension,acc_open_past_24mths,877671,877671,1.0,2017-10-01 +primary_oot,acc_open_past_24mths,376890,376890,1.0,2016-04-01 +probability_calibration,acc_open_past_24mths,14101,0,0.0, +censored_extension,addr_state,88227,88227,1.0,2017-07-01 +conformal_fit,addr_state,49007,49007,1.0,2012-01-01 +maturity_gap,addr_state,541863,541863,1.0,2014-01-01 +pd_development,addr_state,17433,17433,1.0,2007-06-01 +policy_development,addr_state,94885,94885,1.0,2013-02-01 +post_extension,addr_state,877671,877671,1.0,2017-10-01 +primary_oot,addr_state,376890,376890,1.0,2016-04-01 +probability_calibration,addr_state,14101,14101,1.0,2011-01-01 +censored_extension,all_util,88227,88214,0.9998526528160314,2017-07-01 +conformal_fit,all_util,49007,0,0.0, +maturity_gap,all_util,541863,111006,0.20485990001162654,2015-12-01 +pd_development,all_util,17433,0,0.0, +policy_development,all_util,94885,0,0.0, +post_extension,all_util,877671,877456,0.9997550334920489,2017-10-01 +primary_oot,all_util,376890,376867,0.9999389742365147,2016-04-01 +probability_calibration,all_util,14101,0,0.0, +censored_extension,annual_inc,88227,88227,1.0,2017-07-01 +conformal_fit,annual_inc,49007,49007,1.0,2012-01-01 +maturity_gap,annual_inc,541863,541863,1.0,2014-01-01 +pd_development,annual_inc,17433,17429,0.9997705501061206,2007-06-01 +policy_development,annual_inc,94885,94885,1.0,2013-02-01 +post_extension,annual_inc,877671,877671,1.0,2017-10-01 +primary_oot,annual_inc,376890,376890,1.0,2016-04-01 +probability_calibration,annual_inc,14101,14101,1.0,2011-01-01 +censored_extension,annual_inc_joint,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,annual_inc_joint,49007,0,0.0, +maturity_gap,annual_inc_joint,541863,1767,0.0032609718692732294,2015-10-01 +pd_development,annual_inc_joint,17433,0,0.0, +policy_development,annual_inc_joint,94885,0,0.0, +post_extension,annual_inc_joint,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,annual_inc_joint,376890,10047,0.02665764546684709,2016-04-01 +probability_calibration,annual_inc_joint,14101,0,0.0, +censored_extension,application_type,88227,88227,1.0,2017-07-01 +conformal_fit,application_type,49007,49007,1.0,2012-01-01 +maturity_gap,application_type,541863,541863,1.0,2014-01-01 +pd_development,application_type,17433,17433,1.0,2007-06-01 +policy_development,application_type,94885,94885,1.0,2013-02-01 +post_extension,application_type,877671,877671,1.0,2017-10-01 +primary_oot,application_type,376890,376890,1.0,2016-04-01 +probability_calibration,application_type,14101,14101,1.0,2011-01-01 +censored_extension,avg_cur_bal,88227,88223,0.9999546624049327,2017-07-01 +conformal_fit,avg_cur_bal,49007,26456,0.5398412471687718,2012-08-01 +maturity_gap,avg_cur_bal,541863,541858,0.9999907725753557,2014-01-01 +pd_development,avg_cur_bal,17433,0,0.0, +policy_development,avg_cur_bal,94885,94882,0.9999683827791537,2013-02-01 +post_extension,avg_cur_bal,877671,877595,0.999913407187887,2017-10-01 +primary_oot,avg_cur_bal,376890,376888,0.9999946934118709,2016-04-01 +probability_calibration,avg_cur_bal,14101,0,0.0, +censored_extension,bc_open_to_buy,88227,86879,0.9847212304623302,2017-07-01 +conformal_fit,bc_open_to_buy,49007,42516,0.8675495337400779,2012-03-01 +maturity_gap,bc_open_to_buy,541863,536415,0.9899457981076397,2014-01-01 +pd_development,bc_open_to_buy,17433,0,0.0, +policy_development,bc_open_to_buy,94885,94127,0.9920113821995047,2013-02-01 +post_extension,bc_open_to_buy,877671,865887,0.986573556606063,2017-10-01 +primary_oot,bc_open_to_buy,376890,372290,0.9877948473029265,2016-04-01 +probability_calibration,bc_open_to_buy,14101,0,0.0, +censored_extension,bc_util,88227,86833,0.9841998481190565,2017-07-01 +conformal_fit,bc_util,49007,42493,0.8670802130307915,2012-03-01 +maturity_gap,bc_util,541863,536036,0.9892463593196066,2014-01-01 +pd_development,bc_util,17433,0,0.0, +policy_development,bc_util,94885,94087,0.9915898192548875,2013-02-01 +post_extension,bc_util,877671,865450,0.9860756479364136,2017-10-01 +primary_oot,bc_util,376890,372121,0.9873464406060124,2016-04-01 +probability_calibration,bc_util,14101,0,0.0, +censored_extension,chargeoff_within_12_mths,88227,88227,1.0,2017-07-01 +conformal_fit,chargeoff_within_12_mths,49007,49007,1.0,2012-01-01 +maturity_gap,chargeoff_within_12_mths,541863,541863,1.0,2014-01-01 +pd_development,chargeoff_within_12_mths,17433,17288,0.9916824413468709,2007-08-01 +policy_development,chargeoff_within_12_mths,94885,94885,1.0,2013-02-01 +post_extension,chargeoff_within_12_mths,877671,877671,1.0,2017-10-01 +primary_oot,chargeoff_within_12_mths,376890,376890,1.0,2016-04-01 +probability_calibration,chargeoff_within_12_mths,14101,14101,1.0,2011-01-01 +censored_extension,collection_recovery_fee,88227,88227,1.0,2017-07-01 +conformal_fit,collection_recovery_fee,49007,49007,1.0,2012-01-01 +maturity_gap,collection_recovery_fee,541863,541863,1.0,2014-01-01 +pd_development,collection_recovery_fee,17433,17433,1.0,2007-06-01 +policy_development,collection_recovery_fee,94885,94885,1.0,2013-02-01 +post_extension,collection_recovery_fee,877671,877671,1.0,2017-10-01 +primary_oot,collection_recovery_fee,376890,376890,1.0,2016-04-01 +probability_calibration,collection_recovery_fee,14101,14101,1.0,2011-01-01 +censored_extension,collections_12_mths_ex_med,88227,88227,1.0,2017-07-01 +conformal_fit,collections_12_mths_ex_med,49007,49007,1.0,2012-01-01 +maturity_gap,collections_12_mths_ex_med,541863,541863,1.0,2014-01-01 +pd_development,collections_12_mths_ex_med,17433,17288,0.9916824413468709,2007-08-01 +policy_development,collections_12_mths_ex_med,94885,94885,1.0,2013-02-01 +post_extension,collections_12_mths_ex_med,877671,877671,1.0,2017-10-01 +primary_oot,collections_12_mths_ex_med,376890,376890,1.0,2016-04-01 +probability_calibration,collections_12_mths_ex_med,14101,14101,1.0,2011-01-01 +censored_extension,debt_settlement_flag,88227,88227,1.0,2017-07-01 +conformal_fit,debt_settlement_flag,49007,49007,1.0,2012-01-01 +maturity_gap,debt_settlement_flag,541863,541863,1.0,2014-01-01 +pd_development,debt_settlement_flag,17433,17433,1.0,2007-06-01 +policy_development,debt_settlement_flag,94885,94885,1.0,2013-02-01 +post_extension,debt_settlement_flag,877671,877671,1.0,2017-10-01 +primary_oot,debt_settlement_flag,376890,376890,1.0,2016-04-01 +probability_calibration,debt_settlement_flag,14101,14101,1.0,2011-01-01 +censored_extension,deferral_term,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,deferral_term,49007,0,0.0, +maturity_gap,deferral_term,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,deferral_term,17433,0,0.0, +policy_development,deferral_term,94885,0,0.0, +post_extension,deferral_term,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,deferral_term,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,deferral_term,14101,0,0.0, +censored_extension,delinq_2yrs,88227,88227,1.0,2017-07-01 +conformal_fit,delinq_2yrs,49007,49007,1.0,2012-01-01 +maturity_gap,delinq_2yrs,541863,541863,1.0,2014-01-01 +pd_development,delinq_2yrs,17433,17404,0.9983364882693742,2007-06-01 +policy_development,delinq_2yrs,94885,94885,1.0,2013-02-01 +post_extension,delinq_2yrs,877671,877671,1.0,2017-10-01 +primary_oot,delinq_2yrs,376890,376890,1.0,2016-04-01 +probability_calibration,delinq_2yrs,14101,14101,1.0,2011-01-01 +censored_extension,delinq_amnt,88227,88227,1.0,2017-07-01 +conformal_fit,delinq_amnt,49007,49007,1.0,2012-01-01 +maturity_gap,delinq_amnt,541863,541863,1.0,2014-01-01 +pd_development,delinq_amnt,17433,17404,0.9983364882693742,2007-06-01 +policy_development,delinq_amnt,94885,94885,1.0,2013-02-01 +post_extension,delinq_amnt,877671,877671,1.0,2017-10-01 +primary_oot,delinq_amnt,376890,376890,1.0,2016-04-01 +probability_calibration,delinq_amnt,14101,14101,1.0,2011-01-01 +censored_extension,dti,88227,88120,0.9987872193319506,2017-07-01 +conformal_fit,dti,49007,49007,1.0,2012-01-01 +maturity_gap,dti,541863,541851,0.9999778541808538,2014-01-01 +pd_development,dti,17433,17433,1.0,2007-06-01 +policy_development,dti,94885,94885,1.0,2013-02-01 +post_extension,dti,877671,876133,0.9982476349338192,2017-10-01 +primary_oot,dti,376890,376793,0.9997426304757356,2016-04-01 +probability_calibration,dti,14101,14101,1.0,2011-01-01 +censored_extension,dti_joint,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,dti_joint,49007,0,0.0, +maturity_gap,dti_joint,541863,1764,0.0032554354144866877,2015-10-01 +pd_development,dti_joint,17433,0,0.0, +policy_development,dti_joint,94885,0,0.0, +post_extension,dti_joint,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,dti_joint,376890,10047,0.02665764546684709,2016-04-01 +probability_calibration,dti_joint,14101,0,0.0, +censored_extension,earliest_cr_line,88227,88227,1.0,2017-07-01 +conformal_fit,earliest_cr_line,49007,49007,1.0,2012-01-01 +maturity_gap,earliest_cr_line,541863,541863,1.0,2014-01-01 +pd_development,earliest_cr_line,17433,17404,0.9983364882693742,2007-06-01 +policy_development,earliest_cr_line,94885,94885,1.0,2013-02-01 +post_extension,earliest_cr_line,877671,877671,1.0,2017-10-01 +primary_oot,earliest_cr_line,376890,376890,1.0,2016-04-01 +probability_calibration,earliest_cr_line,14101,14101,1.0,2011-01-01 +censored_extension,emp_length,88227,81429,0.9229487571831753,2017-07-01 +conformal_fit,emp_length,49007,47086,0.9608015181504683,2012-01-01 +maturity_gap,emp_length,541863,505899,0.9336289800189347,2014-01-01 +pd_development,emp_length,17433,17153,0.9839385074284404,2007-06-01 +policy_development,emp_length,94885,90082,0.9493808294250935,2013-02-01 +post_extension,emp_length,877671,796883,0.9079518407239159,2017-10-01 +primary_oot,emp_length,376890,349580,0.927538539096288,2016-04-01 +probability_calibration,emp_length,14101,13546,0.9606410892844479,2011-01-01 +censored_extension,emp_title,88227,81322,0.9217359765151257,2017-07-01 +conformal_fit,emp_title,49007,45884,0.936274409778195,2012-01-01 +maturity_gap,emp_title,541863,504837,0.9316690750244988,2014-01-01 +pd_development,emp_title,17433,16382,0.9397120403831813,2007-06-01 +policy_development,emp_title,94885,88121,0.9287137060652368,2013-02-01 +post_extension,emp_title,877671,758744,0.8644970609715942,2017-10-01 +primary_oot,emp_title,376890,349254,0.9266735652312346,2016-04-01 +probability_calibration,emp_title,14101,13080,0.927593787674633,2011-01-01 +censored_extension,fico_range_high,88227,88227,1.0,2017-07-01 +conformal_fit,fico_range_high,49007,49007,1.0,2012-01-01 +maturity_gap,fico_range_high,541863,541863,1.0,2014-01-01 +pd_development,fico_range_high,17433,17433,1.0,2007-06-01 +policy_development,fico_range_high,94885,94885,1.0,2013-02-01 +post_extension,fico_range_high,877671,877671,1.0,2017-10-01 +primary_oot,fico_range_high,376890,376890,1.0,2016-04-01 +probability_calibration,fico_range_high,14101,14101,1.0,2011-01-01 +censored_extension,fico_range_low,88227,88227,1.0,2017-07-01 +conformal_fit,fico_range_low,49007,49007,1.0,2012-01-01 +maturity_gap,fico_range_low,541863,541863,1.0,2014-01-01 +pd_development,fico_range_low,17433,17433,1.0,2007-06-01 +policy_development,fico_range_low,94885,94885,1.0,2013-02-01 +post_extension,fico_range_low,877671,877671,1.0,2017-10-01 +primary_oot,fico_range_low,376890,376890,1.0,2016-04-01 +probability_calibration,fico_range_low,14101,14101,1.0,2011-01-01 +censored_extension,funded_amnt,88227,88227,1.0,2017-07-01 +conformal_fit,funded_amnt,49007,49007,1.0,2012-01-01 +maturity_gap,funded_amnt,541863,541863,1.0,2014-01-01 +pd_development,funded_amnt,17433,17433,1.0,2007-06-01 +policy_development,funded_amnt,94885,94885,1.0,2013-02-01 +post_extension,funded_amnt,877671,877671,1.0,2017-10-01 +primary_oot,funded_amnt,376890,376890,1.0,2016-04-01 +probability_calibration,funded_amnt,14101,14101,1.0,2011-01-01 +censored_extension,funded_amnt_inv,88227,88227,1.0,2017-07-01 +conformal_fit,funded_amnt_inv,49007,49007,1.0,2012-01-01 +maturity_gap,funded_amnt_inv,541863,541863,1.0,2014-01-01 +pd_development,funded_amnt_inv,17433,17433,1.0,2007-06-01 +policy_development,funded_amnt_inv,94885,94885,1.0,2013-02-01 +post_extension,funded_amnt_inv,877671,877671,1.0,2017-10-01 +primary_oot,funded_amnt_inv,376890,376890,1.0,2016-04-01 +probability_calibration,funded_amnt_inv,14101,14101,1.0,2011-01-01 +censored_extension,grade,88227,88227,1.0,2017-07-01 +conformal_fit,grade,49007,49007,1.0,2012-01-01 +maturity_gap,grade,541863,541863,1.0,2014-01-01 +pd_development,grade,17433,17433,1.0,2007-06-01 +policy_development,grade,94885,94885,1.0,2013-02-01 +post_extension,grade,877671,877671,1.0,2017-10-01 +primary_oot,grade,376890,376890,1.0,2016-04-01 +probability_calibration,grade,14101,14101,1.0,2011-01-01 +censored_extension,hardship_amount,88227,5228,0.059256236752921444,2017-07-01 +conformal_fit,hardship_amount,49007,0,0.0, +maturity_gap,hardship_amount,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_amount,17433,0,0.0, +policy_development,hardship_amount,94885,0,0.0, +post_extension,hardship_amount,877671,95172,0.10843698834756987,2017-10-01 +primary_oot,hardship_amount,376890,7010,0.018599591392714054,2016-04-01 +probability_calibration,hardship_amount,14101,0,0.0, +censored_extension,hardship_dpd,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_dpd,49007,0,0.0, +maturity_gap,hardship_dpd,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_dpd,17433,0,0.0, +policy_development,hardship_dpd,94885,0,0.0, +post_extension,hardship_dpd,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,hardship_dpd,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_dpd,14101,0,0.0, +censored_extension,hardship_end_date,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_end_date,49007,0,0.0, +maturity_gap,hardship_end_date,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_end_date,17433,0,0.0, +policy_development,hardship_end_date,94885,0,0.0, +post_extension,hardship_end_date,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,hardship_end_date,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_end_date,14101,0,0.0, +censored_extension,hardship_flag,88227,86578,0.9813095764335181,2017-07-01 +conformal_fit,hardship_flag,49007,49007,1.0,2012-01-01 +maturity_gap,hardship_flag,541863,541863,1.0,2014-01-01 +pd_development,hardship_flag,17433,17433,1.0,2007-06-01 +policy_development,hardship_flag,94885,94885,1.0,2013-02-01 +post_extension,hardship_flag,877671,856898,0.9763316778154912,2017-10-01 +primary_oot,hardship_flag,376890,375134,0.9953408156225955,2016-04-01 +probability_calibration,hardship_flag,14101,14101,1.0,2011-01-01 +censored_extension,hardship_last_payment_amount,88227,5228,0.059256236752921444,2017-07-01 +conformal_fit,hardship_last_payment_amount,49007,0,0.0, +maturity_gap,hardship_last_payment_amount,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_last_payment_amount,17433,0,0.0, +policy_development,hardship_last_payment_amount,94885,0,0.0, +post_extension,hardship_last_payment_amount,877671,95172,0.10843698834756987,2017-10-01 +primary_oot,hardship_last_payment_amount,376890,7010,0.018599591392714054,2016-04-01 +probability_calibration,hardship_last_payment_amount,14101,0,0.0, +censored_extension,hardship_length,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_length,49007,0,0.0, +maturity_gap,hardship_length,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_length,17433,0,0.0, +policy_development,hardship_length,94885,0,0.0, +post_extension,hardship_length,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,hardship_length,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_length,14101,0,0.0, +censored_extension,hardship_loan_status,88227,3570,0.04046380359753817,2017-07-01 +conformal_fit,hardship_loan_status,49007,0,0.0, +maturity_gap,hardship_loan_status,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_loan_status,17433,0,0.0, +policy_development,hardship_loan_status,94885,0,0.0, +post_extension,hardship_loan_status,877671,74246,0.08459434115972841,2017-10-01 +primary_oot,hardship_loan_status,376890,5252,0.013935100427180344,2016-04-01 +probability_calibration,hardship_loan_status,14101,0,0.0, +censored_extension,hardship_payoff_balance_amount,88227,5228,0.059256236752921444,2017-07-01 +conformal_fit,hardship_payoff_balance_amount,49007,0,0.0, +maturity_gap,hardship_payoff_balance_amount,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_payoff_balance_amount,17433,0,0.0, +policy_development,hardship_payoff_balance_amount,94885,0,0.0, +post_extension,hardship_payoff_balance_amount,877671,95172,0.10843698834756987,2017-10-01 +primary_oot,hardship_payoff_balance_amount,376890,7010,0.018599591392714054,2016-04-01 +probability_calibration,hardship_payoff_balance_amount,14101,0,0.0, +censored_extension,hardship_reason,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_reason,49007,0,0.0, +maturity_gap,hardship_reason,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_reason,17433,0,0.0, +policy_development,hardship_reason,94885,0,0.0, +post_extension,hardship_reason,877671,74391,0.08475955113020711,2017-10-01 +primary_oot,hardship_reason,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_reason,14101,0,0.0, +censored_extension,hardship_start_date,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_start_date,49007,0,0.0, +maturity_gap,hardship_start_date,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_start_date,17433,0,0.0, +policy_development,hardship_start_date,94885,0,0.0, +post_extension,hardship_start_date,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,hardship_start_date,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_start_date,14101,0,0.0, +censored_extension,hardship_status,88227,3578,0.04055447878767271,2017-07-01 +conformal_fit,hardship_status,49007,0,0.0, +maturity_gap,hardship_status,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_status,17433,0,0.0, +policy_development,hardship_status,94885,0,0.0, +post_extension,hardship_status,877671,74395,0.0847641086466341,2017-10-01 +primary_oot,hardship_status,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_status,14101,0,0.0, +censored_extension,hardship_type,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,hardship_type,49007,0,0.0, +maturity_gap,hardship_type,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,hardship_type,17433,0,0.0, +policy_development,hardship_type,94885,0,0.0, +post_extension,hardship_type,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,hardship_type,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,hardship_type,14101,0,0.0, +censored_extension,home_ownership,88227,88227,1.0,2017-07-01 +conformal_fit,home_ownership,49007,49007,1.0,2012-01-01 +maturity_gap,home_ownership,541863,541863,1.0,2014-01-01 +pd_development,home_ownership,17433,17433,1.0,2007-06-01 +policy_development,home_ownership,94885,94885,1.0,2013-02-01 +post_extension,home_ownership,877671,877671,1.0,2017-10-01 +primary_oot,home_ownership,376890,376890,1.0,2016-04-01 +probability_calibration,home_ownership,14101,14101,1.0,2011-01-01 +censored_extension,il_util,88227,75393,0.8545343262266653,2017-07-01 +conformal_fit,il_util,49007,0,0.0, +maturity_gap,il_util,541863,95108,0.17552038061281172,2015-12-01 +pd_development,il_util,17433,0,0.0, +policy_development,il_util,94885,0,0.0, +post_extension,il_util,877671,742921,0.8464686653654957,2017-10-01 +primary_oot,il_util,376890,324603,0.861267213245244,2016-04-01 +probability_calibration,il_util,14101,0,0.0, +censored_extension,initial_list_status,88227,88227,1.0,2017-07-01 +conformal_fit,initial_list_status,49007,49007,1.0,2012-01-01 +maturity_gap,initial_list_status,541863,541863,1.0,2014-01-01 +pd_development,initial_list_status,17433,17433,1.0,2007-06-01 +policy_development,initial_list_status,94885,94885,1.0,2013-02-01 +post_extension,initial_list_status,877671,877671,1.0,2017-10-01 +primary_oot,initial_list_status,376890,376890,1.0,2016-04-01 +probability_calibration,initial_list_status,14101,14101,1.0,2011-01-01 +censored_extension,inq_fi,88227,88227,1.0,2017-07-01 +conformal_fit,inq_fi,49007,0,0.0, +maturity_gap,inq_fi,541863,111012,0.20487097292119963,2015-12-01 +pd_development,inq_fi,17433,0,0.0, +policy_development,inq_fi,94885,0,0.0, +post_extension,inq_fi,877671,877671,1.0,2017-10-01 +primary_oot,inq_fi,376890,376890,1.0,2016-04-01 +probability_calibration,inq_fi,14101,0,0.0, +censored_extension,inq_last_12m,88227,88227,1.0,2017-07-01 +conformal_fit,inq_last_12m,49007,0,0.0, +maturity_gap,inq_last_12m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,inq_last_12m,17433,0,0.0, +policy_development,inq_last_12m,94885,0,0.0, +post_extension,inq_last_12m,877671,877671,1.0,2017-10-01 +primary_oot,inq_last_12m,376890,376890,1.0,2016-04-01 +probability_calibration,inq_last_12m,14101,0,0.0, +censored_extension,inq_last_6mths,88227,88227,1.0,2017-07-01 +conformal_fit,inq_last_6mths,49007,49007,1.0,2012-01-01 +maturity_gap,inq_last_6mths,541863,541863,1.0,2014-01-01 +pd_development,inq_last_6mths,17433,17404,0.9983364882693742,2007-06-01 +policy_development,inq_last_6mths,94885,94885,1.0,2013-02-01 +post_extension,inq_last_6mths,877671,877671,1.0,2017-10-01 +primary_oot,inq_last_6mths,376890,376890,1.0,2016-04-01 +probability_calibration,inq_last_6mths,14101,14101,1.0,2011-01-01 +censored_extension,installment,88227,88227,1.0,2017-07-01 +conformal_fit,installment,49007,49007,1.0,2012-01-01 +maturity_gap,installment,541863,541863,1.0,2014-01-01 +pd_development,installment,17433,17433,1.0,2007-06-01 +policy_development,installment,94885,94885,1.0,2013-02-01 +post_extension,installment,877671,877671,1.0,2017-10-01 +primary_oot,installment,376890,376890,1.0,2016-04-01 +probability_calibration,installment,14101,14101,1.0,2011-01-01 +censored_extension,int_rate,88227,88227,1.0,2017-07-01 +conformal_fit,int_rate,49007,49007,1.0,2012-01-01 +maturity_gap,int_rate,541863,541863,1.0,2014-01-01 +pd_development,int_rate,17433,17433,1.0,2007-06-01 +policy_development,int_rate,94885,94885,1.0,2013-02-01 +post_extension,int_rate,877671,877671,1.0,2017-10-01 +primary_oot,int_rate,376890,376890,1.0,2016-04-01 +probability_calibration,int_rate,14101,14101,1.0,2011-01-01 +censored_extension,last_credit_pull_d,88227,88226,0.9999886656012332,2017-07-01 +conformal_fit,last_credit_pull_d,49007,49007,1.0,2012-01-01 +maturity_gap,last_credit_pull_d,541863,541837,0.99995201739185,2014-01-01 +pd_development,last_credit_pull_d,17433,17429,0.9997705501061206,2007-06-01 +policy_development,last_credit_pull_d,94885,94877,0.9999156874110766,2013-02-01 +post_extension,last_credit_pull_d,877671,877658,0.9999851880716123,2017-10-01 +primary_oot,last_credit_pull_d,376890,376879,0.9999708137652896,2016-04-01 +probability_calibration,last_credit_pull_d,14101,14101,1.0,2011-01-01 +censored_extension,last_fico_range_high,88227,88227,1.0,2017-07-01 +conformal_fit,last_fico_range_high,49007,49007,1.0,2012-01-01 +maturity_gap,last_fico_range_high,541863,541863,1.0,2014-01-01 +pd_development,last_fico_range_high,17433,17433,1.0,2007-06-01 +policy_development,last_fico_range_high,94885,94885,1.0,2013-02-01 +post_extension,last_fico_range_high,877671,877671,1.0,2017-10-01 +primary_oot,last_fico_range_high,376890,376890,1.0,2016-04-01 +probability_calibration,last_fico_range_high,14101,14101,1.0,2011-01-01 +censored_extension,last_fico_range_low,88227,88227,1.0,2017-07-01 +conformal_fit,last_fico_range_low,49007,49007,1.0,2012-01-01 +maturity_gap,last_fico_range_low,541863,541863,1.0,2014-01-01 +pd_development,last_fico_range_low,17433,17433,1.0,2007-06-01 +policy_development,last_fico_range_low,94885,94885,1.0,2013-02-01 +post_extension,last_fico_range_low,877671,877671,1.0,2017-10-01 +primary_oot,last_fico_range_low,376890,376890,1.0,2016-04-01 +probability_calibration,last_fico_range_low,14101,14101,1.0,2011-01-01 +censored_extension,last_pymnt_amnt,88227,88227,1.0,2017-07-01 +conformal_fit,last_pymnt_amnt,49007,49007,1.0,2012-01-01 +maturity_gap,last_pymnt_amnt,541863,541863,1.0,2014-01-01 +pd_development,last_pymnt_amnt,17433,17433,1.0,2007-06-01 +policy_development,last_pymnt_amnt,94885,94885,1.0,2013-02-01 +post_extension,last_pymnt_amnt,877671,877671,1.0,2017-10-01 +primary_oot,last_pymnt_amnt,376890,376890,1.0,2016-04-01 +probability_calibration,last_pymnt_amnt,14101,14101,1.0,2011-01-01 +censored_extension,last_pymnt_d,88227,88087,0.9984131841726456,2017-07-01 +conformal_fit,last_pymnt_d,49007,48953,0.9988981165955884,2012-01-01 +maturity_gap,last_pymnt_d,541863,541458,0.9992525786038168,2014-01-01 +pd_development,last_pymnt_d,17433,17392,0.9976481385877359,2007-06-01 +policy_development,last_pymnt_d,94885,94808,0.9991884913316119,2013-02-01 +post_extension,last_pymnt_d,877671,875157,0.9971356009256316,2017-10-01 +primary_oot,last_pymnt_d,376890,376412,0.9987317254371302,2016-04-01 +probability_calibration,last_pymnt_d,14101,14077,0.9982979930501383,2011-01-01 +censored_extension,loan_amnt,88227,88227,1.0,2017-07-01 +conformal_fit,loan_amnt,49007,49007,1.0,2012-01-01 +maturity_gap,loan_amnt,541863,541863,1.0,2014-01-01 +pd_development,loan_amnt,17433,17433,1.0,2007-06-01 +policy_development,loan_amnt,94885,94885,1.0,2013-02-01 +post_extension,loan_amnt,877671,877671,1.0,2017-10-01 +primary_oot,loan_amnt,376890,376890,1.0,2016-04-01 +probability_calibration,loan_amnt,14101,14101,1.0,2011-01-01 +censored_extension,loan_status,88227,88227,1.0,2017-07-01 +conformal_fit,loan_status,49007,49007,1.0,2012-01-01 +maturity_gap,loan_status,541863,541863,1.0,2014-01-01 +pd_development,loan_status,17433,17433,1.0,2007-06-01 +policy_development,loan_status,94885,94885,1.0,2013-02-01 +post_extension,loan_status,877671,877671,1.0,2017-10-01 +primary_oot,loan_status,376890,376890,1.0,2016-04-01 +probability_calibration,loan_status,14101,14101,1.0,2011-01-01 +censored_extension,max_bal_bc,88227,88227,1.0,2017-07-01 +conformal_fit,max_bal_bc,49007,0,0.0, +maturity_gap,max_bal_bc,541863,111012,0.20487097292119963,2015-12-01 +pd_development,max_bal_bc,17433,0,0.0, +policy_development,max_bal_bc,94885,0,0.0, +post_extension,max_bal_bc,877671,877671,1.0,2017-10-01 +primary_oot,max_bal_bc,376890,376890,1.0,2016-04-01 +probability_calibration,max_bal_bc,14101,0,0.0, +censored_extension,mo_sin_old_il_acct,88227,85283,0.9666315300304895,2017-07-01 +conformal_fit,mo_sin_old_il_acct,49007,25127,0.5127226722713082,2012-08-01 +maturity_gap,mo_sin_old_il_acct,541863,522912,0.9650262151134142,2014-01-01 +pd_development,mo_sin_old_il_acct,17433,0,0.0, +policy_development,mo_sin_old_il_acct,94885,90835,0.9573167518575117,2013-02-01 +post_extension,mo_sin_old_il_acct,877671,846638,0.9646416481802407,2017-10-01 +primary_oot,mo_sin_old_il_acct,376890,365451,0.9696489691952559,2016-04-01 +probability_calibration,mo_sin_old_il_acct,14101,0,0.0, +censored_extension,mo_sin_old_rev_tl_op,88227,88227,1.0,2017-07-01 +conformal_fit,mo_sin_old_rev_tl_op,49007,26456,0.5398412471687718,2012-08-01 +maturity_gap,mo_sin_old_rev_tl_op,541863,541863,1.0,2014-01-01 +pd_development,mo_sin_old_rev_tl_op,17433,0,0.0, +policy_development,mo_sin_old_rev_tl_op,94885,94885,1.0,2013-02-01 +post_extension,mo_sin_old_rev_tl_op,877671,877671,1.0,2017-10-01 +primary_oot,mo_sin_old_rev_tl_op,376890,376890,1.0,2016-04-01 +probability_calibration,mo_sin_old_rev_tl_op,14101,0,0.0, +censored_extension,mo_sin_rcnt_rev_tl_op,88227,88227,1.0,2017-07-01 +conformal_fit,mo_sin_rcnt_rev_tl_op,49007,26456,0.5398412471687718,2012-08-01 +maturity_gap,mo_sin_rcnt_rev_tl_op,541863,541863,1.0,2014-01-01 +pd_development,mo_sin_rcnt_rev_tl_op,17433,0,0.0, +policy_development,mo_sin_rcnt_rev_tl_op,94885,94885,1.0,2013-02-01 +post_extension,mo_sin_rcnt_rev_tl_op,877671,877671,1.0,2017-10-01 +primary_oot,mo_sin_rcnt_rev_tl_op,376890,376890,1.0,2016-04-01 +probability_calibration,mo_sin_rcnt_rev_tl_op,14101,0,0.0, +censored_extension,mo_sin_rcnt_tl,88227,88227,1.0,2017-07-01 +conformal_fit,mo_sin_rcnt_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,mo_sin_rcnt_tl,541863,541863,1.0,2014-01-01 +pd_development,mo_sin_rcnt_tl,17433,0,0.0, +policy_development,mo_sin_rcnt_tl,94885,94885,1.0,2013-02-01 +post_extension,mo_sin_rcnt_tl,877671,877671,1.0,2017-10-01 +primary_oot,mo_sin_rcnt_tl,376890,376890,1.0,2016-04-01 +probability_calibration,mo_sin_rcnt_tl,14101,0,0.0, +censored_extension,mort_acc,88227,88227,1.0,2017-07-01 +conformal_fit,mort_acc,49007,42982,0.8770583794151856,2012-03-01 +maturity_gap,mort_acc,541863,541863,1.0,2014-01-01 +pd_development,mort_acc,17433,0,0.0, +policy_development,mort_acc,94885,94885,1.0,2013-02-01 +post_extension,mort_acc,877671,877671,1.0,2017-10-01 +primary_oot,mort_acc,376890,376890,1.0,2016-04-01 +probability_calibration,mort_acc,14101,0,0.0, +censored_extension,mths_since_last_delinq,88227,45046,0.5105693268500573,2017-07-01 +conformal_fit,mths_since_last_delinq,49007,20201,0.4122064194910931,2012-01-01 +maturity_gap,mths_since_last_delinq,541863,280616,0.517872598793422,2014-01-01 +pd_development,mths_since_last_delinq,17433,7049,0.4043480754890151,2007-06-01 +policy_development,mths_since_last_delinq,94885,41288,0.43513727143384096,2013-02-01 +post_extension,mths_since_last_delinq,877671,393306,0.44812463895924554,2017-10-01 +primary_oot,mths_since_last_delinq,376890,199207,0.5285547507230226,2016-04-01 +probability_calibration,mths_since_last_delinq,14101,4739,0.33607545564144387,2011-01-01 +censored_extension,mths_since_last_major_derog,88227,24359,0.2760946195609054,2017-07-01 +conformal_fit,mths_since_last_major_derog,49007,5507,0.11237170200175485,2012-08-01 +maturity_gap,mths_since_last_major_derog,541863,161058,0.29723011167029306,2014-01-01 +pd_development,mths_since_last_major_derog,17433,0,0.0, +policy_development,mths_since_last_major_derog,94885,19510,0.20561732623702378,2013-02-01 +post_extension,mths_since_last_major_derog,877671,204263,0.23273299448198698,2017-10-01 +primary_oot,mths_since_last_major_derog,376890,109114,0.2895115285627106,2016-04-01 +probability_calibration,mths_since_last_major_derog,14101,0,0.0, +censored_extension,mths_since_last_record,88227,13525,0.15329774332120552,2017-07-01 +conformal_fit,mths_since_last_record,49007,2258,0.04607505050298937,2012-01-01 +maturity_gap,mths_since_last_record,541863,101142,0.1866560366734765,2014-01-01 +pd_development,mths_since_last_record,17433,2279,0.13072907703780187,2007-06-01 +policy_development,mths_since_last_record,94885,11586,0.12210570690836275,2013-02-01 +post_extension,mths_since_last_record,877671,102884,0.11722388001882254,2017-10-01 +primary_oot,mths_since_last_record,376890,74664,0.19810554803788905,2016-04-01 +probability_calibration,mths_since_last_record,14101,724,0.05134387632082831,2011-01-01 +censored_extension,mths_since_rcnt_il,88227,85283,0.9666315300304895,2017-07-01 +conformal_fit,mths_since_rcnt_il,49007,0,0.0, +maturity_gap,mths_since_rcnt_il,541863,107494,0.19837855694151474,2015-12-01 +pd_development,mths_since_rcnt_il,17433,0,0.0, +policy_development,mths_since_rcnt_il,94885,0,0.0, +post_extension,mths_since_rcnt_il,877671,846638,0.9646416481802407,2017-10-01 +primary_oot,mths_since_rcnt_il,376890,365451,0.9696489691952559,2016-04-01 +probability_calibration,mths_since_rcnt_il,14101,0,0.0, +censored_extension,mths_since_recent_bc,88227,86955,0.9855826447686082,2017-07-01 +conformal_fit,mths_since_recent_bc,49007,42576,0.8687738486338686,2012-03-01 +maturity_gap,mths_since_recent_bc,541863,536726,0.9905197439205112,2014-01-01 +pd_development,mths_since_recent_bc,17433,0,0.0, +policy_development,mths_since_recent_bc,94885,94221,0.993002055119355,2013-02-01 +post_extension,mths_since_recent_bc,877671,866584,0.9873677038434675,2017-10-01 +primary_oot,mths_since_recent_bc,376890,372525,0.9884183714081032,2016-04-01 +probability_calibration,mths_since_recent_bc,14101,0,0.0, +censored_extension,mths_since_recent_bc_dlq,88227,21199,0.24027791945776236,2017-07-01 +conformal_fit,mths_since_recent_bc_dlq,49007,6748,0.13769461505499214,2012-08-01 +maturity_gap,mths_since_recent_bc_dlq,541863,142316,0.26264203313383644,2014-01-01 +pd_development,mths_since_recent_bc_dlq,17433,0,0.0, +policy_development,mths_since_recent_bc_dlq,94885,21258,0.22403962691679402,2013-02-01 +post_extension,mths_since_recent_bc_dlq,877671,181016,0.20624584838737978,2017-10-01 +primary_oot,mths_since_recent_bc_dlq,376890,95571,0.2535779670460877,2016-04-01 +probability_calibration,mths_since_recent_bc_dlq,14101,0,0.0, +censored_extension,mths_since_recent_inq,88227,77926,0.8832443583030138,2017-07-01 +conformal_fit,mths_since_recent_inq,49007,37444,0.7640541147183055,2012-03-01 +maturity_gap,mths_since_recent_inq,541863,484954,0.8949752981842274,2014-01-01 +pd_development,mths_since_recent_inq,17433,0,0.0, +policy_development,mths_since_recent_inq,94885,84244,0.8878537176582179,2013-02-01 +post_extension,mths_since_recent_inq,877671,772491,0.8801601055520805,2017-10-01 +primary_oot,mths_since_recent_inq,376890,334303,0.8870041656716814,2016-04-01 +probability_calibration,mths_since_recent_inq,14101,0,0.0, +censored_extension,mths_since_recent_revol_delinq,88227,30834,0.34948485157604814,2017-07-01 +conformal_fit,mths_since_recent_revol_delinq,49007,13042,0.2661252474136348,2012-03-01 +maturity_gap,mths_since_recent_revol_delinq,541863,197641,0.3647434868223149,2014-01-01 +pd_development,mths_since_recent_revol_delinq,17433,0,0.0, +policy_development,mths_since_recent_revol_delinq,94885,28666,0.3021130842598936,2013-02-01 +post_extension,mths_since_recent_revol_delinq,877671,262777,0.2994026235343312,2017-10-01 +primary_oot,mths_since_recent_revol_delinq,376890,136865,0.36314309214890284,2016-04-01 +probability_calibration,mths_since_recent_revol_delinq,14101,0,0.0, +censored_extension,next_pymnt_d,88227,28725,0.32558060457683025,2017-07-01 +conformal_fit,next_pymnt_d,49007,0,0.0, +maturity_gap,next_pymnt_d,541863,0,0.0, +pd_development,next_pymnt_d,17433,2438,0.139849710319509,2007-06-01 +policy_development,next_pymnt_d,94885,0,0.0, +post_extension,next_pymnt_d,877671,624064,0.7110454828745624,2017-10-01 +primary_oot,next_pymnt_d,376890,11551,0.03064819973997718,2016-08-01 +probability_calibration,next_pymnt_d,14101,0,0.0, +censored_extension,num_accts_ever_120_pd,88227,88227,1.0,2017-07-01 +conformal_fit,num_accts_ever_120_pd,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_accts_ever_120_pd,541863,541863,1.0,2014-01-01 +pd_development,num_accts_ever_120_pd,17433,0,0.0, +policy_development,num_accts_ever_120_pd,94885,94885,1.0,2013-02-01 +post_extension,num_accts_ever_120_pd,877671,877671,1.0,2017-10-01 +primary_oot,num_accts_ever_120_pd,376890,376890,1.0,2016-04-01 +probability_calibration,num_accts_ever_120_pd,14101,0,0.0, +censored_extension,num_actv_bc_tl,88227,88227,1.0,2017-07-01 +conformal_fit,num_actv_bc_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_actv_bc_tl,541863,541863,1.0,2014-01-01 +pd_development,num_actv_bc_tl,17433,0,0.0, +policy_development,num_actv_bc_tl,94885,94885,1.0,2013-02-01 +post_extension,num_actv_bc_tl,877671,877671,1.0,2017-10-01 +primary_oot,num_actv_bc_tl,376890,376890,1.0,2016-04-01 +probability_calibration,num_actv_bc_tl,14101,0,0.0, +censored_extension,num_actv_rev_tl,88227,88227,1.0,2017-07-01 +conformal_fit,num_actv_rev_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_actv_rev_tl,541863,541863,1.0,2014-01-01 +pd_development,num_actv_rev_tl,17433,0,0.0, +policy_development,num_actv_rev_tl,94885,94885,1.0,2013-02-01 +post_extension,num_actv_rev_tl,877671,877671,1.0,2017-10-01 +primary_oot,num_actv_rev_tl,376890,376890,1.0,2016-04-01 +probability_calibration,num_actv_rev_tl,14101,0,0.0, +censored_extension,num_bc_sats,88227,88227,1.0,2017-07-01 +conformal_fit,num_bc_sats,49007,36031,0.7352214989695349,2012-06-01 +maturity_gap,num_bc_sats,541863,541863,1.0,2014-01-01 +pd_development,num_bc_sats,17433,0,0.0, +policy_development,num_bc_sats,94885,94885,1.0,2013-02-01 +post_extension,num_bc_sats,877671,877671,1.0,2017-10-01 +primary_oot,num_bc_sats,376890,376890,1.0,2016-04-01 +probability_calibration,num_bc_sats,14101,0,0.0, +censored_extension,num_bc_tl,88227,88227,1.0,2017-07-01 +conformal_fit,num_bc_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_bc_tl,541863,541863,1.0,2014-01-01 +pd_development,num_bc_tl,17433,0,0.0, +policy_development,num_bc_tl,94885,94885,1.0,2013-02-01 +post_extension,num_bc_tl,877671,877671,1.0,2017-10-01 +primary_oot,num_bc_tl,376890,376890,1.0,2016-04-01 +probability_calibration,num_bc_tl,14101,0,0.0, +censored_extension,num_il_tl,88227,88227,1.0,2017-07-01 +conformal_fit,num_il_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_il_tl,541863,541863,1.0,2014-01-01 +pd_development,num_il_tl,17433,0,0.0, +policy_development,num_il_tl,94885,94885,1.0,2013-02-01 +post_extension,num_il_tl,877671,877671,1.0,2017-10-01 +primary_oot,num_il_tl,376890,376890,1.0,2016-04-01 +probability_calibration,num_il_tl,14101,0,0.0, +censored_extension,num_op_rev_tl,88227,88227,1.0,2017-07-01 +conformal_fit,num_op_rev_tl,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_op_rev_tl,541863,541863,1.0,2014-01-01 +pd_development,num_op_rev_tl,17433,0,0.0, +policy_development,num_op_rev_tl,94885,94885,1.0,2013-02-01 +post_extension,num_op_rev_tl,877671,877671,1.0,2017-10-01 +primary_oot,num_op_rev_tl,376890,376890,1.0,2016-04-01 +probability_calibration,num_op_rev_tl,14101,0,0.0, +censored_extension,num_rev_accts,88227,88227,1.0,2017-07-01 +conformal_fit,num_rev_accts,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_rev_accts,541863,541862,0.9999981545150711,2014-01-01 +pd_development,num_rev_accts,17433,0,0.0, +policy_development,num_rev_accts,94885,94885,1.0,2013-02-01 +post_extension,num_rev_accts,877671,877671,1.0,2017-10-01 +primary_oot,num_rev_accts,376890,376890,1.0,2016-04-01 +probability_calibration,num_rev_accts,14101,0,0.0, +censored_extension,num_rev_tl_bal_gt_0,88227,88227,1.0,2017-07-01 +conformal_fit,num_rev_tl_bal_gt_0,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_rev_tl_bal_gt_0,541863,541863,1.0,2014-01-01 +pd_development,num_rev_tl_bal_gt_0,17433,0,0.0, +policy_development,num_rev_tl_bal_gt_0,94885,94885,1.0,2013-02-01 +post_extension,num_rev_tl_bal_gt_0,877671,877671,1.0,2017-10-01 +primary_oot,num_rev_tl_bal_gt_0,376890,376890,1.0,2016-04-01 +probability_calibration,num_rev_tl_bal_gt_0,14101,0,0.0, +censored_extension,num_sats,88227,88227,1.0,2017-07-01 +conformal_fit,num_sats,49007,36031,0.7352214989695349,2012-06-01 +maturity_gap,num_sats,541863,541863,1.0,2014-01-01 +pd_development,num_sats,17433,0,0.0, +policy_development,num_sats,94885,94885,1.0,2013-02-01 +post_extension,num_sats,877671,877671,1.0,2017-10-01 +primary_oot,num_sats,376890,376890,1.0,2016-04-01 +probability_calibration,num_sats,14101,0,0.0, +censored_extension,num_tl_120dpd_2m,88227,84791,0.9610550058372154,2017-07-01 +conformal_fit,num_tl_120dpd_2m,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_tl_120dpd_2m,541863,516019,0.9523052874988697,2014-01-01 +pd_development,num_tl_120dpd_2m,17433,0,0.0, +policy_development,num_tl_120dpd_2m,94885,94672,0.9977551773199136,2013-02-01 +post_extension,num_tl_120dpd_2m,877671,859416,0.9792006344062867,2017-10-01 +primary_oot,num_tl_120dpd_2m,376890,360842,0.9574199368516013,2016-04-01 +probability_calibration,num_tl_120dpd_2m,14101,0,0.0, +censored_extension,num_tl_30dpd,88227,88227,1.0,2017-07-01 +conformal_fit,num_tl_30dpd,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_tl_30dpd,541863,541863,1.0,2014-01-01 +pd_development,num_tl_30dpd,17433,0,0.0, +policy_development,num_tl_30dpd,94885,94885,1.0,2013-02-01 +post_extension,num_tl_30dpd,877671,877671,1.0,2017-10-01 +primary_oot,num_tl_30dpd,376890,376890,1.0,2016-04-01 +probability_calibration,num_tl_30dpd,14101,0,0.0, +censored_extension,num_tl_90g_dpd_24m,88227,88227,1.0,2017-07-01 +conformal_fit,num_tl_90g_dpd_24m,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_tl_90g_dpd_24m,541863,541863,1.0,2014-01-01 +pd_development,num_tl_90g_dpd_24m,17433,0,0.0, +policy_development,num_tl_90g_dpd_24m,94885,94885,1.0,2013-02-01 +post_extension,num_tl_90g_dpd_24m,877671,877671,1.0,2017-10-01 +primary_oot,num_tl_90g_dpd_24m,376890,376890,1.0,2016-04-01 +probability_calibration,num_tl_90g_dpd_24m,14101,0,0.0, +censored_extension,num_tl_op_past_12m,88227,88227,1.0,2017-07-01 +conformal_fit,num_tl_op_past_12m,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,num_tl_op_past_12m,541863,541863,1.0,2014-01-01 +pd_development,num_tl_op_past_12m,17433,0,0.0, +policy_development,num_tl_op_past_12m,94885,94885,1.0,2013-02-01 +post_extension,num_tl_op_past_12m,877671,877671,1.0,2017-10-01 +primary_oot,num_tl_op_past_12m,376890,376890,1.0,2016-04-01 +probability_calibration,num_tl_op_past_12m,14101,0,0.0, +censored_extension,open_acc,88227,88227,1.0,2017-07-01 +conformal_fit,open_acc,49007,49007,1.0,2012-01-01 +maturity_gap,open_acc,541863,541863,1.0,2014-01-01 +pd_development,open_acc,17433,17404,0.9983364882693742,2007-06-01 +policy_development,open_acc,94885,94885,1.0,2013-02-01 +post_extension,open_acc,877671,877671,1.0,2017-10-01 +primary_oot,open_acc,376890,376890,1.0,2016-04-01 +probability_calibration,open_acc,14101,14101,1.0,2011-01-01 +censored_extension,open_acc_6m,88227,88227,1.0,2017-07-01 +conformal_fit,open_acc_6m,49007,0,0.0, +maturity_gap,open_acc_6m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_acc_6m,17433,0,0.0, +policy_development,open_acc_6m,94885,0,0.0, +post_extension,open_acc_6m,877671,877671,1.0,2017-10-01 +primary_oot,open_acc_6m,376890,376890,1.0,2016-04-01 +probability_calibration,open_acc_6m,14101,0,0.0, +censored_extension,open_act_il,88227,88227,1.0,2017-07-01 +conformal_fit,open_act_il,49007,0,0.0, +maturity_gap,open_act_il,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_act_il,17433,0,0.0, +policy_development,open_act_il,94885,0,0.0, +post_extension,open_act_il,877671,877671,1.0,2017-10-01 +primary_oot,open_act_il,376890,376890,1.0,2016-04-01 +probability_calibration,open_act_il,14101,0,0.0, +censored_extension,open_il_12m,88227,88227,1.0,2017-07-01 +conformal_fit,open_il_12m,49007,0,0.0, +maturity_gap,open_il_12m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_il_12m,17433,0,0.0, +policy_development,open_il_12m,94885,0,0.0, +post_extension,open_il_12m,877671,877671,1.0,2017-10-01 +primary_oot,open_il_12m,376890,376890,1.0,2016-04-01 +probability_calibration,open_il_12m,14101,0,0.0, +censored_extension,open_il_24m,88227,88227,1.0,2017-07-01 +conformal_fit,open_il_24m,49007,0,0.0, +maturity_gap,open_il_24m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_il_24m,17433,0,0.0, +policy_development,open_il_24m,94885,0,0.0, +post_extension,open_il_24m,877671,877671,1.0,2017-10-01 +primary_oot,open_il_24m,376890,376890,1.0,2016-04-01 +probability_calibration,open_il_24m,14101,0,0.0, +censored_extension,open_rv_12m,88227,88227,1.0,2017-07-01 +conformal_fit,open_rv_12m,49007,0,0.0, +maturity_gap,open_rv_12m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_rv_12m,17433,0,0.0, +policy_development,open_rv_12m,94885,0,0.0, +post_extension,open_rv_12m,877671,877671,1.0,2017-10-01 +primary_oot,open_rv_12m,376890,376890,1.0,2016-04-01 +probability_calibration,open_rv_12m,14101,0,0.0, +censored_extension,open_rv_24m,88227,88227,1.0,2017-07-01 +conformal_fit,open_rv_24m,49007,0,0.0, +maturity_gap,open_rv_24m,541863,111012,0.20487097292119963,2015-12-01 +pd_development,open_rv_24m,17433,0,0.0, +policy_development,open_rv_24m,94885,0,0.0, +post_extension,open_rv_24m,877671,877671,1.0,2017-10-01 +primary_oot,open_rv_24m,376890,376890,1.0,2016-04-01 +probability_calibration,open_rv_24m,14101,0,0.0, +censored_extension,orig_projected_additional_accrued_interest,88227,5094,0.05773742731816791,2017-07-01 +conformal_fit,orig_projected_additional_accrued_interest,49007,0,0.0, +maturity_gap,orig_projected_additional_accrued_interest,541863,1800,0.0033218728719251914,2014-08-01 +pd_development,orig_projected_additional_accrued_interest,17433,0,0.0, +policy_development,orig_projected_additional_accrued_interest,94885,0,0.0, +post_extension,orig_projected_additional_accrued_interest,877671,94886,0.1081111259230395,2017-10-01 +primary_oot,orig_projected_additional_accrued_interest,376890,6199,0.01644776990633872,2016-04-01 +probability_calibration,orig_projected_additional_accrued_interest,14101,0,0.0, +censored_extension,out_prncp,88227,88227,1.0,2017-07-01 +conformal_fit,out_prncp,49007,49007,1.0,2012-01-01 +maturity_gap,out_prncp,541863,541863,1.0,2014-01-01 +pd_development,out_prncp,17433,17433,1.0,2007-06-01 +policy_development,out_prncp,94885,94885,1.0,2013-02-01 +post_extension,out_prncp,877671,877671,1.0,2017-10-01 +primary_oot,out_prncp,376890,376890,1.0,2016-04-01 +probability_calibration,out_prncp,14101,14101,1.0,2011-01-01 +censored_extension,out_prncp_inv,88227,88227,1.0,2017-07-01 +conformal_fit,out_prncp_inv,49007,49007,1.0,2012-01-01 +maturity_gap,out_prncp_inv,541863,541863,1.0,2014-01-01 +pd_development,out_prncp_inv,17433,17433,1.0,2007-06-01 +policy_development,out_prncp_inv,94885,94885,1.0,2013-02-01 +post_extension,out_prncp_inv,877671,877671,1.0,2017-10-01 +primary_oot,out_prncp_inv,376890,376890,1.0,2016-04-01 +probability_calibration,out_prncp_inv,14101,14101,1.0,2011-01-01 +censored_extension,payment_plan_start_date,88227,3579,0.04056581318643952,2017-07-01 +conformal_fit,payment_plan_start_date,49007,0,0.0, +maturity_gap,payment_plan_start_date,541863,2205,0.0040692942681083595,2014-08-01 +pd_development,payment_plan_start_date,17433,0,0.0, +policy_development,payment_plan_start_date,94885,0,0.0, +post_extension,payment_plan_start_date,877671,74396,0.08476524802574086,2017-10-01 +primary_oot,payment_plan_start_date,376890,5254,0.013940407015309507,2016-04-01 +probability_calibration,payment_plan_start_date,14101,0,0.0, +censored_extension,pct_tl_nvr_dlq,88227,88227,1.0,2017-07-01 +conformal_fit,pct_tl_nvr_dlq,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,pct_tl_nvr_dlq,541863,541863,1.0,2014-01-01 +pd_development,pct_tl_nvr_dlq,17433,0,0.0, +policy_development,pct_tl_nvr_dlq,94885,94771,0.9987985456078411,2013-02-01 +post_extension,pct_tl_nvr_dlq,877671,877669,0.9999977212417865,2017-10-01 +primary_oot,pct_tl_nvr_dlq,376890,376890,1.0,2016-04-01 +probability_calibration,pct_tl_nvr_dlq,14101,0,0.0, +censored_extension,percent_bc_gt_75,88227,86878,0.9847098960635633,2017-07-01 +conformal_fit,percent_bc_gt_75,49007,42516,0.8675495337400779,2012-03-01 +maturity_gap,percent_bc_gt_75,541863,536046,0.9892648141688951,2014-01-01 +pd_development,percent_bc_gt_75,17433,0,0.0, +policy_development,percent_bc_gt_75,94885,94125,0.9919903040522738,2013-02-01 +post_extension,percent_bc_gt_75,877671,865842,0.9865222845462593,2017-10-01 +primary_oot,percent_bc_gt_75,376890,372279,0.9877656610682162,2016-04-01 +probability_calibration,percent_bc_gt_75,14101,0,0.0, +censored_extension,policy_code,88227,88227,1.0,2017-07-01 +conformal_fit,policy_code,49007,49007,1.0,2012-01-01 +maturity_gap,policy_code,541863,541863,1.0,2014-01-01 +pd_development,policy_code,17433,17433,1.0,2007-06-01 +policy_development,policy_code,94885,94885,1.0,2013-02-01 +post_extension,policy_code,877671,877671,1.0,2017-10-01 +primary_oot,policy_code,376890,376890,1.0,2016-04-01 +probability_calibration,policy_code,14101,14101,1.0,2011-01-01 +censored_extension,pub_rec,88227,88227,1.0,2017-07-01 +conformal_fit,pub_rec,49007,49007,1.0,2012-01-01 +maturity_gap,pub_rec,541863,541863,1.0,2014-01-01 +pd_development,pub_rec,17433,17404,0.9983364882693742,2007-06-01 +policy_development,pub_rec,94885,94885,1.0,2013-02-01 +post_extension,pub_rec,877671,877671,1.0,2017-10-01 +primary_oot,pub_rec,376890,376890,1.0,2016-04-01 +probability_calibration,pub_rec,14101,14101,1.0,2011-01-01 +censored_extension,pub_rec_bankruptcies,88227,88227,1.0,2017-07-01 +conformal_fit,pub_rec_bankruptcies,49007,49007,1.0,2012-01-01 +maturity_gap,pub_rec_bankruptcies,541863,541863,1.0,2014-01-01 +pd_development,pub_rec_bankruptcies,17433,16068,0.9217002237136466,2007-08-01 +policy_development,pub_rec_bankruptcies,94885,94885,1.0,2013-02-01 +post_extension,pub_rec_bankruptcies,877671,877671,1.0,2017-10-01 +primary_oot,pub_rec_bankruptcies,376890,376890,1.0,2016-04-01 +probability_calibration,pub_rec_bankruptcies,14101,14101,1.0,2011-01-01 +censored_extension,purpose,88227,88227,1.0,2017-07-01 +conformal_fit,purpose,49007,49007,1.0,2012-01-01 +maturity_gap,purpose,541863,541863,1.0,2014-01-01 +pd_development,purpose,17433,17433,1.0,2007-06-01 +policy_development,purpose,94885,94885,1.0,2013-02-01 +post_extension,purpose,877671,877671,1.0,2017-10-01 +primary_oot,purpose,376890,376890,1.0,2016-04-01 +probability_calibration,purpose,14101,14101,1.0,2011-01-01 +censored_extension,pymnt_plan,88227,88227,1.0,2017-07-01 +conformal_fit,pymnt_plan,49007,49007,1.0,2012-01-01 +maturity_gap,pymnt_plan,541863,541863,1.0,2014-01-01 +pd_development,pymnt_plan,17433,17433,1.0,2007-06-01 +policy_development,pymnt_plan,94885,94885,1.0,2013-02-01 +post_extension,pymnt_plan,877671,877671,1.0,2017-10-01 +primary_oot,pymnt_plan,376890,376890,1.0,2016-04-01 +probability_calibration,pymnt_plan,14101,14101,1.0,2011-01-01 +censored_extension,recoveries,88227,88227,1.0,2017-07-01 +conformal_fit,recoveries,49007,49007,1.0,2012-01-01 +maturity_gap,recoveries,541863,541863,1.0,2014-01-01 +pd_development,recoveries,17433,17433,1.0,2007-06-01 +policy_development,recoveries,94885,94885,1.0,2013-02-01 +post_extension,recoveries,877671,877671,1.0,2017-10-01 +primary_oot,recoveries,376890,376890,1.0,2016-04-01 +probability_calibration,recoveries,14101,14101,1.0,2011-01-01 +censored_extension,revol_bal,88227,88227,1.0,2017-07-01 +conformal_fit,revol_bal,49007,49007,1.0,2012-01-01 +maturity_gap,revol_bal,541863,541863,1.0,2014-01-01 +pd_development,revol_bal,17433,17433,1.0,2007-06-01 +policy_development,revol_bal,94885,94885,1.0,2013-02-01 +post_extension,revol_bal,877671,877671,1.0,2017-10-01 +primary_oot,revol_bal,376890,376890,1.0,2016-04-01 +probability_calibration,revol_bal,14101,14101,1.0,2011-01-01 +censored_extension,revol_bal_joint,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,revol_bal_joint,49007,0,0.0, +maturity_gap,revol_bal_joint,541863,0,0.0, +pd_development,revol_bal_joint,17433,0,0.0, +policy_development,revol_bal_joint,94885,0,0.0, +post_extension,revol_bal_joint,877671,99854,0.11377156132537135,2017-10-01 +primary_oot,revol_bal_joint,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,revol_bal_joint,14101,0,0.0, +censored_extension,revol_util,88227,88125,0.9988438913257847,2017-07-01 +conformal_fit,revol_util,49007,48964,0.9991225743261167,2012-01-01 +maturity_gap,revol_util,541863,541600,0.9995146374637132,2014-01-01 +pd_development,revol_util,17433,17354,0.9954683645958814,2007-06-01 +policy_development,revol_util,94885,94831,0.9994308900247668,2013-02-01 +post_extension,revol_util,877671,876483,0.9986464176211816,2017-10-01 +primary_oot,revol_util,376890,376633,0.9993181034254026,2016-04-01 +probability_calibration,revol_util,14101,14094,0.9995035813062904,2011-01-01 +censored_extension,sec_app_chargeoff_within_12_mths,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_chargeoff_within_12_mths,49007,0,0.0, +maturity_gap,sec_app_chargeoff_within_12_mths,541863,0,0.0, +pd_development,sec_app_chargeoff_within_12_mths,17433,0,0.0, +policy_development,sec_app_chargeoff_within_12_mths,94885,0,0.0, +post_extension,sec_app_chargeoff_within_12_mths,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_chargeoff_within_12_mths,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_chargeoff_within_12_mths,14101,0,0.0, +censored_extension,sec_app_collections_12_mths_ex_med,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_collections_12_mths_ex_med,49007,0,0.0, +maturity_gap,sec_app_collections_12_mths_ex_med,541863,0,0.0, +pd_development,sec_app_collections_12_mths_ex_med,17433,0,0.0, +policy_development,sec_app_collections_12_mths_ex_med,94885,0,0.0, +post_extension,sec_app_collections_12_mths_ex_med,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_collections_12_mths_ex_med,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_collections_12_mths_ex_med,14101,0,0.0, +censored_extension,sec_app_earliest_cr_line,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_earliest_cr_line,49007,0,0.0, +maturity_gap,sec_app_earliest_cr_line,541863,0,0.0, +pd_development,sec_app_earliest_cr_line,17433,0,0.0, +policy_development,sec_app_earliest_cr_line,94885,0,0.0, +post_extension,sec_app_earliest_cr_line,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_earliest_cr_line,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_earliest_cr_line,14101,0,0.0, +censored_extension,sec_app_fico_range_high,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_fico_range_high,49007,0,0.0, +maturity_gap,sec_app_fico_range_high,541863,0,0.0, +pd_development,sec_app_fico_range_high,17433,0,0.0, +policy_development,sec_app_fico_range_high,94885,0,0.0, +post_extension,sec_app_fico_range_high,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_fico_range_high,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_fico_range_high,14101,0,0.0, +censored_extension,sec_app_fico_range_low,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_fico_range_low,49007,0,0.0, +maturity_gap,sec_app_fico_range_low,541863,0,0.0, +pd_development,sec_app_fico_range_low,17433,0,0.0, +policy_development,sec_app_fico_range_low,94885,0,0.0, +post_extension,sec_app_fico_range_low,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_fico_range_low,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_fico_range_low,14101,0,0.0, +censored_extension,sec_app_inq_last_6mths,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_inq_last_6mths,49007,0,0.0, +maturity_gap,sec_app_inq_last_6mths,541863,0,0.0, +pd_development,sec_app_inq_last_6mths,17433,0,0.0, +policy_development,sec_app_inq_last_6mths,94885,0,0.0, +post_extension,sec_app_inq_last_6mths,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_inq_last_6mths,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_inq_last_6mths,14101,0,0.0, +censored_extension,sec_app_mort_acc,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_mort_acc,49007,0,0.0, +maturity_gap,sec_app_mort_acc,541863,0,0.0, +pd_development,sec_app_mort_acc,17433,0,0.0, +policy_development,sec_app_mort_acc,94885,0,0.0, +post_extension,sec_app_mort_acc,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_mort_acc,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_mort_acc,14101,0,0.0, +censored_extension,sec_app_num_rev_accts,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_num_rev_accts,49007,0,0.0, +maturity_gap,sec_app_num_rev_accts,541863,0,0.0, +pd_development,sec_app_num_rev_accts,17433,0,0.0, +policy_development,sec_app_num_rev_accts,94885,0,0.0, +post_extension,sec_app_num_rev_accts,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_num_rev_accts,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_num_rev_accts,14101,0,0.0, +censored_extension,sec_app_open_acc,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_open_acc,49007,0,0.0, +maturity_gap,sec_app_open_acc,541863,0,0.0, +pd_development,sec_app_open_acc,17433,0,0.0, +policy_development,sec_app_open_acc,94885,0,0.0, +post_extension,sec_app_open_acc,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_open_acc,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_open_acc,14101,0,0.0, +censored_extension,sec_app_open_act_il,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,sec_app_open_act_il,49007,0,0.0, +maturity_gap,sec_app_open_act_il,541863,0,0.0, +pd_development,sec_app_open_act_il,17433,0,0.0, +policy_development,sec_app_open_act_il,94885,0,0.0, +post_extension,sec_app_open_act_il,877671,99855,0.1137727007044781,2017-10-01 +primary_oot,sec_app_open_act_il,376890,4298,0.0114038578895699,2017-03-01 +probability_calibration,sec_app_open_act_il,14101,0,0.0, +censored_extension,sec_app_revol_util,88227,7548,0.08555204189193784,2017-07-01 +conformal_fit,sec_app_revol_util,49007,0,0.0, +maturity_gap,sec_app_revol_util,541863,0,0.0, +pd_development,sec_app_revol_util,17433,0,0.0, +policy_development,sec_app_revol_util,94885,0,0.0, +post_extension,sec_app_revol_util,877671,97757,0.1113822833385175,2017-10-01 +primary_oot,sec_app_revol_util,376890,4235,0.011236700363501287,2017-03-01 +probability_calibration,sec_app_revol_util,14101,0,0.0, +censored_extension,sub_grade,88227,88227,1.0,2017-07-01 +conformal_fit,sub_grade,49007,49007,1.0,2012-01-01 +maturity_gap,sub_grade,541863,541863,1.0,2014-01-01 +pd_development,sub_grade,17433,17433,1.0,2007-06-01 +policy_development,sub_grade,94885,94885,1.0,2013-02-01 +post_extension,sub_grade,877671,877671,1.0,2017-10-01 +primary_oot,sub_grade,376890,376890,1.0,2016-04-01 +probability_calibration,sub_grade,14101,14101,1.0,2011-01-01 +censored_extension,tax_liens,88227,88227,1.0,2017-07-01 +conformal_fit,tax_liens,49007,49007,1.0,2012-01-01 +maturity_gap,tax_liens,541863,541863,1.0,2014-01-01 +pd_development,tax_liens,17433,17328,0.9939769402856651,2007-07-01 +policy_development,tax_liens,94885,94885,1.0,2013-02-01 +post_extension,tax_liens,877671,877671,1.0,2017-10-01 +primary_oot,tax_liens,376890,376890,1.0,2016-04-01 +probability_calibration,tax_liens,14101,14101,1.0,2011-01-01 +censored_extension,title,88227,88227,1.0,2017-07-01 +conformal_fit,title,49007,49006,0.9999795947517701,2012-01-01 +maturity_gap,title,541863,534343,0.9861219533350681,2014-01-01 +pd_development,title,17433,17424,0.9994837377387713,2007-06-01 +policy_development,title,94885,94881,0.9999578437055383,2013-02-01 +post_extension,title,877671,877671,1.0,2017-10-01 +primary_oot,title,376890,368388,0.9774416938629308,2016-04-01 +probability_calibration,title,14101,14101,1.0,2011-01-01 +censored_extension,tot_coll_amt,88227,88227,1.0,2017-07-01 +conformal_fit,tot_coll_amt,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,tot_coll_amt,541863,541863,1.0,2014-01-01 +pd_development,tot_coll_amt,17433,0,0.0, +policy_development,tot_coll_amt,94885,94885,1.0,2013-02-01 +post_extension,tot_coll_amt,877671,877671,1.0,2017-10-01 +primary_oot,tot_coll_amt,376890,376890,1.0,2016-04-01 +probability_calibration,tot_coll_amt,14101,0,0.0, +censored_extension,tot_cur_bal,88227,88227,1.0,2017-07-01 +conformal_fit,tot_cur_bal,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,tot_cur_bal,541863,541863,1.0,2014-01-01 +pd_development,tot_cur_bal,17433,0,0.0, +policy_development,tot_cur_bal,94885,94885,1.0,2013-02-01 +post_extension,tot_cur_bal,877671,877671,1.0,2017-10-01 +primary_oot,tot_cur_bal,376890,376890,1.0,2016-04-01 +probability_calibration,tot_cur_bal,14101,0,0.0, +censored_extension,tot_hi_cred_lim,88227,88227,1.0,2017-07-01 +conformal_fit,tot_hi_cred_lim,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,tot_hi_cred_lim,541863,541863,1.0,2014-01-01 +pd_development,tot_hi_cred_lim,17433,0,0.0, +policy_development,tot_hi_cred_lim,94885,94885,1.0,2013-02-01 +post_extension,tot_hi_cred_lim,877671,877671,1.0,2017-10-01 +primary_oot,tot_hi_cred_lim,376890,376890,1.0,2016-04-01 +probability_calibration,tot_hi_cred_lim,14101,0,0.0, +censored_extension,total_acc,88227,88227,1.0,2017-07-01 +conformal_fit,total_acc,49007,49007,1.0,2012-01-01 +maturity_gap,total_acc,541863,541863,1.0,2014-01-01 +pd_development,total_acc,17433,17404,0.9983364882693742,2007-06-01 +policy_development,total_acc,94885,94885,1.0,2013-02-01 +post_extension,total_acc,877671,877671,1.0,2017-10-01 +primary_oot,total_acc,376890,376890,1.0,2016-04-01 +probability_calibration,total_acc,14101,14101,1.0,2011-01-01 +censored_extension,total_bal_ex_mort,88227,88227,1.0,2017-07-01 +conformal_fit,total_bal_ex_mort,49007,42982,0.8770583794151856,2012-03-01 +maturity_gap,total_bal_ex_mort,541863,541863,1.0,2014-01-01 +pd_development,total_bal_ex_mort,17433,0,0.0, +policy_development,total_bal_ex_mort,94885,94885,1.0,2013-02-01 +post_extension,total_bal_ex_mort,877671,877671,1.0,2017-10-01 +primary_oot,total_bal_ex_mort,376890,376890,1.0,2016-04-01 +probability_calibration,total_bal_ex_mort,14101,0,0.0, +censored_extension,total_bal_il,88227,88227,1.0,2017-07-01 +conformal_fit,total_bal_il,49007,0,0.0, +maturity_gap,total_bal_il,541863,111012,0.20487097292119963,2015-12-01 +pd_development,total_bal_il,17433,0,0.0, +policy_development,total_bal_il,94885,0,0.0, +post_extension,total_bal_il,877671,877671,1.0,2017-10-01 +primary_oot,total_bal_il,376890,376890,1.0,2016-04-01 +probability_calibration,total_bal_il,14101,0,0.0, +censored_extension,total_bc_limit,88227,88227,1.0,2017-07-01 +conformal_fit,total_bc_limit,49007,42982,0.8770583794151856,2012-03-01 +maturity_gap,total_bc_limit,541863,541863,1.0,2014-01-01 +pd_development,total_bc_limit,17433,0,0.0, +policy_development,total_bc_limit,94885,94885,1.0,2013-02-01 +post_extension,total_bc_limit,877671,877671,1.0,2017-10-01 +primary_oot,total_bc_limit,376890,376890,1.0,2016-04-01 +probability_calibration,total_bc_limit,14101,0,0.0, +censored_extension,total_cu_tl,88227,88227,1.0,2017-07-01 +conformal_fit,total_cu_tl,49007,0,0.0, +maturity_gap,total_cu_tl,541863,111012,0.20487097292119963,2015-12-01 +pd_development,total_cu_tl,17433,0,0.0, +policy_development,total_cu_tl,94885,0,0.0, +post_extension,total_cu_tl,877671,877671,1.0,2017-10-01 +primary_oot,total_cu_tl,376890,376890,1.0,2016-04-01 +probability_calibration,total_cu_tl,14101,0,0.0, +censored_extension,total_il_high_credit_limit,88227,88227,1.0,2017-07-01 +conformal_fit,total_il_high_credit_limit,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,total_il_high_credit_limit,541863,541863,1.0,2014-01-01 +pd_development,total_il_high_credit_limit,17433,0,0.0, +policy_development,total_il_high_credit_limit,94885,94885,1.0,2013-02-01 +post_extension,total_il_high_credit_limit,877671,877671,1.0,2017-10-01 +primary_oot,total_il_high_credit_limit,376890,376890,1.0,2016-04-01 +probability_calibration,total_il_high_credit_limit,14101,0,0.0, +censored_extension,total_pymnt,88227,88227,1.0,2017-07-01 +conformal_fit,total_pymnt,49007,49007,1.0,2012-01-01 +maturity_gap,total_pymnt,541863,541863,1.0,2014-01-01 +pd_development,total_pymnt,17433,17433,1.0,2007-06-01 +policy_development,total_pymnt,94885,94885,1.0,2013-02-01 +post_extension,total_pymnt,877671,877671,1.0,2017-10-01 +primary_oot,total_pymnt,376890,376890,1.0,2016-04-01 +probability_calibration,total_pymnt,14101,14101,1.0,2011-01-01 +censored_extension,total_pymnt_inv,88227,88227,1.0,2017-07-01 +conformal_fit,total_pymnt_inv,49007,49007,1.0,2012-01-01 +maturity_gap,total_pymnt_inv,541863,541863,1.0,2014-01-01 +pd_development,total_pymnt_inv,17433,17433,1.0,2007-06-01 +policy_development,total_pymnt_inv,94885,94885,1.0,2013-02-01 +post_extension,total_pymnt_inv,877671,877671,1.0,2017-10-01 +primary_oot,total_pymnt_inv,376890,376890,1.0,2016-04-01 +probability_calibration,total_pymnt_inv,14101,14101,1.0,2011-01-01 +censored_extension,total_rec_int,88227,88227,1.0,2017-07-01 +conformal_fit,total_rec_int,49007,49007,1.0,2012-01-01 +maturity_gap,total_rec_int,541863,541863,1.0,2014-01-01 +pd_development,total_rec_int,17433,17433,1.0,2007-06-01 +policy_development,total_rec_int,94885,94885,1.0,2013-02-01 +post_extension,total_rec_int,877671,877671,1.0,2017-10-01 +primary_oot,total_rec_int,376890,376890,1.0,2016-04-01 +probability_calibration,total_rec_int,14101,14101,1.0,2011-01-01 +censored_extension,total_rec_late_fee,88227,88227,1.0,2017-07-01 +conformal_fit,total_rec_late_fee,49007,49007,1.0,2012-01-01 +maturity_gap,total_rec_late_fee,541863,541863,1.0,2014-01-01 +pd_development,total_rec_late_fee,17433,17433,1.0,2007-06-01 +policy_development,total_rec_late_fee,94885,94885,1.0,2013-02-01 +post_extension,total_rec_late_fee,877671,877671,1.0,2017-10-01 +primary_oot,total_rec_late_fee,376890,376890,1.0,2016-04-01 +probability_calibration,total_rec_late_fee,14101,14101,1.0,2011-01-01 +censored_extension,total_rec_prncp,88227,88227,1.0,2017-07-01 +conformal_fit,total_rec_prncp,49007,49007,1.0,2012-01-01 +maturity_gap,total_rec_prncp,541863,541863,1.0,2014-01-01 +pd_development,total_rec_prncp,17433,17433,1.0,2007-06-01 +policy_development,total_rec_prncp,94885,94885,1.0,2013-02-01 +post_extension,total_rec_prncp,877671,877671,1.0,2017-10-01 +primary_oot,total_rec_prncp,376890,376890,1.0,2016-04-01 +probability_calibration,total_rec_prncp,14101,14101,1.0,2011-01-01 +censored_extension,total_rev_hi_lim,88227,88227,1.0,2017-07-01 +conformal_fit,total_rev_hi_lim,49007,26457,0.5398616524170017,2012-08-01 +maturity_gap,total_rev_hi_lim,541863,541863,1.0,2014-01-01 +pd_development,total_rev_hi_lim,17433,0,0.0, +policy_development,total_rev_hi_lim,94885,94885,1.0,2013-02-01 +post_extension,total_rev_hi_lim,877671,877671,1.0,2017-10-01 +primary_oot,total_rev_hi_lim,376890,376890,1.0,2016-04-01 +probability_calibration,total_rev_hi_lim,14101,0,0.0, +censored_extension,url,88227,88227,1.0,2017-07-01 +conformal_fit,url,49007,49007,1.0,2012-01-01 +maturity_gap,url,541863,541863,1.0,2014-01-01 +pd_development,url,17433,17433,1.0,2007-06-01 +policy_development,url,94885,94885,1.0,2013-02-01 +post_extension,url,877671,877671,1.0,2017-10-01 +primary_oot,url,376890,376890,1.0,2016-04-01 +probability_calibration,url,14101,14101,1.0,2011-01-01 +censored_extension,verification_status,88227,88227,1.0,2017-07-01 +conformal_fit,verification_status,49007,49007,1.0,2012-01-01 +maturity_gap,verification_status,541863,541863,1.0,2014-01-01 +pd_development,verification_status,17433,17433,1.0,2007-06-01 +policy_development,verification_status,94885,94885,1.0,2013-02-01 +post_extension,verification_status,877671,877671,1.0,2017-10-01 +primary_oot,verification_status,376890,376890,1.0,2016-04-01 +probability_calibration,verification_status,14101,14101,1.0,2011-01-01 +censored_extension,verification_status_joint,88227,7699,0.08726353610572728,2017-07-01 +conformal_fit,verification_status_joint,49007,0,0.0, +maturity_gap,verification_status_joint,541863,1767,0.0032609718692732294,2015-10-01 +pd_development,verification_status_joint,17433,0,0.0, +policy_development,verification_status_joint,94885,0,0.0, +post_extension,verification_status_joint,877671,91431,0.10417457110921974,2017-10-01 +primary_oot,verification_status_joint,376890,10047,0.02665764546684709,2016-04-01 +probability_calibration,verification_status_joint,14101,0,0.0, +censored_extension,zip_code,88227,88227,1.0,2017-07-01 +conformal_fit,zip_code,49007,49007,1.0,2012-01-01 +maturity_gap,zip_code,541863,541863,1.0,2014-01-01 +pd_development,zip_code,17433,17433,1.0,2007-06-01 +policy_development,zip_code,94885,94885,1.0,2013-02-01 +post_extension,zip_code,877671,877671,1.0,2017-10-01 +primary_oot,zip_code,376890,376889,0.9999973467059354,2016-04-01 +probability_calibration,zip_code,14101,14101,1.0,2011-01-01 diff --git a/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/status_inventory.csv b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/status_inventory.csv new file mode 100644 index 0000000..ff015c5 --- /dev/null +++ b/reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/status_inventory.csv @@ -0,0 +1,12 @@ +loan_status,rows,term36_rows,term60_rows +Fully Paid,1497783,1175607.0,322176.0 +Current,1031016,645536.0,385480.0 +Charged Off,362548,217692.0,144856.0 +Late (31-120 days),16154,9481.0,6673.0 +In Grace Period,10028,5793.0,4235.0 +Late (16-30 days),2719,1650.0,1069.0 +Issued,2062,1573.0,489.0 +Does not meet the credit policy. Status:Fully Paid,1988,1789.0,199.0 +Does not meet the credit policy. Status:Charged Off,761,649.0,112.0 +Default,433,307.0,126.0 +__MISSING__,1,0.0,0.0 diff --git a/reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv b/reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv deleted file mode 100644 index cac2648..0000000 --- a/reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv +++ /dev/null @@ -1,46 +0,0 @@ -policy_id,is_crpto_champion,realized_total_return,weighted_miscoverage_V,gamma_cp,satisficing_pass_rate,mdcp_worst_source_coverage,crc_ltt_risk_control_pass,crc_ltt_operational_pass,crc_ltt_source_hardened_pass,failed_gate_count,failed_gates,diagnostic_selector_rank,decision,claim_boundary -crpto_rt0p155_g0p45_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,166621.64409318674,0.041775,0.157534,1.0,0.8636363636363636,True,True,True,0,none,16,append_source_hardened_gate_pass,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_economic_champion,True,170464.5429284627,0.03645,0.18591,1.0,0.5,True,True,False,1,gate_source_defended_80,25,protect_official_champion_source_caveat,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p45_u0p10_alpha0p01_incumbent_region_rs42,False,169389.5810115051,0.03545,0.177578,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,14,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p45_u0p05_alpha0p01_incumbent_region_rs42,False,169337.1631898248,0.03545,0.179102,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,26,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p45_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,169262.37545367464,0.04425,0.180169,1.0,0.5,True,True,False,1,gate_source_defended_80,38,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p50_u0p05_alpha0p01_conservative_proxy_rs42,False,168801.31532762162,0.038067,0.172713,1.0,0.75,True,True,False,1,gate_source_defended_80,29,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p50_u0p00_alpha0p01_conservative_proxy_rs42,False,168669.5041537059,0.042149,0.173656,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,23,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p45_u0p10_alpha0p01_incumbent_region_rs42,False,168365.23091736945,0.040431,0.17018,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,7,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p45_u0p05_alpha0p01_incumbent_region_rs42,False,168244.12224854622,0.04545,0.171708,1.0,0.75,True,True,False,1,gate_source_defended_80,19,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p50_u0p10_alpha0p01_conservative_proxy_rs42,False,168128.1097153701,0.03545,0.171845,1.0,0.5,True,True,False,1,gate_source_defended_80,17,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p50_u0p10_alpha0p01_incumbent_region_rs42,False,167692.389704731,0.03905,0.157538,1.0,0.5,True,True,False,1,gate_source_defended_80,3,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p45_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,167494.8582103625,0.04425,0.172774,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,34,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p50_u0p10_alpha0p01_incumbent_region_rs42,False,167226.8353415202,0.03545,0.164424,1.0,0.5,True,True,False,1,gate_source_defended_80,22,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p50_u0p05_alpha0p01_incumbent_region_rs42,False,167124.6310620346,0.04545,0.165827,1.0,0.5,True,True,False,1,gate_source_defended_80,24,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p55_u0p10_alpha0p01_incumbent_region_rs42,False,167069.47757462133,0.03905,0.1533,1.0,0.5,True,True,False,1,gate_source_defended_80,2,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p45_u0p10_alpha0p01_incumbent_region_rs42,False,167067.29591823329,0.042975,0.155394,1.0,0.5,True,True,False,1,gate_source_defended_80,12,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p55_u0p05_alpha0p01_incumbent_region_rs42,False,166969.46159398026,0.042975,0.154754,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,8,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p50_u0p05_alpha0p01_incumbent_region_rs42,False,166777.3884430658,0.043789,0.158941,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,31,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p45_u0p05_alpha0p01_conservative_proxy_rs42,False,166574.19464587132,0.03645,0.186841,1.0,0.75,True,True,False,1,gate_source_defended_80,44,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p45_u0p10_alpha0p01_incumbent_region_rs42,False,166559.14303245154,0.04545,0.162785,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,18,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p50_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,166313.57843638843,0.04545,0.16677,1.0,0.6666666666666666,True,True,False,1,gate_source_defended_80,36,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p55_u0p10_alpha0p01_conservative_proxy_rs42,False,166269.82231927008,0.030034,0.159714,1.0,0.5,True,True,False,1,gate_source_defended_80,5,append_operational_pass_source_fragile,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p50_u0p10_alpha0p01_incumbent_region_rs42,False,166244.65010111933,0.042975,0.150881,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,15,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p45_u0p00_alpha0p01_conservative_proxy_rs42,False,166210.47853140024,0.03525,0.187564,0.8,0.6,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,45,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p55_u0p05_alpha0p01_conservative_proxy_rs42,False,166186.4620978511,0.045084,0.161059,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,32,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p45_u0p05_alpha0p01_incumbent_region_rs42,False,165829.33653326883,0.045805,0.164313,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,42,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p45_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,165590.76458837948,0.048175,0.165085,0.8,0.6,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,43,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p55_u0p05_alpha0p01_incumbent_region_rs42,False,165547.8905382124,0.042975,0.147556,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,9,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p50_u0p05_alpha0p01_incumbent_region_rs42,False,165517.85855195278,0.042975,0.152083,0.8,0.0,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,20,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p55_u0p10_alpha0p01_incumbent_region_rs42,False,165504.23210778716,0.040975,0.146459,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,13,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p50_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,165401.62753483138,0.042975,0.159886,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,35,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p175_g0p55_u0p00_alpha0p01_conservative_proxy_rs42,False,165375.90860423315,0.047375,0.161873,0.8,0.75,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,40,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p50_u0p10_alpha0p01_incumbent_region_rs42,False,164689.02196237698,0.042926,0.143365,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,4,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p55_u0p05_alpha0p01_incumbent_region_rs42,False,164273.35080211036,0.041975,0.140719,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,1,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p45_u0p05_alpha0p01_incumbent_region_rs42,False,164022.0917987445,0.042975,0.156642,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,39,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p50_u0p05_alpha0p01_incumbent_region_rs42,False,163947.7110439952,0.042975,0.144086,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,11,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p170_g0p55_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,163897.7680059752,0.042975,0.155326,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,41,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p55_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,163847.98926196393,0.041975,0.141562,0.8,0.8695652173913043,True,False,False,1,gate_satisficing_clean,10,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p50_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,163748.62808782174,0.041975,0.152638,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,37,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p165_g0p55_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,163116.36710524172,0.042975,0.148401,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,27,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p55_u0p05_alpha0p01_incumbent_region_rs42,False,162753.66070085138,0.040015,0.134188,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,21,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p160_g0p55_u0p10_alpha0p01_incumbent_region_rs42,False,162351.17715355012,0.041975,0.139696,0.8,0.6666666666666666,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,6,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p55_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,162326.28515545817,0.041975,0.13543,0.8,0.6,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,28,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p50_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,False,162164.37781018534,0.041975,0.145103,0.8,0.5,True,False,False,2,gate_satisficing_clean;gate_source_defended_80,33,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. -crpto_rt0p155_g0p55_u0p10_alpha0p01_incumbent_region_rs42,False,160978.00154814168,0.038975,0.133529,0.8,0.6666666666666666,True,False,False,3,gate_return_floor_95pct_champion;gate_satisficing_clean;gate_source_defended_80,30,append_risk_pass_operational_gap,Decision-risk governance screen over retained candidates; not a promotion protocol. diff --git a/reports/crpto/extended/crpto_extended_decision_risk_governance_croms_lite_2026-05-18.csv b/reports/crpto/extended/crpto_extended_decision_risk_governance_croms_lite_2026-05-18.csv deleted file mode 100644 index a546035..0000000 --- a/reports/crpto/extended/crpto_extended_decision_risk_governance_croms_lite_2026-05-18.csv +++ /dev/null @@ -1,6 +0,0 @@ -selection_strategy,selected_policy_id,realized_total_return,delta_return_vs_official,mdcp_worst_source_coverage,delta_worst_source_coverage_vs_official,weighted_miscoverage_V,gamma_cp,satisficing_pass_rate,thesis_use,ijds_use,claim_boundary -official_crpto_champion,crpto_economic_champion,170464.5429284627,0.0,0.5,0.0,0.03645,0.18591,1.0,reference champion,official reference,Protected CRPTO champion. -source_defended_return,crpto_rt0p155_g0p45_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,166621.64409318674,-3842.8988352759625,0.8636363636363636,0.3636363636363636,0.041775,0.157534,1.0,source-governance tradeoff,none now,agenda extendida CRPTO/tesis selector audit only; not a promotion protocol. -croms_lite_balanced_score,crpto_rt0p175_g0p50_u0p05_alpha0p01_conservative_proxy_rs42,168801.31532762162,-1663.2276008410845,0.75,0.25,0.038067,0.172713,1.0,CROMS-lite tradeoff evidence,none now,agenda extendida CRPTO/tesis selector audit only; not a promotion protocol. -coverage_only_source,crpto_rt0p160_g0p55_u0p00_alpha0p01_forced_incumbent_neighbors_rs42,163847.98926196393,-6616.553666498774,0.8695652173913043,0.3695652173913043,0.041975,0.141562,0.8,coverage-only negative control,none now,agenda extendida CRPTO/tesis selector audit only; not a promotion protocol. -diagnostic_selector_rank,crpto_rt0p160_g0p55_u0p05_alpha0p01_incumbent_region_rs42,164273.35080211036,-6191.19212635234,0.6666666666666666,0.1666666666666666,0.041975,0.140719,0.8,diagnostic selector anchor,none now,agenda extendida CRPTO/tesis selector audit only; not a promotion protocol. diff --git a/reports/crpto/extended/crpto_extended_evidence_cards_2026-06-06.csv b/reports/crpto/extended/crpto_extended_evidence_cards_2026-06-06.csv deleted file mode 100644 index bfab4c8..0000000 --- a/reports/crpto/extended/crpto_extended_evidence_cards_2026-06-06.csv +++ /dev/null @@ -1,5 +0,0 @@ -card_id,card_name,thesis_sink,ijds_sink,status,key_evidence,claim_boundary,stop_rule,source_table -EC01,PyEPO 1.3.7 / DFL formal suite,DFL comparator appendix; agenda extendida CRPTO/tesis,SPO+ comparator context only; no champion change,append_for_thesis,SPO+ full mean regret 0.122379; RFYL 0.125405; CaVE 0.128109; temporal SPO+ mean regret 0.061835; all smoke and nonnegative-regret gates passed.,"DFL can reduce regret but does not provide conformal coverage, funded-set bound, or replacement for the CRPTO champion.",Do not import heavy solver logs or dependencies; reopen only for a thesis DFL appendix or reviewer-requested comparator table.,reports/crpto/extended/crpto_extended_pyepo_dfl_full_summary_20260528.csv -EC02,FICO proxy versus calibrated champion governance,Model-risk and score-governance appendix,Context only if reviewer asks for traditional-score benchmark,append_for_thesis,"Latest 40pct OOT n=103865: AUC delta +0.10757, Gini/SomersD +0.21514, Brier -0.010815, ECE -0.022137, rank shift >=20pp share 0.486526.","Score-governance evidence only; no legal fair-lending, protected-attribute inference, or champion replacement claim.",Do not rerun unless a reviewer asks for score-governance replication or a new external score dataset appears.,reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv -EC03,IFRS9/SICR prudential absorption,Prudential uncertainty appendix and thesis chapter support,Implication/limitation only; not second IJDS contribution,append_for_thesis,"ECL scenario uplift, conformal ECL range high/point 3.26x, SICR t*=0.30 recall 75.8pct, added ECL USD 56.6M, CIF/KM over-reserve caveat.","IFRS9-inspired diagnostic, not contractual IFRS9 or audited allowance process.","Keep false any contractual IFRS9 claim without DPD panel, cure, EAD paths, recovery/prepayment timing and governed macro scenarios.",reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv -EC04,CRC/LTT and CROMS-lite decision-risk governance,Decision-risk governance appendix; future selector protocol,Reviewer-defense only; current A5/A10 remain official,append_or_future_gate,CRC/LTT lane: 22 policies pass operational gates and 1 passes source-hardening; CROMS-lite tables expose return versus source-defense tradeoffs.,"Gate passes and selector screens are governance evidence over retained artifacts, not end-to-end training or promotion protocol.",Open a direct CRC/LTT loss gate only with declared split and fixed monotone loss before outcomes.,reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv diff --git a/reports/crpto/extended/crpto_extended_future_experiment_gate_register_2026-05-19.csv b/reports/crpto/extended/crpto_extended_future_experiment_gate_register_2026-05-19.csv deleted file mode 100644 index 7c9e4ca..0000000 --- a/reports/crpto/extended/crpto_extended_future_experiment_gate_register_2026-05-19.csv +++ /dev/null @@ -1,5 +0,0 @@ -experiment,decision,current_state,next_gate,max_artifacts,stop_rule,thesis_destination,ijds_destination -FICO proxy vs champion ML,append_executed,Champion beats FICO proxy on latest OOT AUC/Gini/SomersD/Brier/ECE and observable slices,Only rerun if a reviewer asks for score-governance replication or a new external score dataset appears,one table plus one memo,No legal fair-lending claim; no new run unless it changes score-governance text,metrics appendix,light context only -CRC/LTT direct decision-loss gate,ready_but_not_reopened,Lab 4 gate evidence exists; direct prospective calibration over V or violation not yet run,Open only with declared split and fixed monotone loss before outcomes,one gate table plus one decision memo,Park if gate sensitivity depends on split choice or does not change a claim,governance appendix,reviewer-defense only -Non-monotonic CRC composite risk,park,Useful for return plus V plus CVaR plus source/fairness proxy but not implemented,Open only after a concrete composite loss and non-monotonic CRC proof path are selected,one prototype table plus one proof-boundary memo,Park if it cannot beat the current monotone claim boundary cleanly,future-method appendix,none -Online conformal and drift replay,park_with_caveat,Retrospective source/online diagnostics exist but strict source holdouts remain fragile,Open only with prospective split or a temporal replay that can change the online/source caveat,one replay table plus one source-governance memo,No live deployment claim without external/prospective validation,source-governance appendix,limitations/future work diff --git a/reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv b/reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv deleted file mode 100644 index 4bf1e8b..0000000 --- a/reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv +++ /dev/null @@ -1,27 +0,0 @@ -section,subgroup,n,metric,champion_value,fico_value,delta_champion_minus_fico,interpretation,claim_boundary -global_eval_split,all_oot_latest_40pct,103865,auc,0.700477,0.592906,0.10757,higher is better,Score-governance evidence only; no legal fair-lending claim. -global_eval_split,all_oot_latest_40pct,103865,gini_somers_d,0.400953,0.185813,0.21514,higher is better,Score-governance evidence only; no legal fair-lending claim. -global_eval_split,all_oot_latest_40pct,103865,brier,0.14038,0.151195,-0.010815,lower is better,Score-governance evidence only; no legal fair-lending claim. -global_eval_split,all_oot_latest_40pct,103865,ece_10bin,0.028995,0.051132,-0.022137,lower is better,Score-governance evidence only; no legal fair-lending claim. -global_eval_split,all_oot_latest_40pct,103865,decile_band_mae,0.030545,0.051076,-0.020532,lower is better,Score-governance evidence only; no legal fair-lending claim. -rank_difference,all_oot_latest_40pct,103865,share_abs_rank_shift_ge_20pp,0.486526,,,share of loans materially reordered by champion versus FICO proxy,Score-governance evidence only; no legal fair-lending claim. -slice_eval,income_q1,21875,auc,0.691213,0.601882,0.089332,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,income_q1,21875,brier,0.157875,0.166456,-0.008581,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,income_q5,22051,auc,0.6881,0.582085,0.106016,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,income_q5,22051,brier,0.128667,0.141021,-0.012354,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,dti_q5,20746,auc,0.670988,0.595209,0.07578,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,dti_q5,20746,brier,0.169872,0.176652,-0.00678,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,no_mortgage,42980,auc,0.688556,0.579246,0.10931,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,no_mortgage,42980,brier,0.164283,0.175253,-0.01097,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,thin_file_total_acc_le_10,10844,auc,0.692357,0.555161,0.137197,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,thin_file_total_acc_le_10,10844,brier,0.176124,0.190246,-0.014122,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_A,29070,auc,0.625159,0.580788,0.044371,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_A,29070,brier,0.07281,0.088542,-0.015731,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_B,27963,auc,0.622421,0.52886,0.093561,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_B,27963,brier,0.126742,0.136652,-0.00991,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_C,25455,auc,0.613867,0.510169,0.103697,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_C,25455,brier,0.177308,0.1832,-0.005892,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_D,18354,auc,0.624718,0.50827,0.116448,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_D,18354,brier,0.202656,0.210124,-0.007468,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_E,2934,auc,0.63859,0.484914,0.153676,higher is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. -slice_eval,grade_E,2934,brier,0.227683,0.260794,-0.033111,lower is better; subgroup diagnostic only,Score-governance evidence only; no legal fair-lending claim. diff --git a/reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv b/reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv deleted file mode 100644 index cc159a8..0000000 --- a/reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv +++ /dev/null @@ -1,8 +0,0 @@ -anchor_id,prudential_component,thesis_use,anchor_value,decision,claim_boundary -P2A01,ECL scenario propagation,Prudential stress appendix,baseline ECL USD 870.3M; severe ECL USD 1.479B; severe uplift 69.9%,append,"Scenario stress is IFRS9-inspired and diagnostic, not a production allowance process." -P2A02,Conformal ECL range,Uncertainty propagation appendix,baseline point ECL USD 432.1M; high conformal ECL USD 1.408B; baseline high/point ratio 3.26x,append,"Conformal ECL range is an uncertainty diagnostic, not audited regulatory capital." -P2A03,SICR conformal trigger,SICR governance appendix,t*=0.30; F1 0.2515; recall of missed defaults 75.8%; precision 8.9%; added ECL USD 56.6M,append_strong,"Conformal width is a complementary SICR signal, not a replacement for contractual staging policy." -P2A04,Competing risks CIF correction,Lifetime PD method caveat,KM baseline ECL about USD 1.003B versus CIF-adjusted USD 870.3M; reduction about 13.3%; retained competing-risk analysis reports about USD 125.8M over-reserve,append,"Prepayment must be treated as competing risk, but current CIF correction remains a proxy diagnostic." -P2A05,Stage misclassification cost,Threshold governance appendix,minimum observed cost at pd_threshold 0.15; total cost about USD 97.3M; stable band approximately 0.13 to 0.18,append,"Stage thresholds are governance decisions with loss tradeoffs, not fixed accounting constants." -P2A06,Time-series to ECL intervals,Temporal stress context,Jan 2021 point ECL USD 109.6M; adverse 90pct ECL USD 352.2M; range about USD 304.7M,context,"Temporal ECL intervals are planning evidence only, not a deployed monthly forecasting policy." -P2A07,Paper 2 standalone identity,agenda extendida CRPTO/tesis publication architecture,agenda extendida CRPTO/tesis absorbs the prudential value of Paper 2 for future full-paper framing,supersede_near_term,A separate Paper 2 is parked unless new contractual servicing and macro scenario data enable a true IFRS9 paper. diff --git a/reports/crpto/extended/crpto_extended_pyepo_dfl_execution_ledger_20260528.csv b/reports/crpto/extended/crpto_extended_pyepo_dfl_execution_ledger_20260528.csv deleted file mode 100644 index c66e7d3..0000000 --- a/reports/crpto/extended/crpto_extended_pyepo_dfl_execution_ledger_20260528.csv +++ /dev/null @@ -1,4 +0,0 @@ -run_label,mode,runtime_seconds,pyepo_version,gurobipy_version,standard_oracle_backend,cave_oracle_backend,smoke_gate_passed,nonnegative_regret_tolerance,best_method,best_mean_regret,spo_plus_improvement_vs_two_stage_pct,claim_boundary -crpto_spoplus_paired_closeout,paired,143.92372059822083,1.3.7,13.0.2,exact_topk_numpy_lexicographic,gurobi_binary_optDatasetConstrs,True,True,spo_plus,0.1843661805008014,48.51149804244712,Comparator evidence only; no CRPTO champion replacement and no conformal coverage guarantee. -extended_dfl_full_suite,extended_agenda_full,4906.13067483902,1.3.7,13.0.2,exact_topk_numpy_lexicographic,gurobi_binary_optDatasetConstrs,True,True,spo_plus,0.1223785577521161,57.6590045195108,Comparator evidence only; no CRPTO champion replacement and no conformal coverage guarantee. -extended_dfl_temporal_suite,temporal,2377.626832485199,1.3.7,13.0.2,exact_topk_numpy_lexicographic,gurobi_binary_optDatasetConstrs,True,True,spo_plus,0.0618354020273836,61.071811924299126,Comparator evidence only; no CRPTO champion replacement and no conformal coverage guarantee. diff --git a/reports/crpto/extended/crpto_extended_pyepo_dfl_full_summary_20260528.csv b/reports/crpto/extended/crpto_extended_pyepo_dfl_full_summary_20260528.csv deleted file mode 100644 index dd73253..0000000 --- a/reports/crpto/extended/crpto_extended_pyepo_dfl_full_summary_20260528.csv +++ /dev/null @@ -1,8 +0,0 @@ -method,method_display,mean_regret,std_regret,median_regret,n_observations,improvement_vs_two_stage_pct,auditability_score,wilcoxon_vs_two_stage_pvalue,coverage_90,coverage_95,avg_width_90,min_grade_coverage_90,claim_boundary -spo_plus,SPO+,0.1223785577521161,0.0471458928335204,0.1185608617961406,5000,57.6590045195108,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -rfyl,RFYL,0.1254046659517844,0.0483653593583641,0.1201284509152174,5000,56.61202018735212,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -cave,CaVE,0.1281086149948951,0.0488088824117199,0.1229889914393425,5000,55.67649769565711,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -pairwise_ltr,Pairwise LTR,0.2242441021240662,0.0752175754561666,0.2170756086707115,5000,22.41517914805266,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -two_stage,Two-stage Ridge,0.2890308953295593,0.0845953416623312,0.2817149423062801,5000,0.0,1.0,,,,,,DFL regret comparator; no conformal or funded-set guarantee. -pfyl_mul,PFYL-Mul,0.7297965469065893,0.1560128138327949,0.7200621472438797,5000,-152.49776310227386,1.0,1.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -crpto_robust,CRPTO robust,0.910584806515046,0.1940923924645769,0.906894788146019,5000,-215.04756725965296,3.0,1.0,0.929338423586606,0.9623395902033092,0.764155715329014,0.9003509420022164,DFL regret comparator; no conformal or funded-set guarantee. diff --git a/reports/crpto/extended/crpto_extended_pyepo_dfl_temporal_summary_20260528.csv b/reports/crpto/extended/crpto_extended_pyepo_dfl_temporal_summary_20260528.csv deleted file mode 100644 index 954ec7d..0000000 --- a/reports/crpto/extended/crpto_extended_pyepo_dfl_temporal_summary_20260528.csv +++ /dev/null @@ -1,8 +0,0 @@ -method,method_display,mean_regret,std_regret,median_regret,n_observations,improvement_vs_two_stage_pct,auditability_score,wilcoxon_vs_two_stage_pvalue,coverage_90,coverage_95,avg_width_90,min_grade_coverage_90,claim_boundary -spo_plus,SPO+,0.0618354020273836,0.03380288935725,0.0581223666667938,4000,61.071811924299126,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -rfyl,RFYL,0.0714477125165904,0.0389106162370471,0.0653685815632343,4000,55.02042692078969,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -cave,CaVE,0.0722844527933302,0.0401676482170303,0.0663828151300549,4000,54.49366100242058,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -pairwise_ltr,Pairwise LTR,0.101961096906175,0.0501978972064505,0.0945730153471231,4000,35.810868832878825,1.0,0.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -two_stage,Two-stage Ridge,0.1588448003741014,0.0690559441094283,0.1495100259780883,4000,0.0,1.0,,,,,,DFL regret comparator; no conformal or funded-set guarantee. -pfyl_mul,PFYL-Mul,0.3881303746810818,0.1253694356853472,0.3741091026458889,4000,-144.34565835552985,1.0,1.0,,,,,DFL regret comparator; no conformal or funded-set guarantee. -crpto_robust,CRPTO robust,0.5156464260656085,0.1746959563322267,0.509262406732887,4000,-224.6227906767878,3.0,1.0,0.929338423586606,0.9623395902033092,0.764155715329014,0.9003509420022164,DFL regret comparator; no conformal or funded-set guarantee. diff --git a/reports/crpto/extended/crpto_extended_strong_appendix_register_2026-06-06.csv b/reports/crpto/extended/crpto_extended_strong_appendix_register_2026-06-06.csv deleted file mode 100644 index e1e626c..0000000 --- a/reports/crpto/extended/crpto_extended_strong_appendix_register_2026-06-06.csv +++ /dev/null @@ -1,8 +0,0 @@ -appendix_block,status_taxonomy,why_it_belongs,curated_source_table,thesis_sink,ijds_surface,claim_boundary -CRC/LTT gates,append_strong,Decision-risk gates make agenda extendida CRPTO/tesis a governed lab rather than a search diary,reports/crpto/extended/crpto_extended_decision_risk_governance_crc_ltt_2026-05-18.csv,agenda extendida CRPTO/tesis appendix,reviewer-defense only,Gate passes are governance evidence and do not promote a new champion -CROMS-lite selector,append,Selector table shows objective choice and source defense tradeoffs,reports/crpto/extended/crpto_extended_decision_risk_governance_croms_lite_2026-05-18.csv,agenda extendida CRPTO/tesis appendix,appendix/future-work caveat only,Screen over retained finalists not full end-to-end CROMS training -CVaR/OCE challenger,append_strong,Tail-risk challenger explains why the economic champion was retained,covered_by_existing_crpto_appendix_or_literature_audit,agenda extendida CRPTO/tesis appendix,reviewer-defense appendix,Challenger improves tail views but lacks paired wealth dominance -IFRS9/SICR proxy,append_strong,agenda extendida CRPTO/tesis absorbs the useful prudential contribution of Paper 2,reports/crpto/extended/crpto_extended_prudential_ifrs9_absorption_2026-05-18.csv,agenda extendida CRPTO/tesis appendix,applied implication only,IFRS9-inspired ECL/SICR proxy not contractual IFRS9 -Source/fairness governance,append,Observable source and proxy diagnostics clarify MRM boundaries,covered_by_existing_crpto_appendix_or_literature_audit,agenda extendida CRPTO/tesis appendix,limitations or governance context,No fair-lending legal claim without protected attributes or approved proxy protocol -Metrics/FICO vs champion design,bounded_experiment,Calibration-gated score comparison can test Albanesi/Wuthrich implications without changing CRPTO,reports/crpto/extended/crpto_extended_metric_governance_fico_vs_champion_2026-05-19.csv,agenda extendida CRPTO/tesis appendix,context only if stable,Appendix governance experiment only unless it changes a specific score-governance claim -Angelopoulos/CRC future gates,context,Modern CRC/LTT/non-monotonic/online sources sharpen future gates,covered_by_existing_crpto_appendix_or_literature_audit,agenda extendida CRPTO/tesis appendix,theory/future-work language only,No current champion change or deployment claim diff --git a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf b/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf deleted file mode 100644 index 7e7d9de..0000000 Binary files a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png b/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png deleted file mode 100644 index ffe2cb2..0000000 Binary files a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig16_tail_risk_frontier.pdf b/reports/crpto/figures/crpto_fig16_tail_risk_frontier.pdf deleted file mode 100644 index eadee36..0000000 Binary files a/reports/crpto/figures/crpto_fig16_tail_risk_frontier.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig16_tail_risk_frontier.png b/reports/crpto/figures/crpto_fig16_tail_risk_frontier.png deleted file mode 100644 index 8c33ec0..0000000 Binary files a/reports/crpto/figures/crpto_fig16_tail_risk_frontier.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig17_tail_risk_lgd.pdf b/reports/crpto/figures/crpto_fig17_tail_risk_lgd.pdf deleted file mode 100644 index 5d15094..0000000 Binary files a/reports/crpto/figures/crpto_fig17_tail_risk_lgd.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig17_tail_risk_lgd.png b/reports/crpto/figures/crpto_fig17_tail_risk_lgd.png deleted file mode 100644 index 69d2b92..0000000 Binary files a/reports/crpto/figures/crpto_fig17_tail_risk_lgd.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.pdf b/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.pdf deleted file mode 100644 index b97d23e..0000000 Binary files a/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png b/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png deleted file mode 100644 index 6b254cd..0000000 Binary files a/reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig19_online_coverage_aci.pdf b/reports/crpto/figures/crpto_fig19_online_coverage_aci.pdf deleted file mode 100644 index 2da7d2d..0000000 Binary files a/reports/crpto/figures/crpto_fig19_online_coverage_aci.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig19_online_coverage_aci.png b/reports/crpto/figures/crpto_fig19_online_coverage_aci.png deleted file mode 100644 index 4b9b09d..0000000 Binary files a/reports/crpto/figures/crpto_fig19_online_coverage_aci.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf b/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf deleted file mode 100644 index 9162529..0000000 Binary files a/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig20_bound_claim_layers.png b/reports/crpto/figures/crpto_fig20_bound_claim_layers.png deleted file mode 100644 index 6b9ad48..0000000 Binary files a/reports/crpto/figures/crpto_fig20_bound_claim_layers.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig21_end_to_end_arc.pdf b/reports/crpto/figures/crpto_fig21_end_to_end_arc.pdf deleted file mode 100644 index 803b76b..0000000 Binary files a/reports/crpto/figures/crpto_fig21_end_to_end_arc.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig21_end_to_end_arc.png b/reports/crpto/figures/crpto_fig21_end_to_end_arc.png deleted file mode 100644 index 7166b5a..0000000 Binary files a/reports/crpto/figures/crpto_fig21_end_to_end_arc.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig22_external_replication.pdf b/reports/crpto/figures/crpto_fig22_external_replication.pdf deleted file mode 100644 index 85dc8fe..0000000 Binary files a/reports/crpto/figures/crpto_fig22_external_replication.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig22_external_replication.png b/reports/crpto/figures/crpto_fig22_external_replication.png deleted file mode 100644 index 6847fa5..0000000 Binary files a/reports/crpto/figures/crpto_fig22_external_replication.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.pdf b/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.pdf deleted file mode 100644 index 2f739af..0000000 Binary files a/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png b/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png deleted file mode 100644 index 665d390..0000000 Binary files a/reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.pdf b/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.pdf deleted file mode 100644 index 9ff4e1b..0000000 Binary files a/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png b/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png deleted file mode 100644 index 70cb668..0000000 Binary files a/reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf deleted file mode 100644 index 2c0bf99..0000000 Binary files a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf and /dev/null differ diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png deleted file mode 100644 index 966ab67..0000000 Binary files a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png and /dev/null differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.pdf b/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.pdf new file mode 100644 index 0000000..7a99fb7 Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.pdf differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png b/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png new file mode 100644 index 0000000..3c7aa95 Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.pdf b/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.pdf new file mode 100644 index 0000000..5971cec Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.pdf differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png b/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png new file mode 100644 index 0000000..f712a3f Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.pdf b/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.pdf new file mode 100644 index 0000000..9c47815 Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.pdf differ diff --git a/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png b/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png new file mode 100644 index 0000000..83f5c83 Binary files /dev/null and b/reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png differ diff --git a/reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json b/reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json new file mode 100644 index 0000000..bda509a --- /dev/null +++ b/reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json @@ -0,0 +1,4934 @@ +{ + "schema_version": "2026-07-15.4", + "status": "active_ijds_v5_endpoint_reason_audited_paper_facing_evidence", + "source_registry": { + "schema_version": "2026-07-15.4", + "status": "active_ijds_paper_evidence_source_registry", + "sources": [ + "allocation_granularity_freeze", + "allocation_granularity_summary", + "credit_receipt", + "credit_summary", + "endpoint_sensitivity_summary", + "fit_label_completion_freeze", + "fit_label_completion_summary", + "label_lag_sensitivity", + "missingness_receipt", + "missingness_summary", + "raw_data_audit", + "rolling_origin_receipt", + "rolling_origin_summary", + "solver_tie_audit", + "structural_sensitivity_config", + "structural_sensitivity_freeze", + "structural_sensitivity_summary", + "two_ruler_manifest", + "v4_config", + "v4_receipt", + "v4_summary" + ] + }, + "lineages": { + "binary_geometry": { + "outcome_free": { + "run_tag": "ijds-binary-geometry-frontier-v4-2026-07-12-v1", + "protocol_tag": "protocol/ijds-binary-geometry-frontier-v4-2026-07-12-v1", + "protocol_commit": "2f8a7606e4eb65aa3ae3701fb3af8d9a51c953cd", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "freeze_sha256": "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c", + "paper_role": "outcome_free_freeze", + "dvc_tracked": true + }, + "evaluation": { + "run_tag": "ijds-binary-geometry-frontier-v4-2026-07-15-v5", + "protocol_tag": "protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "verified_evaluation", + "dvc_tracked": true + } + }, + "two_ruler": { + "outcome_free": { + "run_tag": "ijds-normalized-objective-frontier-2026-07-13-v1c", + "protocol_tag": "protocol/ijds-normalized-objective-frontier-2026-07-13-v1c", + "protocol_commit": "46f4df915d38eb5a6cc144484c6e6fe56d8ed397", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "freeze_sha256": "7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185", + "paper_role": "outcome_free_freeze", + "dvc_tracked": true + }, + "evaluation": { + "run_tag": "ijds-normalized-objective-frontier-2026-07-15-v5", + "protocol_tag": "protocol/ijds-normalized-objective-frontier-2026-07-15-v5", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "verified_evaluation", + "dvc_tracked": true + } + }, + "credit_controls": { + "outcome_free": { + "run_tag": "ijds-credit-risk-controls-2026-07-13-v1b", + "protocol_tag": "protocol/ijds-credit-risk-controls-2026-07-13-v1b", + "protocol_commit": "1776cbf8b201ae5b92756e5ea397a403d6cc7c9f", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "freeze_sha256": "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35", + "paper_role": "outcome_free_freeze", + "dvc_tracked": true + }, + "evaluation": { + "run_tag": "ijds-credit-risk-controls-2026-07-15-v5", + "protocol_tag": "protocol/ijds-credit-risk-controls-2026-07-15-v5", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "verified_evaluation", + "dvc_tracked": true + } + }, + "diagnostics": { + "raw_data_audit": { + "run_tag": "ijds-raw-data-contract-2026-07-14-v2", + "status": "complete_full_archive_data_contract_audit", + "paper_role": "data_contract_diagnostic", + "dvc_tracked": false + }, + "label_lag_sensitivity": { + "run_tag": "ijds-label-lag-sensitivity-2026-07-14-v1", + "protocol_tag": "protocol/ijds-label-lag-sensitivity-2026-07-14-v1", + "protocol_commit": "688f75dc4f285c75bc499c9e041dd30fb3acd70d", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "fit_label_timing_sensitivity", + "dvc_tracked": false + }, + "solver_tie_audit": { + "run_tag": "ijds-policy-support-tie-audit-2026-07-12-v1", + "protocol_tag": "protocol/ijds-policy-support-tie-audit-2026-07-12-v1", + "protocol_commit": "115eaf1b81ed5f05ffe601e9c20079059c40c200", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "evaluated_cap_numerical_diagnostic", + "dvc_tracked": false + } + } + }, + "sensitivities": { + "endpoint_availability": { + "run_tag": "ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_tag": "protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_commit": "8865f1cfbd387576bdf805f3e52f030261e4b717", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "evaluation_endpoint_timing_sensitivity", + "dvc_tracked": true + }, + "portfolio_structure": { + "run_tag": "ijds-portfolio-structure-sensitivity-2026-07-15-v6", + "protocol_tag": "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6", + "protocol_commit": "490c653a43e2003d83184f47e1277bd2d4390c43", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "complete_budget_concentration_lgd_sensitivity", + "dvc_tracked": true + }, + "rolling_origin": { + "run_tag": "ijds-rolling-origin-2017-2026-07-15-v4", + "protocol_tag": "protocol/ijds-rolling-origin-stability-2026-07-15-v4", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "two_origin_retrospective_recurrence", + "dvc_tracked": true + }, + "missingness_encoding": { + "run_tag": "ijds-missingness-sensitivity-2026-07-15-v3", + "protocol_tag": "protocol/ijds-missingness-sensitivity-2026-07-15-v3", + "protocol_commit": "199afb083da37af6a51d5ba9e3c4d6280b952fe9", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "feature_semantics_preserving_encoding_sensitivity", + "dvc_tracked": true + }, + "fit_label_completion": { + "run_tag": "ijds-fit-label-completion-sensitivity-2026-07-16-v2", + "protocol_tag": "protocol/ijds-fit-label-completion-sensitivity-2026-07-16-v2", + "protocol_commit": "fbcafcf84645024b9753aba2f04a4263b8e76236", + "scientific_uv_lock_sha256": "25cefb168506538c22b86a348c42869ea7fda64338815f2adea3fe7e07608f93", + "paper_role": "declared_fit_label_scenario_sensitivity", + "dvc_tracked": true + }, + "allocation_granularity": { + "run_tag": "ijds-allocation-granularity-sensitivity-2026-07-16-v3", + "protocol_tag": "protocol/ijds-allocation-granularity-sensitivity-2026-07-16-v3", + "protocol_commit": "fb1a7b1837d1f8ab2b81239533f51c996f41671c", + "scientific_uv_lock_sha256": "25cefb168506538c22b86a348c42869ea7fda64338815f2adea3fe7e07608f93", + "paper_role": "continuous_allocation_granularity_diagnostic", + "dvc_tracked": true + } + }, + "replay_dependencies": { + "binary_geometry_v3_reason_reference": { + "run_tag": "ijds-binary-geometry-frontier-v4-2026-07-14-v3", + "protocol_tag": "protocol/ijds-binary-geometry-frontier-v4-2026-07-14-v3", + "protocol_commit": "688f75dc4f285c75bc499c9e041dd30fb3acd70d", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "non_evidence_v5_reason_recovery_reference", + "dvc_tracked": true + }, + "normalized_frontier_v3_reason_reference": { + "run_tag": "ijds-normalized-objective-frontier-2026-07-14-v3", + "protocol_tag": "protocol/ijds-normalized-objective-frontier-2026-07-14-v3", + "protocol_commit": "a1ae516a6c9674686dba245cb275475073b298a0", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "non_evidence_v5_reason_recovery_reference", + "dvc_tracked": true + }, + "credit_controls_v3_reason_reference": { + "run_tag": "ijds-credit-risk-controls-2026-07-14-v3", + "protocol_tag": "protocol/ijds-credit-risk-controls-2026-07-14-v3", + "protocol_commit": "688f75dc4f285c75bc499c9e041dd30fb3acd70d", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "non_evidence_v5_reason_recovery_reference", + "dvc_tracked": true + }, + "portfolio_structure_v5_shards": { + "run_tag": "ijds-portfolio-structure-sensitivity-2026-07-15-v5", + "protocol_tag": "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v5", + "protocol_commit": "481528a6973eded94189435957f9b8064bc4bb06", + "scientific_uv_lock_sha256": "d293ed325a3b25a13000560c8551d34937735ba8c9fe59ffeeed5d64f7120dd2", + "paper_role": "non_evidence_structural_v6_replay_dependency", + "dvc_tracked": true, + "dvc_roots": [ + "data/processed" + ] + } + }, + "run_tag": "ijds-binary-geometry-frontier-v4-2026-07-15-v5", + "protocol_tag": "protocol/ijds-binary-geometry-frontier-v4-2026-07-15-v5", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "claim_boundary": { + "previously_inspected_archive": true, + "confirmatory": false, + "prospective": false, + "causal": false, + "selected_set_validity": false, + "policy_winner": false, + "nested_scopes_are_independent_replications": false + }, + "design": { + "primary_oot_candidates": 376890, + "primary_oot_resolved": 364814, + "primary_oot_unresolved": 12076, + "residual_windows": 8, + "learners": 5, + "v4_detailed_coverage_learners": 2, + "credit_control_learners": 5, + "portfolio_learners": 1, + "taxonomy_diagnostics": [ + 1, + 2, + 5, + 10 + ], + "policies": 9, + "v4_policies_are_supporting_not_closed_family": true, + "oot_months": 15, + "development_months": 11, + "two_ruler_gamma_grid": [ + 0.0, + 0.25, + 0.5, + 0.75, + 1.0 + ], + "two_ruler_primary_contrast": "gamma_1_minus_gamma_0", + "two_ruler_interior_coordinates": [ + 0.25, + 0.5, + 0.75 + ], + "two_ruler_tracks": 6, + "frontier_caps": 3067, + "development_support_lower": 0.0555726278946077, + "development_support_upper": 0.09999720664228194, + "evaluation_endpoint": "terminal_default_reconstructed_as_observable_by_2020-09-30", + "archive_is_verified_point_in_time_snapshot": false + }, + "coverage": { + "catboost_all_eight_upper_below_nominal": true, + "logistic_all_eight_upper_below_nominal": true, + "catboost_bound_min": 0.8424845445620738, + "catboost_bound_max": 0.8825970442304121, + "logistic_bound_min": 0.8500305128817427, + "logistic_bound_max": 0.8962217092520364, + "rows": [ + { + "learner": "catboost_platt", + "window_id": "w01_2012m01_m06", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8770469335058413, + "coverage_lower": 0.8547135769057285, + "coverage_upper": 0.8805221682719095, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 14948, + "fit_prevalence": 0.13674070109713674, + "fit_residual_quantile": 0.8219519891493827, + "fit_score_min": 0.009701465766812493, + "fit_score_max": 0.5281745670852075, + "scores_below_fit_range": 139, + "scores_above_fit_range": 3, + "rows": 376890, + "mean_width": 0.6649918872956118, + "lower_positive_share": 0.01250497492637109, + "upper_saturated_share": 0.18708376449361883, + "set_empty_count": 4713, + "set_empty_share": 0.01250497492637109, + "set_zero_only_count": 301667, + "set_zero_only_share": 0.8004112605800101, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 70510, + "set_both_share": 0.18708376449361883, + "width_q00": 0.054537479077481146, + "width_q10": 0.0896353066409487, + "width_q25": 0.1426944072201049, + "width_q50": 0.9785780281145038, + "width_q75": 0.9953786894773093, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9003880117741504 + }, + { + "learner": "catboost_platt", + "window_id": "w02_2012m02_m07", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.879165821487114, + "coverage_lower": 0.8570245960359787, + "coverage_upper": 0.8825970442304121, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 16674, + "fit_prevalence": 0.14081804006237256, + "fit_residual_quantile": 0.82766418729986, + "fit_score_min": 0.009701465766812493, + "fit_score_max": 0.4831174891615209, + "scores_below_fit_range": 139, + "scores_above_fit_range": 32, + "rows": 376890, + "mean_width": 0.665340945235394, + "lower_positive_share": 0.012327204224044151, + "upper_saturated_share": 0.19626946854519886, + "set_empty_count": 4646, + "set_empty_share": 0.012327204224044151, + "set_zero_only_count": 298272, + "set_zero_only_share": 0.791403327230757, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 73972, + "set_both_share": 0.19626946854519886, + "width_q00": 0.0544282437770001, + "width_q10": 0.08952607134046764, + "width_q25": 0.14283617519766612, + "width_q50": 0.9793001695195559, + "width_q75": 0.9960733268504931, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9004438047259206 + }, + { + "learner": "catboost_platt", + "window_id": "w03_2012m03_m08", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8777349553471084, + "coverage_lower": 0.8556528430045902, + "coverage_upper": 0.8811430390830216, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 19049, + "fit_prevalence": 0.14116226573573415, + "fit_residual_quantile": 0.8266480025829714, + "fit_score_min": 0.009701465766812493, + "fit_score_max": 0.4831174891615209, + "scores_below_fit_range": 139, + "scores_above_fit_range": 32, + "rows": 376890, + "mean_width": 0.6653200651843035, + "lower_positive_share": 0.014086338188861472, + "upper_saturated_share": 0.19673644830056516, + "set_empty_count": 5309, + "set_empty_share": 0.014086338188861472, + "set_zero_only_count": 297433, + "set_zero_only_share": 0.7891772135105733, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 74148, + "set_both_share": 0.19673644830056516, + "width_q00": 0.054239731200646374, + "width_q10": 0.08933755876411392, + "width_q25": 0.14275810252805138, + "width_q50": 0.9793603622579774, + "width_q75": 0.9961042627119951, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9003622237387789 + }, + { + "learner": "catboost_platt", + "window_id": "w04_2012m04_m09", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8748732230670972, + "coverage_lower": 0.8525697153015469, + "coverage_upper": 0.8783411605508239, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 21767, + "fit_prevalence": 0.14126889327881656, + "fit_residual_quantile": 0.8255389571463173, + "fit_score_min": 0.009701465766812493, + "fit_score_max": 0.4831174891615209, + "scores_below_fit_range": 139, + "scores_above_fit_range": 32, + "rows": 376890, + "mean_width": 0.665031208384886, + "lower_positive_share": 0.015378492398312505, + "upper_saturated_share": 0.18653453262225053, + "set_empty_count": 5796, + "set_empty_share": 0.015378492398312505, + "set_zero_only_count": 300791, + "set_zero_only_share": 0.7980869749794369, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 70303, + "set_both_share": 0.18653453262225053, + "width_q00": 0.0539599419893023, + "width_q10": 0.08905776955276984, + "width_q25": 0.14283617519766612, + "width_q50": 0.9786919648415489, + "width_q75": 0.9953512704484286, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002618642899802 + }, + { + "learner": "catboost_platt", + "window_id": "w05_2012m05_m10", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.87251311627295, + "coverage_lower": 0.8499323410013532, + "coverage_upper": 0.8760327947146382, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 24270, + "fit_prevalence": 0.13976102183765965, + "fit_residual_quantile": 0.8238022572208143, + "fit_score_min": 0.009371217051990668, + "fit_score_max": 0.4831174891615209, + "scores_below_fit_range": 119, + "scores_above_fit_range": 32, + "rows": 376890, + "mean_width": 0.6647076525088493, + "lower_positive_share": 0.01602058956194115, + "upper_saturated_share": 0.17684470269840005, + "set_empty_count": 6038, + "set_empty_share": 0.01602058956194115, + "set_zero_only_count": 304201, + "set_zero_only_share": 0.8071347077396588, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 66651, + "set_both_share": 0.17684470269840005, + "width_q00": 0.053862759496348916, + "width_q10": 0.08896058705981646, + "width_q25": 0.14283524655328017, + "width_q50": 0.9779902667559446, + "width_q75": 0.9946330525536466, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002472187886279 + }, + { + "learner": "catboost_platt", + "window_id": "w06_2012m06_m11", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8721129123334083, + "coverage_lower": 0.8494255618350182, + "coverage_upper": 0.8756692934277905, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 26685, + "fit_prevalence": 0.1380925613640622, + "fit_residual_quantile": 0.8228775147793896, + "fit_score_min": 0.009275781579087032, + "fit_score_max": 0.4831174891615209, + "scores_below_fit_range": 113, + "scores_above_fit_range": 32, + "rows": 376890, + "mean_width": 0.6645688455708323, + "lower_positive_share": 0.015450131338056197, + "upper_saturated_share": 0.17227838361325587, + "set_empty_count": 5823, + "set_empty_share": 0.015450131338056197, + "set_zero_only_count": 306137, + "set_zero_only_share": 0.812271485048688, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 64930, + "set_both_share": 0.17227838361325587, + "width_q00": 0.05389117253953069, + "width_q10": 0.08898900010299823, + "width_q25": 0.14289258993779208, + "width_q50": 0.9776415565716892, + "width_q75": 0.9942949912817858, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002435825370058 + }, + { + "learner": "catboost_platt", + "window_id": "w07_2012m07_m12", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8694814343747773, + "coverage_lower": 0.8465175515402372, + "coverage_upper": 0.8730929448910822, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 28411, + "fit_prevalence": 0.13248389708211608, + "fit_residual_quantile": 0.8164243127235671, + "fit_score_min": 0.009275781579087032, + "fit_score_max": 0.49345656902413654, + "scores_below_fit_range": 113, + "scores_above_fit_range": 18, + "rows": 376890, + "mean_width": 0.6641544351090525, + "lower_positive_share": 0.01606569556103903, + "upper_saturated_share": 0.1612910928918252, + "set_empty_count": 6055, + "set_empty_share": 0.01606569556103903, + "set_zero_only_count": 310046, + "set_zero_only_share": 0.8226432115471358, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 60789, + "set_both_share": 0.1612910928918252, + "width_q00": 0.05390479242464117, + "width_q10": 0.08900261998810871, + "width_q25": 0.142788277391194, + "width_q50": 0.9767883025628927, + "width_q75": 0.9934566726155654, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002147055717856 + }, + { + "learner": "catboost_platt", + "window_id": "w08_2012m08_2013m01", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8658549288130389, + "coverage_lower": 0.8424845445620738, + "coverage_upper": 0.8695640637851894, + "score_min": 0.005424291130823528, + "score_max": 0.5544847575803251, + "fit_rows": 30129, + "fit_prevalence": 0.12761790965514952, + "fit_residual_quantile": 0.8091359890849514, + "fit_score_min": 0.009275781579087032, + "fit_score_max": 0.49345656902413654, + "scores_below_fit_range": 113, + "scores_above_fit_range": 18, + "rows": 376890, + "mean_width": 0.5005686881638921, + "lower_positive_share": 0.016068348855103613, + "upper_saturated_share": 0.14419591923372868, + "set_empty_count": 6056, + "set_empty_share": 0.016068348855103613, + "set_zero_only_count": 316488, + "set_zero_only_share": 0.8397357319111677, + "set_one_only_count": 0, + "set_one_only_share": 0.0, + "set_both_count": 54346, + "set_both_share": 0.14419591923372868, + "width_q00": 0.05408270522336448, + "width_q10": 0.08918053278683202, + "width_q25": 0.14269205142933994, + "width_q50": 0.2108606159144652, + "width_q75": 0.9855777292913334, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002290152344917 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w01_2012m01_m06", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8930578322103867, + "coverage_lower": 0.8718193637400833, + "coverage_upper": 0.8962217092520364, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 14948, + "fit_prevalence": 0.13674070109713674, + "fit_residual_quantile": 0.8132704993326145, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.4387428023626731, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1395, + "rows": 376890, + "mean_width": 0.7502982548753632, + "lower_positive_share": 0.00845074159569105, + "upper_saturated_share": 0.23931120486083474, + "set_empty_count": 2900, + "set_empty_share": 0.007694552787285415, + "set_zero_only_count": 283796, + "set_zero_only_share": 0.7529942423518798, + "set_one_only_count": 285, + "set_one_only_share": 0.0007561888084056356, + "set_both_count": 89909, + "set_both_share": 0.2385550160524291, + "width_q00": 0.05584550688919372, + "width_q10": 0.10655796562651992, + "width_q25": 0.15853748633016512, + "width_q50": 0.9834217079741655, + "width_q75": 0.9992278304184096, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9004549103559004 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w02_2012m02_m07", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8921011803275094, + "coverage_lower": 0.8708509114065112, + "coverage_upper": 0.8952771365650455, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 16674, + "fit_prevalence": 0.14081804006237256, + "fit_residual_quantile": 0.813209200317482, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.4387428023626731, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1395, + "rows": 376890, + "mean_width": 0.7500855689952323, + "lower_positive_share": 0.008891188410411525, + "upper_saturated_share": 0.2384409244076521, + "set_empty_count": 3062, + "set_empty_share": 0.008124386425747565, + "set_zero_only_count": 283962, + "set_zero_only_share": 0.7534346891666003, + "set_one_only_count": 289, + "set_one_only_share": 0.0007668019846639603, + "set_both_count": 89577, + "set_both_share": 0.23767412242298813, + "width_q00": 0.055803534960441346, + "width_q10": 0.10651599369776756, + "width_q25": 0.1585032799436104, + "width_q50": 0.983287132927271, + "width_q75": 0.9991742093226285, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9003838311143098 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w03_2012m03_m08", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8918188446715313, + "coverage_lower": 0.8706413011754093, + "coverage_upper": 0.8949693544535541, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 19049, + "fit_prevalence": 0.14116226573573415, + "fit_residual_quantile": 0.8106670055920027, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.4387428023626731, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1395, + "rows": 376890, + "mean_width": 0.750093920569841, + "lower_positive_share": 0.00950675263339436, + "upper_saturated_share": 0.23981267743904056, + "set_empty_count": 3294, + "set_empty_share": 0.008739950648730399, + "set_zero_only_count": 283213, + "set_zero_only_share": 0.751447371912229, + "set_one_only_count": 289, + "set_one_only_share": 0.0007668019846639603, + "set_both_count": 90094, + "set_both_share": 0.2390458754543766, + "width_q00": 0.055762309681025385, + "width_q10": 0.1064747684183516, + "width_q25": 0.15843614395650368, + "width_q50": 0.9834163815599517, + "width_q75": 0.9992604741626012, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9003622237387789 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w04_2012m04_m09", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8881567045124364, + "coverage_lower": 0.8666640133726021, + "coverage_upper": 0.8913741409960466, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 21767, + "fit_prevalence": 0.14126889327881656, + "fit_residual_quantile": 0.8077035644116417, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.4387428023626731, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1395, + "rows": 376890, + "mean_width": 0.7496760765048108, + "lower_positive_share": 0.011305686009180398, + "upper_saturated_share": 0.22668683170155748, + "set_empty_count": 3972, + "set_empty_share": 0.010538884024516438, + "set_zero_only_count": 287482, + "set_zero_only_share": 0.7627742842739261, + "set_one_only_count": 289, + "set_one_only_share": 0.0007668019846639603, + "set_both_count": 85147, + "set_both_share": 0.2259200297168935, + "width_q00": 0.05568070201998956, + "width_q10": 0.10639316075731577, + "width_q25": 0.15823378694722007, + "width_q50": 0.9825390355131967, + "width_q75": 0.9983719080437077, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9003078053934855 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w05_2012m05_m10", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8856924350490936, + "coverage_lower": 0.8636631377855608, + "coverage_upper": 0.8890127092785693, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 24270, + "fit_prevalence": 0.13976102183765965, + "fit_residual_quantile": 0.8061316208044618, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.4387428023626731, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1395, + "rows": 376890, + "mean_width": 0.7491068029128565, + "lower_positive_share": 0.010719308020907958, + "upper_saturated_share": 0.20815887924858711, + "set_empty_count": 3751, + "set_empty_share": 0.009952506036243997, + "set_zero_only_count": 294686, + "set_zero_only_share": 0.7818886147151689, + "set_one_only_count": 289, + "set_one_only_share": 0.0007668019846639603, + "set_both_count": 78164, + "set_both_share": 0.20739207726392317, + "width_q00": 0.05578939611884056, + "width_q10": 0.10650185485616677, + "width_q25": 0.158237310021505, + "width_q50": 0.9813043415737832, + "width_q75": 0.9971105921513819, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.900288421920066 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w06_2012m06_m11", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8830445103532211, + "coverage_lower": 0.8605614370240654, + "coverage_upper": 0.886468200270636, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 26685, + "fit_prevalence": 0.1380925613640622, + "fit_residual_quantile": 0.8027175079686439, + "fit_score_min": 0.002386024381188275, + "fit_score_max": 0.43322525310526877, + "scores_below_fit_range": 2268, + "scores_above_fit_range": 1476, + "rows": 376890, + "mean_width": 0.7482864173070811, + "lower_positive_share": 0.009997612035341876, + "upper_saturated_share": 0.18963092679561674, + "set_empty_count": 3476, + "set_empty_share": 0.009222850168484172, + "set_zero_only_count": 301944, + "set_zero_only_share": 0.8011462230358991, + "set_one_only_count": 292, + "set_one_only_share": 0.0007747618668577038, + "set_both_count": 71178, + "set_both_share": 0.18885616492875906, + "width_q00": 0.05580733474546511, + "width_q10": 0.10651979348279132, + "width_q25": 0.1583289847525134, + "width_q50": 0.979905548780851, + "width_q75": 0.9958109701446924, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002435825370058 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w07_2012m07_m12", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8774279495852682, + "coverage_lower": 0.8545172331449494, + "coverage_upper": 0.8809758815569529, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 28411, + "fit_prevalence": 0.13248389708211608, + "fit_residual_quantile": 0.7953701100393342, + "fit_score_min": 0.00026321481438389247, + "fit_score_max": 0.9937191448732018, + "scores_below_fit_range": 929, + "scores_above_fit_range": 87, + "rows": 376890, + "mean_width": 0.6012897474623677, + "lower_positive_share": 0.011770012470482103, + "upper_saturated_share": 0.16899360556130436, + "set_empty_count": 4142, + "set_empty_share": 0.010989944015495237, + "set_zero_only_count": 309056, + "set_zero_only_share": 0.8200164504232004, + "set_one_only_count": 294, + "set_one_only_share": 0.0007800684549868662, + "set_both_count": 63398, + "set_both_share": 0.1682135371063175, + "width_q00": 0.05571640945671315, + "width_q10": 0.10642886819403935, + "width_q25": 0.15824444009055916, + "width_q50": 0.955000941141985, + "width_q75": 0.9894183165928746, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002147055717856 + }, + { + "learner": "numeric_logistic_platt", + "window_id": "w08_2012m08_2013m01", + "taxonomy_groups": 5, + "role": "primary_oot", + "conformal_group": -1, + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8732340315886999, + "coverage_lower": 0.8500305128817427, + "coverage_upper": 0.8768632757568522, + "score_min": 1.9895529665330944e-46, + "score_max": 1.0, + "fit_rows": 30129, + "fit_prevalence": 0.12761790965514952, + "fit_residual_quantile": 0.7860933691064516, + "fit_score_min": 1.0309719899645506e-06, + "fit_score_max": 0.9937191448732018, + "scores_below_fit_range": 153, + "scores_above_fit_range": 87, + "rows": 376890, + "mean_width": 0.6003912914346058, + "lower_positive_share": 0.013468120671814058, + "upper_saturated_share": 0.1554432327734883, + "set_empty_count": 4779, + "set_empty_share": 0.012680092334633448, + "set_zero_only_count": 313526, + "set_zero_only_share": 0.8318766748918782, + "set_one_only_count": 297, + "set_one_only_share": 0.0007880283371806097, + "set_both_count": 58288, + "set_both_share": 0.15465520443630767, + "width_q00": 0.05573682003560672, + "width_q10": 0.10644927877293292, + "width_q25": 0.15820549584760407, + "width_q50": 0.9517732360026724, + "width_q75": 0.9874696670924321, + "width_q90": 1.0, + "width_q100": 1.0, + "fit_coverage": 0.9002622058481862 + } + ] + }, + "evaluation_endpoint": { + "mode": "conservative_terminal_status_reconstruction", + "cutoff": "2020-09-30", + "terminal_status_source": "distributed_archive_final_status", + "archive_is_verified_point_in_time_snapshot": false, + "charged_off_reporting_lag_months": 6, + "role": "reconstructed_endpoint_cutoff_not_verified_archive_snapshot", + "terminal_statuses_after_cutoff_reclassified_unresolved": true, + "primary_oot_candidates": 376890, + "primary_oot_resolved": 364814, + "primary_oot_unresolved": 12076, + "reason_census": [ + { + "role": "primary_oot", + "snapshot_resolution": "charged_off_by_reconstructed_cutoff", + "candidate_rows": 56972, + "resolved_rows": 56972, + "unresolved_rows": 0 + }, + { + "role": "primary_oot", + "snapshot_resolution": "fully_paid_by_reconstructed_cutoff", + "candidate_rows": 307842, + "resolved_rows": 307842, + "unresolved_rows": 0 + }, + { + "role": "primary_oot", + "snapshot_resolution": "nonterminal_or_unresolved_status", + "candidate_rows": 11551, + "resolved_rows": 0, + "unresolved_rows": 11551 + }, + { + "role": "primary_oot", + "snapshot_resolution": "terminal_after_reconstructed_cutoff", + "candidate_rows": 47, + "resolved_rows": 0, + "unresolved_rows": 47 + }, + { + "role": "primary_oot", + "snapshot_resolution": "terminal_availability_date_missing", + "candidate_rows": 478, + "resolved_rows": 0, + "unresolved_rows": 478 + } + ], + "reason_census_partitions_primary_candidates": true, + "primary_oot_nonterminal_or_unresolved_status": 11551, + "primary_oot_terminal_after_cutoff": 47, + "primary_oot_terminal_availability_date_missing": 478, + "missingness_mechanism_identified": false, + "operational_event_dates_identified": false, + "recovery_audit": { + "float_atol": 5e-14, + "float_rtol": 5e-14, + "maximum_absolute_drift": 3.3306690738754696e-16, + "maximum_relative_drift": 4.2430279119375826e-16 + }, + "last_payment_date_max": "2020-12-01", + "last_credit_pull_date_max": "2020-10-01", + "last_payment_rows_after_cutoff": 36485, + "last_credit_pull_rows_after_cutoff": 40214 + }, + "sensitivity": { + "evaluation_endpoint_availability": { + "scope": "complete_nonselective_retrospective_lag_grid", + "run_tag": "ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_tag": "protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_commit": "8865f1cfbd387576bdf805f3e52f030261e4b717", + "charged_off_lags_months": [ + 0, + 3, + 6, + 8, + 12 + ], + "endpoint_or_result_selected": false, + "allocation_refit": false, + "six_month_endpoint_reconciles_to_active_evaluation": true, + "reconciliation": { + "charged_off_lag_months": 6, + "coverage_cells_exact": 120, + "two_ruler_contrasts_exact": 48, + "exact_support_envelopes_exact": 648, + "byte_value_equal_after_lag_column_removed": true + }, + "fit_label_lag_crossed_factorially": false, + "estimand_boundary": "This family changes evaluation-outcome availability while holding scores, fitted residual recipes, supports, and allocations fixed. The separate label-lag family changes conformal-fit labels and was not crossed factorially with endpoint availability.", + "rows": [ + { + "charged_off_lag_months": 0, + "primary_candidates": 376890, + "primary_resolved": 364861, + "primary_unresolved": 12029, + "primary_defaults": 57019, + "coverage_cells": 40, + "coverage_upper_below_0_90_cells": 40, + "coverage_upper_at_or_above_0_90_cells": 0, + "coverage_upper_max": 0.8976412215765873, + "coverage_upper_max_learner": "woe_scorecard_borrower_platt", + "coverage_upper_max_window": "w02_2012m02_m07", + "two_ruler_payoff_gamma_1_lower_cells": 32, + "two_ruler_payoff_gamma_1_higher_cells": 0, + "two_ruler_payoff_crosses_zero_cells": 16, + "two_ruler_payoff_exact_zero_cells": 0, + "two_ruler_default_gamma_1_lower_cells": 0, + "two_ruler_default_gamma_1_higher_cells": 33, + "two_ruler_default_crosses_zero_cells": 15, + "two_ruler_default_exact_zero_cells": 0, + "two_ruler_miscoverage_gamma_1_lower_cells": 0, + "two_ruler_miscoverage_gamma_1_higher_cells": 40, + "two_ruler_miscoverage_crosses_zero_cells": 8, + "two_ruler_miscoverage_exact_zero_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells": 6, + "development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells": 66, + "broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "development_admissible_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_terminal_default_crosses_zero_cells": 72, + "broad_stress_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_terminal_default_crosses_zero_cells": 72, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells": 27, + "development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells": 45, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells": 72 + }, + { + "charged_off_lag_months": 3, + "primary_candidates": 376890, + "primary_resolved": 364861, + "primary_unresolved": 12029, + "primary_defaults": 57019, + "coverage_cells": 40, + "coverage_upper_below_0_90_cells": 40, + "coverage_upper_at_or_above_0_90_cells": 0, + "coverage_upper_max": 0.8976412215765873, + "coverage_upper_max_learner": "woe_scorecard_borrower_platt", + "coverage_upper_max_window": "w02_2012m02_m07", + "two_ruler_payoff_gamma_1_lower_cells": 32, + "two_ruler_payoff_gamma_1_higher_cells": 0, + "two_ruler_payoff_crosses_zero_cells": 16, + "two_ruler_payoff_exact_zero_cells": 0, + "two_ruler_default_gamma_1_lower_cells": 0, + "two_ruler_default_gamma_1_higher_cells": 33, + "two_ruler_default_crosses_zero_cells": 15, + "two_ruler_default_exact_zero_cells": 0, + "two_ruler_miscoverage_gamma_1_lower_cells": 0, + "two_ruler_miscoverage_gamma_1_higher_cells": 40, + "two_ruler_miscoverage_crosses_zero_cells": 8, + "two_ruler_miscoverage_exact_zero_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells": 6, + "development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells": 66, + "broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "development_admissible_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_terminal_default_crosses_zero_cells": 72, + "broad_stress_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_terminal_default_crosses_zero_cells": 72, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells": 27, + "development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells": 45, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells": 72 + }, + { + "charged_off_lag_months": 6, + "primary_candidates": 376890, + "primary_resolved": 364814, + "primary_unresolved": 12076, + "primary_defaults": 56972, + "coverage_cells": 40, + "coverage_upper_below_0_90_cells": 40, + "coverage_upper_at_or_above_0_90_cells": 0, + "coverage_upper_max": 0.8977261269866539, + "coverage_upper_max_learner": "woe_scorecard_borrower_platt", + "coverage_upper_max_window": "w02_2012m02_m07", + "two_ruler_payoff_gamma_1_lower_cells": 32, + "two_ruler_payoff_gamma_1_higher_cells": 0, + "two_ruler_payoff_crosses_zero_cells": 16, + "two_ruler_payoff_exact_zero_cells": 0, + "two_ruler_default_gamma_1_lower_cells": 0, + "two_ruler_default_gamma_1_higher_cells": 33, + "two_ruler_default_crosses_zero_cells": 15, + "two_ruler_default_exact_zero_cells": 0, + "two_ruler_miscoverage_gamma_1_lower_cells": 0, + "two_ruler_miscoverage_gamma_1_higher_cells": 40, + "two_ruler_miscoverage_crosses_zero_cells": 8, + "two_ruler_miscoverage_exact_zero_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells": 6, + "development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells": 66, + "broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "development_admissible_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_terminal_default_crosses_zero_cells": 72, + "broad_stress_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_terminal_default_crosses_zero_cells": 72, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells": 27, + "development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells": 45, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells": 72 + }, + { + "charged_off_lag_months": 8, + "primary_candidates": 376890, + "primary_resolved": 364570, + "primary_unresolved": 12320, + "primary_defaults": 56728, + "coverage_cells": 40, + "coverage_upper_below_0_90_cells": 40, + "coverage_upper_at_or_above_0_90_cells": 0, + "coverage_upper_max": 0.8981506540369869, + "coverage_upper_max_learner": "woe_scorecard_borrower_platt", + "coverage_upper_max_window": "w02_2012m02_m07", + "two_ruler_payoff_gamma_1_lower_cells": 32, + "two_ruler_payoff_gamma_1_higher_cells": 0, + "two_ruler_payoff_crosses_zero_cells": 16, + "two_ruler_payoff_exact_zero_cells": 0, + "two_ruler_default_gamma_1_lower_cells": 0, + "two_ruler_default_gamma_1_higher_cells": 33, + "two_ruler_default_crosses_zero_cells": 15, + "two_ruler_default_exact_zero_cells": 0, + "two_ruler_miscoverage_gamma_1_lower_cells": 0, + "two_ruler_miscoverage_gamma_1_higher_cells": 40, + "two_ruler_miscoverage_crosses_zero_cells": 8, + "two_ruler_miscoverage_exact_zero_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells": 6, + "development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells": 66, + "broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "development_admissible_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_terminal_default_crosses_zero_cells": 72, + "broad_stress_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_terminal_default_crosses_zero_cells": 72, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells": 27, + "development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells": 45, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells": 72 + }, + { + "charged_off_lag_months": 12, + "primary_candidates": 376890, + "primary_resolved": 363288, + "primary_unresolved": 13602, + "primary_defaults": 55446, + "coverage_cells": 40, + "coverage_upper_below_0_90_cells": 39, + "coverage_upper_at_or_above_0_90_cells": 1, + "coverage_upper_max": 0.9004112605800101, + "coverage_upper_max_learner": "woe_scorecard_borrower_platt", + "coverage_upper_max_window": "w02_2012m02_m07", + "two_ruler_payoff_gamma_1_lower_cells": 31, + "two_ruler_payoff_gamma_1_higher_cells": 0, + "two_ruler_payoff_crosses_zero_cells": 17, + "two_ruler_payoff_exact_zero_cells": 0, + "two_ruler_default_gamma_1_lower_cells": 0, + "two_ruler_default_gamma_1_higher_cells": 32, + "two_ruler_default_crosses_zero_cells": 16, + "two_ruler_default_exact_zero_cells": 0, + "two_ruler_miscoverage_gamma_1_lower_cells": 0, + "two_ruler_miscoverage_gamma_1_higher_cells": 40, + "two_ruler_miscoverage_crosses_zero_cells": 8, + "two_ruler_miscoverage_exact_zero_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells": 72, + "development_admissible_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "development_admissible_exact_frontier_terminal_default_crosses_zero_cells": 72, + "broad_stress_exact_frontier_terminal_default_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_terminal_default_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_terminal_default_crosses_zero_cells": 72, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells": 26, + "development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells": 46, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells": 0, + "broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells": 72 + } + ] + }, + "portfolio_structure": { + "scope": "complete_nonselective_budget_by_purpose_cap_by_lgd_grid", + "run_tag": "ijds-portfolio-structure-sensitivity-2026-07-15-v6", + "protocol_tag": "protocol/ijds-portfolio-structure-sensitivity-2026-07-15-v6", + "protocol_commit": "490c653a43e2003d83184f47e1277bd2d4390c43", + "scenario_or_result_selected": false, + "baseline_reconciles_to_active_evaluation": true, + "scenario_count": 36, + "complete_cartesian_grid": true, + "every_scenario_has_adverse_default_and_miscoverage_cells": true, + "minimum_adverse_default_cells_per_scenario": 17, + "minimum_adverse_miscoverage_cells_per_scenario": 21, + "universally_favorable_scenarios": 0, + "universally_adverse_scenarios": 0, + "scenarios_with_any_favorable_payoff_cell": 26, + "scenarios_with_any_favorable_default_cell": 20, + "scenarios_with_any_favorable_miscoverage_cell": 20, + "portfolios_per_scenario": 1440, + "purpose_cap_binding_share_by_cap": { + "0.20": 1.0, + "0.25": 1.0, + "0.30": 1.0, + "1.00": 0.0 + }, + "frontier_constraint_binding_share_by_budget": { + "500000": 1.0, + "1000000": 1.0, + "2000000": 1.0 + }, + "maximum_loan_weight_by_budget": { + "500000": 0.08, + "1000000": 0.04, + "2000000": 0.02 + }, + "direction_totals": { + "standardized_payoff": { + "gamma_1_lower": 740, + "gamma_1_higher": 223, + "crosses_zero": 709, + "exact_zero": 56 + }, + "funded_default": { + "gamma_1_lower": 133, + "gamma_1_higher": 1072, + "crosses_zero": 467, + "exact_zero": 56 + }, + "funded_binary_miscoverage": { + "gamma_1_lower": 131, + "gamma_1_higher": 1205, + "crosses_zero": 336, + "exact_zero": 56 + } + }, + "interpretation": "The complete structural grid rules out universal favorable direction, while one-sided favorable cells in many scenarios also rule out universal adversity.", + "estimand_boundary": "This complete retrospective assumption sensitivity changes budget, purpose concentration, and LGD without selecting a scenario. Direction remains conditional on ruler, coordinate, window, metric, and scenario.", + "rows": [ + { + "scenario_id": "b0500k_p020_l025", + "budget": 500000.0, + "purpose_cap": 0.2, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 16, + "standardized_payoff_gamma_1_higher_cells": 1, + "standardized_payoff_crosses_zero_cells": 31, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 39, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 39, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 9, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000448 + }, + { + "scenario_id": "b0500k_p020_l045", + "budget": 500000.0, + "purpose_cap": 0.2, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 32, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 8, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 39, + "funded_binary_miscoverage_gamma_1_lower_cells": 1, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000001126 + }, + { + "scenario_id": "b0500k_p020_l065", + "budget": 500000.0, + "purpose_cap": 0.2, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 31, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 1, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 0, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 3, + "funded_binary_miscoverage_crosses_zero_cells": 5, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.0800000000000165 + }, + { + "scenario_id": "b0500k_p025_l025", + "budget": 500000.0, + "purpose_cap": 0.25, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 16, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 24, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 39, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 9, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000282 + }, + { + "scenario_id": "b0500k_p025_l045", + "budget": 500000.0, + "purpose_cap": 0.25, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 30, + "standardized_payoff_gamma_1_higher_cells": 2, + "standardized_payoff_crosses_zero_cells": 8, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 23, + "funded_binary_miscoverage_gamma_1_lower_cells": 1, + "funded_binary_miscoverage_crosses_zero_cells": 16, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000006 + }, + { + "scenario_id": "b0500k_p025_l065", + "budget": 500000.0, + "purpose_cap": 0.25, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 32, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 8, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000108 + }, + { + "scenario_id": "b0500k_p030_l025", + "budget": 500000.0, + "purpose_cap": 0.3, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 17, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 23, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 40, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 7, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 6, + "funded_binary_miscoverage_crosses_zero_cells": 2, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000001112 + }, + { + "scenario_id": "b0500k_p030_l045", + "budget": 500000.0, + "purpose_cap": 0.3, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 24, + "standardized_payoff_gamma_1_higher_cells": 1, + "standardized_payoff_crosses_zero_cells": 15, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 24, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000001121 + }, + { + "scenario_id": "b0500k_p030_l065", + "budget": 500000.0, + "purpose_cap": 0.3, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 31, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 9, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 9, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000001108 + }, + { + "scenario_id": "b0500k_p100_l025", + "budget": 500000.0, + "purpose_cap": 1.0, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 15, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 25, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 25, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 23, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 24, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 16, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000001209 + }, + { + "scenario_id": "b0500k_p100_l045", + "budget": 500000.0, + "purpose_cap": 1.0, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 30, + "standardized_payoff_gamma_1_higher_cells": 1, + "standardized_payoff_crosses_zero_cells": 9, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 1, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 30, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 10, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000625 + }, + { + "scenario_id": "b0500k_p100_l065", + "budget": 500000.0, + "purpose_cap": 1.0, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 24, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 8, + "standardized_payoff_exact_zero_cells": 8, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 8, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 8, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.08000000000000271 + }, + { + "scenario_id": "b1000k_p020_l025", + "budget": 1000000.0, + "purpose_cap": 0.2, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 1, + "standardized_payoff_gamma_1_higher_cells": 17, + "standardized_payoff_crosses_zero_cells": 30, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 15, + "funded_default_crosses_zero_cells": 1, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 16, + "funded_binary_miscoverage_crosses_zero_cells": 0, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000256 + }, + { + "scenario_id": "b1000k_p020_l045", + "budget": 1000000.0, + "purpose_cap": 0.2, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 11, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 29, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 25, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 34, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 6, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.0400000000000051 + }, + { + "scenario_id": "b1000k_p020_l065", + "budget": 1000000.0, + "purpose_cap": 0.2, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 33, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 15, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 33, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.040000000000000084 + }, + { + "scenario_id": "b1000k_p025_l025", + "budget": 1000000.0, + "purpose_cap": 0.25, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 14, + "standardized_payoff_gamma_1_higher_cells": 15, + "standardized_payoff_crosses_zero_cells": 19, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 15, + "funded_binary_miscoverage_crosses_zero_cells": 1, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000237 + }, + { + "scenario_id": "b1000k_p025_l045", + "budget": 1000000.0, + "purpose_cap": 0.25, + "lgd": 0.45, + "is_baseline": true, + "standardized_payoff_gamma_1_lower_cells": 32, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 16, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 33, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000284 + }, + { + "scenario_id": "b1000k_p025_l065", + "budget": 1000000.0, + "purpose_cap": 0.25, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 25, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 23, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 38, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 10, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000424 + }, + { + "scenario_id": "b1000k_p030_l025", + "budget": 1000000.0, + "purpose_cap": 0.3, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 28, + "standardized_payoff_gamma_1_higher_cells": 9, + "standardized_payoff_crosses_zero_cells": 11, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 14, + "funded_binary_miscoverage_crosses_zero_cells": 2, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.0400000000000025 + }, + { + "scenario_id": "b1000k_p030_l045", + "budget": 1000000.0, + "purpose_cap": 0.3, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 18, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 22, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 33, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 7, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.040000000000002485 + }, + { + "scenario_id": "b1000k_p030_l065", + "budget": 1000000.0, + "purpose_cap": 0.3, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 31, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 9, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 26, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 14, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 4, + "funded_binary_miscoverage_crosses_zero_cells": 12, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000229 + }, + { + "scenario_id": "b1000k_p100_l025", + "budget": 1000000.0, + "purpose_cap": 1.0, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 18, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 22, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 0, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000118 + }, + { + "scenario_id": "b1000k_p100_l045", + "budget": 1000000.0, + "purpose_cap": 1.0, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 16, + "standardized_payoff_gamma_1_higher_cells": 16, + "standardized_payoff_crosses_zero_cells": 16, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 16, + "funded_default_crosses_zero_cells": 1, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 24, + "funded_binary_miscoverage_gamma_1_lower_cells": 10, + "funded_binary_miscoverage_crosses_zero_cells": 14, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.04000000000000138 + }, + { + "scenario_id": "b1000k_p100_l065", + "budget": 1000000.0, + "purpose_cap": 1.0, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 31, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 9, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 24, + "funded_binary_miscoverage_gamma_1_lower_cells": 3, + "funded_binary_miscoverage_crosses_zero_cells": 21, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.0400000000000017 + }, + { + "scenario_id": "b2000k_p020_l025", + "budget": 2000000.0, + "purpose_cap": 0.2, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 0, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 48, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 17, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 31, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 26, + "funded_binary_miscoverage_gamma_1_lower_cells": 1, + "funded_binary_miscoverage_crosses_zero_cells": 21, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000003775 + }, + { + "scenario_id": "b2000k_p020_l045", + "budget": 2000000.0, + "purpose_cap": 0.2, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 2, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 46, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 22, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 26, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 33, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 15, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000000912 + }, + { + "scenario_id": "b2000k_p020_l065", + "budget": 2000000.0, + "purpose_cap": 0.2, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 24, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 16, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 24, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 24, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 16, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000005261 + }, + { + "scenario_id": "b2000k_p025_l025", + "budget": 2000000.0, + "purpose_cap": 0.25, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 0, + "standardized_payoff_gamma_1_higher_cells": 9, + "standardized_payoff_crosses_zero_cells": 39, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 21, + "funded_binary_miscoverage_gamma_1_lower_cells": 1, + "funded_binary_miscoverage_crosses_zero_cells": 26, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000063124 + }, + { + "scenario_id": "b2000k_p025_l045", + "budget": 2000000.0, + "purpose_cap": 0.25, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 9, + "standardized_payoff_gamma_1_higher_cells": 16, + "standardized_payoff_crosses_zero_cells": 23, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 17, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 31, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 5, + "funded_binary_miscoverage_crosses_zero_cells": 11, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000009092 + }, + { + "scenario_id": "b2000k_p025_l065", + "budget": 2000000.0, + "purpose_cap": 0.25, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 20, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 28, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 16, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000003237 + }, + { + "scenario_id": "b2000k_p030_l025", + "budget": 2000000.0, + "purpose_cap": 0.3, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 16, + "standardized_payoff_gamma_1_higher_cells": 16, + "standardized_payoff_crosses_zero_cells": 16, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 33, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 7, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 32, + "funded_binary_miscoverage_gamma_1_lower_cells": 3, + "funded_binary_miscoverage_crosses_zero_cells": 13, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000002446 + }, + { + "scenario_id": "b2000k_p030_l045", + "budget": 2000000.0, + "purpose_cap": 0.3, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 9, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 31, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 31, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 17, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 37, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 11, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000004733 + }, + { + "scenario_id": "b2000k_p030_l065", + "budget": 2000000.0, + "purpose_cap": 0.3, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 31, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 17, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 33, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 48, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 0, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 1.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000000279 + }, + { + "scenario_id": "b2000k_p100_l025", + "budget": 2000000.0, + "purpose_cap": 1.0, + "lgd": 0.25, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 8, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 32, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 32, + "funded_default_gamma_1_lower_cells": 8, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 35, + "funded_binary_miscoverage_gamma_1_lower_cells": 8, + "funded_binary_miscoverage_crosses_zero_cells": 5, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000001707 + }, + { + "scenario_id": "b2000k_p100_l045", + "budget": 2000000.0, + "purpose_cap": 1.0, + "lgd": 0.45, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 32, + "standardized_payoff_gamma_1_higher_cells": 0, + "standardized_payoff_crosses_zero_cells": 16, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 40, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 8, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.02000000000000391 + }, + { + "scenario_id": "b2000k_p100_l065", + "budget": 2000000.0, + "purpose_cap": 1.0, + "lgd": 0.65, + "is_baseline": false, + "standardized_payoff_gamma_1_lower_cells": 33, + "standardized_payoff_gamma_1_higher_cells": 8, + "standardized_payoff_crosses_zero_cells": 7, + "standardized_payoff_exact_zero_cells": 0, + "funded_default_gamma_1_higher_cells": 33, + "funded_default_gamma_1_lower_cells": 0, + "funded_default_crosses_zero_cells": 15, + "funded_default_exact_zero_cells": 0, + "funded_binary_miscoverage_gamma_1_higher_cells": 40, + "funded_binary_miscoverage_gamma_1_lower_cells": 0, + "funded_binary_miscoverage_crosses_zero_cells": 8, + "funded_binary_miscoverage_exact_zero_cells": 0, + "activity_purpose_cap_binding_share": 0.0, + "activity_frontier_constraint_binding_share": 1.0, + "activity_portfolios": 1440, + "activity_maximum_loan_weight": 0.020000000000002353 + } + ] + }, + "rolling_origin": { + "scope": "two_origin_retrospective_recurrence_not_replication", + "run_tag": "ijds-rolling-origin-2017-2026-07-15-v4", + "protocol_tag": "protocol/ijds-rolling-origin-stability-2026-07-15-v4", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "origins": [ + "primary_2016", + "rolling_2017" + ], + "origin_count": 2, + "window_cells": 16, + "all_sixteen_upper_below_nominal": true, + "primary_2016_upper_max": 0.8825970442304121, + "rolling_2017_upper_max": 0.8747681732702159, + "model_or_origin_selected": false, + "independent_replication_claim_authorized": false, + "rows": [ + { + "origin": "primary_2016", + "window_id": "w01_2012m01_m06", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8770469335058413, + "coverage_lower": 0.8547135769057285, + "coverage_upper": 0.8805221682719095, + "mean_width": 0.6649918872956118 + }, + { + "origin": "primary_2016", + "window_id": "w02_2012m02_m07", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.879165821487114, + "coverage_lower": 0.8570245960359787, + "coverage_upper": 0.8825970442304121, + "mean_width": 0.665340945235394 + }, + { + "origin": "primary_2016", + "window_id": "w03_2012m03_m08", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8777349553471084, + "coverage_lower": 0.8556528430045902, + "coverage_upper": 0.8811430390830216, + "mean_width": 0.6653200651843035 + }, + { + "origin": "primary_2016", + "window_id": "w04_2012m04_m09", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8748732230670972, + "coverage_lower": 0.8525697153015469, + "coverage_upper": 0.8783411605508239, + "mean_width": 0.665031208384886 + }, + { + "origin": "primary_2016", + "window_id": "w05_2012m05_m10", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.87251311627295, + "coverage_lower": 0.8499323410013532, + "coverage_upper": 0.8760327947146382, + "mean_width": 0.6647076525088493 + }, + { + "origin": "primary_2016", + "window_id": "w06_2012m06_m11", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8721129123334083, + "coverage_lower": 0.8494255618350182, + "coverage_upper": 0.8756692934277905, + "mean_width": 0.6645688455708323 + }, + { + "origin": "primary_2016", + "window_id": "w07_2012m07_m12", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8694814343747773, + "coverage_lower": 0.8465175515402372, + "coverage_upper": 0.8730929448910822, + "mean_width": 0.6641544351090525 + }, + { + "origin": "primary_2016", + "window_id": "w08_2012m08_2013m01", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "coverage_resolved": 0.8658549288130389, + "coverage_lower": 0.8424845445620738, + "coverage_upper": 0.8695640637851894, + "mean_width": 0.5005686881638921 + }, + { + "origin": "rolling_2017", + "window_id": "w01_2013m01_m06", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8571061112708236, + "coverage_lower": 0.7535568380779456, + "coverage_upper": 0.8747681732702159, + "mean_width": 0.562496881282411 + }, + { + "origin": "rolling_2017", + "window_id": "w02_2013m02_m07", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8565916690623534, + "coverage_lower": 0.7524285065819337, + "coverage_upper": 0.8743401854613838, + "mean_width": 0.5624542821226854 + }, + { + "origin": "rolling_2017", + "window_id": "w03_2013m03_m08", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8546700761071855, + "coverage_lower": 0.7501588742623695, + "coverage_upper": 0.8725374489332728, + "mean_width": 0.5622224070109993 + }, + { + "origin": "rolling_2017", + "window_id": "w04_2013m04_m09", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8552601715816072, + "coverage_lower": 0.7506257700538227, + "coverage_upper": 0.8731988846378316, + "mean_width": 0.5621829530516125 + }, + { + "origin": "rolling_2017", + "window_id": "w05_2013m05_m10", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8554720007262714, + "coverage_lower": 0.7508203099669282, + "coverage_upper": 0.8734063938784774, + "mean_width": 0.5621735581049848 + }, + { + "origin": "rolling_2017", + "window_id": "w06_2013m06_m11", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8561831414262153, + "coverage_lower": 0.7517411322222942, + "coverage_upper": 0.8739900136177939, + "mean_width": 0.5622719357684526 + }, + { + "origin": "rolling_2017", + "window_id": "w07_2013m07_m12", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.855956181628361, + "coverage_lower": 0.7509759418974127, + "coverage_upper": 0.873886258997471, + "mean_width": 0.5621830723291171 + }, + { + "origin": "rolling_2017", + "window_id": "w08_2013m08_2014m01", + "candidate_rows": 77105, + "resolved_rows": 66091, + "unresolved_rows": 11014, + "coverage_resolved": 0.8553963474603199, + "coverage_lower": 0.7498994877115621, + "coverage_upper": 0.8735231178263407, + "mean_width": 0.5620254873690599 + } + ] + }, + "missingness_encoding": { + "scope": "three_declared_feature_semantics_preserving_catboost_encodings", + "run_tag": "ijds-missingness-sensitivity-2026-07-15-v3", + "protocol_tag": "protocol/ijds-missingness-sensitivity-2026-07-15-v3", + "protocol_commit": "199afb083da37af6a51d5ba9e3c4d6280b952fe9", + "specifications": [ + { + "id": "catboost_platt", + "encoding": "active_sentinel_convention", + "imported": true + }, + { + "id": "catboost_missing_indicators_platt", + "encoding": "active_mappings_plus_explicit_missing_indicators", + "imported": false, + "added_numeric_features": [ + "delinq_recency_missing", + "bankruptcy_count_missing" + ] + }, + { + "id": "catboost_native_missing_platt", + "encoding": "native_numeric_nan_with_nullable_binary_semantics", + "imported": false, + "dropped_numeric_features": [ + "delinq_recency", + "has_bankruptcy" + ], + "added_numeric_features": [ + "delinq_recency_native", + "has_bankruptcy_native" + ] + } + ], + "all_three_all_eight_upper_below_nominal": true, + "model_or_encoding_selected": false, + "missingness_mechanism_identified": false, + "portfolio_claim_authorized": false, + "rows": [ + { + "learner": "catboost_missing_indicators_platt", + "coverage_lower_min": 0.8455066465016318, + "coverage_upper_max": 0.8843322985486481, + "windows_with_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "roc_auc": 0.6436322403148622, + "brier": 0.12965175159650838, + "log_loss": 0.42810867198186225, + "ece_10": 0.04742800450704818, + "calibration_slope": 0.7913021921642017 + }, + { + "learner": "catboost_native_missing_platt", + "coverage_lower_min": 0.8452837698002069, + "coverage_upper_max": 0.8800366154580912, + "windows_with_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "roc_auc": 0.6442620078596174, + "brier": 0.12953346396196225, + "log_loss": 0.4267357005890355, + "ece_10": 0.0457674296671821, + "calibration_slope": 0.8184948803074936 + }, + { + "learner": "catboost_platt", + "coverage_lower_min": 0.8424845445620738, + "coverage_upper_max": 0.8825970442304121, + "windows_with_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "roc_auc": 0.6406052990079556, + "brier": 0.12987780627268664, + "log_loss": 0.42846570011698926, + "ece_10": 0.047215103762021676, + "calibration_slope": 0.7954274631320678 + } + ] + }, + "fit_label_completion": { + "scope": "observed_only_plus_three_declared_fit_label_stress_rules", + "run_tag": "ijds-fit-label-completion-sensitivity-2026-07-16-v2", + "protocol_tag": "protocol/ijds-fit-label-completion-sensitivity-2026-07-16-v2", + "protocol_commit": "fbcafcf84645024b9753aba2f04a4263b8e76236", + "coverage_cells": 32, + "phase_cells": 32, + "all_scenarios_all_windows_upper_below_nominal": true, + "w7_w8_crossing_by_scenario": { + "observed_only": true, + "all_unavailable_nondefault": true, + "all_unavailable_default": false, + "hindsight_terminal": true + }, + "w7_w8_crossing_scenarios": 3, + "w7_w8_crossing_in_all_scenarios": false, + "unavailable_fit_labels_by_split": { + "pd_development": 41, + "probability_calibration": 24, + "conformal_fit": 150 + }, + "unavailable_fit_labels_total": 215, + "scenario_or_result_selected": false, + "evaluation_outcomes_passed_to_fitting": false, + "scenarios_are_sharp_bounds_over_all_label_assignments": false, + "observed_only_active_replay": { + "coverage_max_abs_difference": 2.7411228735729765e-06, + "coverage_tolerance": 1e-05, + "geometry_max_abs_difference": 4.5158205730366774e-08, + "geometry_tolerance": 2e-05 + }, + "estimand_boundary": "The observed-only fit and three declared stress rules vary 215 fit labels that were unavailable at their information cutoffs. Every scenario retains all eight overall coverage upper bounds below 0.90, but the W7--W8 stratum-2 crossing fails under the all-default scenario. Nonlinear refitting means these scenarios are not sharp bounds over all 2^215 label assignments.", + "rows": [ + { + "fit_label_scenario": "all_unavailable_default", + "windows": 8, + "coverage_lower_min": 0.8496617050067659, + "coverage_upper_max": 0.8846692668948499, + "windows_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "mean_width_min": 0.6522207777018078, + "mean_width_max": 0.6537859842285183, + "w7_w8_stratum2_crossing": false + }, + { + "fit_label_scenario": "all_unavailable_nondefault", + "windows": 8, + "coverage_lower_min": 0.8424659715036218, + "coverage_upper_max": 0.8832524078643637, + "windows_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "mean_width_min": 0.49225574027300617, + "mean_width_max": 0.6485696557890056, + "w7_w8_stratum2_crossing": true + }, + { + "fit_label_scenario": "hindsight_terminal", + "windows": 8, + "coverage_lower_min": 0.8489798084321685, + "coverage_upper_max": 0.8845790548966542, + "windows_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "mean_width_min": 0.5003892753736984, + "mean_width_max": 0.6537743949931867, + "w7_w8_stratum2_crossing": true + }, + { + "fit_label_scenario": "observed_only", + "windows": 8, + "coverage_lower_min": 0.8424845445620738, + "coverage_upper_max": 0.8825943909363475, + "windows_upper_below_nominal": 8, + "all_windows_upper_below_nominal": true, + "mean_width_min": 0.5005686859367924, + "mean_width_max": 0.6653409371377675, + "w7_w8_stratum2_crossing": true + } + ] + }, + "allocation_granularity": { + "scope": "deterministic_usd25_floor_with_residual_cash", + "run_tag": "ijds-allocation-granularity-sensitivity-2026-07-16-v3", + "protocol_tag": "protocol/ijds-allocation-granularity-sensitivity-2026-07-16-v3", + "protocol_commit": "fb1a7b1837d1f8ab2b81239533f51c996f41671c", + "tracks": 96, + "cash_share_max": 3.333333333332966e-05, + "payoff_rate_perturbation_abs_max": 3.4483590284581524e-06, + "default_rate_perturbation_abs_max": 1.2842831024732755e-05, + "miscoverage_rate_perturbation_abs_max": 1.1901916329666367e-05, + "portfolios": 1440, + "source_rows": 143175, + "rounded_positive_rows": 143167, + "changed_rows": 2985, + "cash_residual_min": 25.0, + "cash_residual_mean": 26.006944444444443, + "cash_residual_max": 75.0, + "monthly_cash_share_max": 7.5e-05, + "scenario_or_result_selected": false, + "outcomes_passed_to_rounding": false, + "integer_policy_or_reoptimization_claim_authorized": false, + "estimand_boundary": "This deterministic diagnostic floors each continuous exposure to a USD 25 lot and holds the residual as cash. It supports numerical adequacy of the continuous relaxation for this archive, not optimality of an integer policy or robustness to other lot rules.", + "rows": [ + { + "tracks": 96, + "cash_share_max": 3.333333333332966e-05, + "payoff_rate_perturbation_abs_max": 3.4483590284581524e-06, + "default_rate_perturbation_abs_max": 1.2842831024732755e-05, + "miscoverage_rate_perturbation_abs_max": 1.1901916329666367e-05, + "portfolios": 1440, + "source_rows": 143175, + "rounded_positive_rows": 143167, + "changed_rows": 2985, + "cash_residual_min": 25.0, + "cash_residual_mean": 26.006944444444443, + "cash_residual_max": 75.0, + "monthly_cash_share_max": 7.5e-05 + } + ] + } + }, + "data_contract": { + "raw_rows": 2925493, + "valid_loan_rows": 2925492, + "raw_schema_columns": 142, + "term36_rows_all_dates": 2060077, + "term60_rows_all_dates": 865415, + "active_design_rows": 640543, + "eligible_raw_features": 30, + "late_schema_features": 48, + "declared_coverage_exceptions": 2, + "coverage_exceptions_requiring_sensitivity": 2, + "coverage_exception_rows": [ + { + "feature": "mths_since_last_delinq", + "minimum_fitting_coverage": 0.3360754556414438, + "primary_oot_coverage": 0.5285547507230226, + "coverage_exception": "structurally_nullable_credit_history_field", + "missingness_semantics": "frozen feature engineering maps missing recency to 999", + "requires_sensitivity": true + }, + { + "feature": "pub_rec_bankruptcies", + "minimum_fitting_coverage": 0.9217002237136466, + "primary_oot_coverage": 1.0, + "coverage_exception": "legacy_schema_partial_coverage", + "missingness_semantics": "frozen feature engineering maps missing bankruptcy count to no recorded bankruptcy", + "requires_sensitivity": true + } + ], + "primary_oot_funded_ratio": 0.9999961757560076, + "primary_oot_requested_minus_funded_usd": 18000.0, + "sampling": "none_all_eligible_rows_within_each_declared_temporal_role", + "population_boundary": "The active 640,543-row design is the exhaustive eligible 36-month population for the declared dates, horizon, schema, and observability rules; it is not a sample from the raw archive.", + "excluded_scope": "Sixty-month contracts, immature issue dates, and late-schema fields define different horizons, censoring regimes, or temporal feature support.", + "manifest": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json", + "bytes": 3654, + "sha256": "ac3cda119d919006210435eb0ff2234ffd6281d46edf8d29f04157b5f64ba6a5" + } + }, + "credit_risk_controls": { + "scope": "coverage_only_five_model_temporal_transport_robustness", + "outcome_free_run_tag": "ijds-credit-risk-controls-2026-07-13-v1b", + "verified_evaluation_run_tag": "ijds-credit-risk-controls-2026-07-15-v5", + "all_five_all_eight_upper_below_nominal": true, + "learners_reported": [ + "catboost_platt", + "numeric_logistic_platt", + "catboost_monotonic_platt", + "woe_scorecard_platform_platt", + "woe_scorecard_borrower_platt" + ], + "portfolio_learner": "catboost_platt", + "controls_enter_portfolio_optimization": false, + "model_or_feature_selected_from_oot": false, + "scorecard_superiority_claim_authorized": false, + "rows": [ + { + "learner": "catboost_platt", + "learner_label": "CatBoost", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "default_rate": 0.156167252353254, + "roc_auc": 0.6406052990079556, + "gini": 0.2812105980159112, + "ks": 0.20588181451177345, + "average_precision": 0.2350998561447269, + "brier": 0.12987780627268664, + "log_loss": 0.42846570011698926, + "ece_10": 0.047215103762021676, + "mean_calibration_error": -0.0471089772795769, + "calibration_intercept": 0.020228532871166095, + "calibration_slope": 0.7954274631320676, + "coverage_lower_min": 0.8424845445620738, + "coverage_upper_max": 0.8825970442304121, + "windows_upper_below_0_90": 8 + }, + { + "learner": "numeric_logistic_platt", + "learner_label": "Numeric logistic", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "default_rate": 0.156167252353254, + "roc_auc": 0.6420446945807546, + "gini": 0.2840893891615093, + "ks": 0.2096638411920359, + "average_precision": 0.24160749827700292, + "brier": 0.128845865115775, + "log_loss": 0.4327000703812235, + "ece_10": 0.031149374770624953, + "mean_calibration_error": -0.028922756555184087, + "calibration_intercept": -0.5991391826510897, + "calibration_slope": 0.5432095920187805, + "coverage_lower_min": 0.8500305128817427, + "coverage_upper_max": 0.8962217092520364, + "windows_upper_below_0_90": 8 + }, + { + "learner": "catboost_monotonic_platt", + "learner_label": "Monotonic CatBoost", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "default_rate": 0.156167252353254, + "roc_auc": 0.6519537792141734, + "gini": 0.3039075584283468, + "ks": 0.21969571391561254, + "average_precision": 0.24540282093264182, + "brier": 0.12861326735049058, + "log_loss": 0.42267695477753886, + "ece_10": 0.04156277982947754, + "mean_calibration_error": -0.04126410836574358, + "calibration_intercept": 0.052284509244371016, + "calibration_slope": 0.8351490384153631, + "coverage_lower_min": 0.8483960837379607, + "coverage_upper_max": 0.8864894266231527, + "windows_upper_below_0_90": 8 + }, + { + "learner": "woe_scorecard_platform_platt", + "learner_label": "Platform-signal WOE scorecard", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "default_rate": 0.156167252353254, + "roc_auc": 0.6330661809401453, + "gini": 0.26613236188029066, + "ks": 0.18843820687722823, + "average_precision": 0.22388791917890766, + "brier": 0.12948509376427816, + "log_loss": 0.4239586771759354, + "ece_10": 0.035967619770033946, + "mean_calibration_error": -0.03551023236967947, + "calibration_intercept": 0.15367586807582181, + "calibration_slope": 0.9186554972959701, + "coverage_lower_min": 0.8489081694924249, + "coverage_upper_max": 0.8949083286900688, + "windows_upper_below_0_90": 8 + }, + { + "learner": "woe_scorecard_borrower_platt", + "learner_label": "Pricing-excluded application WOE scorecard", + "candidate_rows": 376890, + "resolved_rows": 364814, + "unresolved_rows": 12076, + "default_rate": 0.156167252353254, + "roc_auc": 0.6129392722275023, + "gini": 0.22587854445500466, + "ks": 0.15809243291132968, + "average_precision": 0.2151918293521198, + "brier": 0.13019024412756694, + "log_loss": 0.42796595827034617, + "ece_10": 0.03176877520031669, + "mean_calibration_error": -0.029751737101973152, + "calibration_intercept": -0.28557359284456246, + "calibration_slope": 0.7098359251738742, + "coverage_lower_min": 0.8520125235479848, + "coverage_upper_max": 0.8977261269866539, + "windows_upper_below_0_90": 8 + } + ], + "declared_descriptive_differences": { + "platform_minus_borrower_oot_auc": 0.020126908712643, + "platform_minus_borrower_oot_brier": -0.0007051503632887757, + "monotonic_minus_active_oot_auc": 0.011348480206217815, + "monotonic_minus_active_oot_brier": -0.0012645389221960512 + }, + "endpoint_recovery_audit": { + "float_atol": 5e-14, + "float_rtol": 5e-14, + "maximum_absolute_drift": 4.440892098500626e-16, + "maximum_relative_drift": 4.942751060322734e-16 + }, + "calibration": { + "all_primary_oot_mean_calibration_error_negative": true, + "all_primary_oot_slopes_below_one": true, + "optimizer_success_rows": 30, + "optimizer_total_rows": 30 + }, + "woe_iv": { + "optbinning_problems": 45, + "all_optimal": true, + "platform_features": 26, + "pricing_excluded_application_features": 19, + "top_platform_iv": [ + { + "name": "int_rate_bucket__grade", + "iv": 0.33756858301375126 + }, + { + "name": "sub_grade", + "iv": 0.31932538983142866 + }, + { + "name": "grade", + "iv": 0.29954351738889673 + }, + { + "name": "int_rate", + "iv": 0.2784288707439742 + }, + { + "name": "int_rate_bucket", + "iv": 0.24485815197270147 + } + ], + "top_pricing_excluded_application_iv": [ + { + "name": "fico_score", + "iv": 0.2135744222493438 + }, + { + "name": "inq_last_6mths", + "iv": 0.17086393760483487 + }, + { + "name": "purpose", + "iv": 0.08887832866960639 + }, + { + "name": "rev_utilization", + "iv": 0.0739026629287923 + }, + { + "name": "delinq_recency", + "iv": 0.04928782736183494 + } + ] + }, + "temporal_shift": { + "primary_oot_score_psi": { + "catboost_platt": 0.1397574884533394, + "numeric_logistic_platt": 0.005167903809465062, + "catboost_monotonic_platt": 0.09370550856801749, + "woe_scorecard_platform_platt": 0.14886689865662606, + "woe_scorecard_borrower_platt": 0.07233216453444681 + }, + "top_primary_oot_feature_psi": [ + { + "learner": "woe_scorecard_platform_platt", + "feature": "int_rate_bucket__grade", + "psi": 0.9628044275190203 + }, + { + "learner": "woe_scorecard_platform_platt", + "feature": "verification_status", + "psi": 0.6081334177376476 + }, + { + "learner": "woe_scorecard_borrower_platt", + "feature": "verification_status", + "psi": 0.6081334177376476 + }, + { + "learner": "woe_scorecard_borrower_platt", + "feature": "dti", + "psi": 0.5522627527176915 + }, + { + "learner": "woe_scorecard_platform_platt", + "feature": "dti", + "psi": 0.5522627527176915 + } + ], + "recent_chargeoff_early_role_variation": [ + { + "role": "pd_development", + "rows": 17433, + "unique_observed": 1, + "constant_observed": true + }, + { + "role": "probability_calibration", + "rows": 14101, + "unique_observed": 1, + "constant_observed": true + } + ] + }, + "interpretation": "WOE/IV, a pricing-excluded application scorecard, and domain-safe monotonic constraints are predeclared coverage-only specification controls. They strengthen model-class robustness but do not define the paper's novelty, select a learner, or authorize a portfolio policy." + }, + "binary_phase_transition": { + "stratum": 2, + "w7_fit_prevalence": 0.10170349131388093, + "w8_fit_prevalence": 0.0971465213209362, + "w7_residual_quantile": 0.8884345991499274, + "w8_residual_quantile": 0.1118010883671265, + "w7_mean_width": 0.9842633701640712, + "w8_mean_width": 0.2076312400549422, + "w8_oot_coverage_bound": [ + 0.8225359596189609, + 0.8547066934861538 + ], + "label_lag_sensitivity": { + "admissible_lags_months": [ + 0, + 3, + 6 + ], + "nonadmissible_lags_months": [ + 8, + 12 + ], + "minimum_monthly_retention_by_lag": { + "0": 0.9971164936562861, + "3": 0.9966694421315571, + "6": 0.9929564746252483, + "8": 0.9868159653241828, + "12": 0.9745349467220517 + }, + "locked_retention_threshold": 0.99, + "w7_to_w8_threshold_crossing_at_all_admissible_lags": true, + "crossing_disappears_outside_locked_retention_scope": true, + "causal_interpretation_authorized": false, + "rows": [ + { + "charged_off_lag_months": 0, + "window_id": "w07_2012m07_m12", + "window_rows": 28503, + "retained_rows": 28453, + "window_retention": 0.9982457986878576, + "minimum_monthly_retention": 0.997116493656286, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 5939, + "phase_prevalence": 0.1032160296346186, + "phase_residual_quantile": 0.8886909233974287, + "phase_interval_width_mean": 0.98515359958887, + "phase_set_both_share": 0.0220575854520963 + }, + { + "charged_off_lag_months": 0, + "window_id": "w08_2012m08_2013m01", + "window_rows": 30259, + "retained_rows": 30198, + "window_retention": 0.9979840708549522, + "minimum_monthly_retention": 0.997116493656286, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 6251, + "phase_prevalence": 0.0990241561350184, + "phase_residual_quantile": 0.1118700059392816, + "phase_interval_width_mean": 0.2082540244531064, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 3, + "window_id": "w07_2012m07_m12", + "window_rows": 28503, + "retained_rows": 28449, + "window_retention": 0.998105462582886, + "minimum_monthly_retention": 0.9966694421315572, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 5937, + "phase_prevalence": 0.102913929594071, + "phase_residual_quantile": 0.8886749896176616, + "phase_interval_width_mean": 0.9851387217798036, + "phase_set_both_share": 0.0212228398180899 + }, + { + "charged_off_lag_months": 3, + "window_id": "w08_2012m08_2013m01", + "window_rows": 30259, + "retained_rows": 30191, + "window_retention": 0.9977527347235532, + "minimum_monthly_retention": 0.9966694421315572, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 6249, + "phase_prevalence": 0.0987357977276364, + "phase_residual_quantile": 0.1118616046782793, + "phase_interval_width_mean": 0.2082462684749756, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 6, + "window_id": "w07_2012m07_m12", + "window_rows": 28503, + "retained_rows": 28411, + "window_retention": 0.9967722695856576, + "minimum_monthly_retention": 0.9929564746252484, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 5929, + "phase_prevalence": 0.1017034913138809, + "phase_residual_quantile": 0.8884345991499274, + "phase_interval_width_mean": 0.9848983200473448, + "phase_set_both_share": 0.0129870129870129 + }, + { + "charged_off_lag_months": 6, + "window_id": "w08_2012m08_2013m01", + "window_rows": 30259, + "retained_rows": 30129, + "window_retention": 0.9957037575597344, + "minimum_monthly_retention": 0.9929564746252484, + "passes_locked_retention": true, + "phase_stratum": 2, + "phase_rows": 6238, + "phase_prevalence": 0.0971465213209362, + "phase_residual_quantile": 0.1118010883671265, + "phase_interval_width_mean": 0.2081817207788981, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 8, + "window_id": "w07_2012m07_m12", + "window_rows": 28503, + "retained_rows": 28349, + "window_retention": 0.9945970599586008, + "minimum_monthly_retention": 0.9868159653241828, + "passes_locked_retention": false, + "phase_stratum": 2, + "phase_rows": 5916, + "phase_prevalence": 0.0997295469912102, + "phase_residual_quantile": 0.1118832525211423, + "phase_interval_width_mean": 0.2083465264137485, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 8, + "window_id": "w08_2012m08_2013m01", + "window_rows": 30259, + "retained_rows": 30035, + "window_retention": 0.9925972437952344, + "minimum_monthly_retention": 0.9868159653241828, + "passes_locked_retention": false, + "phase_stratum": 2, + "phase_rows": 6217, + "phase_prevalence": 0.0940968312691008, + "phase_residual_quantile": 0.1117123594604784, + "phase_interval_width_mean": 0.208087780959762, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 12, + "window_id": "w07_2012m07_m12", + "window_rows": 28503, + "retained_rows": 28052, + "window_retention": 0.984177104164474, + "minimum_monthly_retention": 0.9745349467220517, + "passes_locked_retention": false, + "phase_stratum": 2, + "phase_rows": 5873, + "phase_prevalence": 0.0931380895624042, + "phase_residual_quantile": 0.1116902104787908, + "phase_interval_width_mean": 0.2081424372028483, + "phase_set_both_share": 0.0 + }, + { + "charged_off_lag_months": 12, + "window_id": "w08_2012m08_2013m01", + "window_rows": 30259, + "retained_rows": 29700, + "window_retention": 0.9815261575068576, + "minimum_monthly_retention": 0.9745349467220517, + "passes_locked_retention": false, + "phase_stratum": 2, + "phase_rows": 6170, + "phase_prevalence": 0.0871961102106969, + "phase_residual_quantile": 0.1115043934501935, + "phase_interval_width_mean": 0.2078719347850636, + "phase_set_both_share": 0.0 + } + ] + }, + "rows": [ + { + "window_id": "w01_2012m01_m06", + "fit_rows": 3249, + "fit_prevalence": 0.11665127731609726, + "fit_residual_quantile": 0.8919735883395368, + "coverage_lower": 0.8470909711943527, + "coverage_upper": 0.8755930016319405, + "mean_width": 0.9875801457582358, + "set_empty_share": 0.0, + "set_zero_only_share": 0.8844484926689185, + "set_both_share": 0.11555150733108151 + }, + { + "window_id": "w02_2012m02_m07", + "fit_rows": 3621, + "fit_prevalence": 0.11654239160452913, + "fit_residual_quantile": 0.8920116585417792, + "coverage_lower": 0.8473313345224993, + "coverage_upper": 0.8758080635571243, + "mean_width": 0.9876137950975035, + "set_empty_share": 0.0, + "set_zero_only_share": 0.8831707718192974, + "set_both_share": 0.11682922818070263 + }, + { + "window_id": "w03_2012m03_m08", + "fit_rows": 4018, + "fit_prevalence": 0.11398705823792932, + "fit_residual_quantile": 0.8916085888119072, + "coverage_lower": 0.845092160360292, + "coverage_upper": 0.8738851569319519, + "mean_width": 0.9872553690886569, + "set_empty_share": 0.0, + "set_zero_only_share": 0.8954419522562526, + "set_both_share": 0.10455804774374738 + }, + { + "window_id": "w04_2012m04_m09", + "fit_rows": 4511, + "fit_prevalence": 0.10995344712923964, + "fit_residual_quantile": 0.890281077239412, + "coverage_lower": 0.837223424038863, + "coverage_upper": 0.8673573949675509, + "mean_width": 0.9860405848595958, + "set_empty_share": 0.0, + "set_zero_only_share": 0.9347223803559908, + "set_both_share": 0.06527761964400926 + }, + { + "window_id": "w05_2012m05_m10", + "fit_rows": 5001, + "fit_prevalence": 0.10657868426314737, + "fit_residual_quantile": 0.8894148014498154, + "coverage_lower": 0.8320872392374157, + "coverage_upper": 0.8630941085683201, + "mean_width": 0.9852197606709866, + "set_empty_share": 0.0, + "set_zero_only_share": 0.9606310169898921, + "set_both_share": 0.03936898301010791 + }, + { + "window_id": "w06_2012m06_m11", + "fit_rows": 5502, + "fit_prevalence": 0.10523446019629226, + "fit_residual_quantile": 0.8892521057704302, + "coverage_lower": 0.8311890394322365, + "coverage_upper": 0.8623603678823991, + "mean_width": 0.9850630399535071, + "set_empty_share": 0.0, + "set_zero_only_share": 0.9660202158209673, + "set_both_share": 0.033979784179032725 + }, + { + "window_id": "w07_2012m07_m12", + "fit_rows": 5929, + "fit_prevalence": 0.10170349131388093, + "fit_residual_quantile": 0.8884345991499274, + "coverage_lower": 0.8264829784811568, + "coverage_upper": 0.858375396915759, + "mean_width": 0.9842633701640712, + "set_empty_share": 0.0, + "set_zero_only_share": 0.9904360696800638, + "set_both_share": 0.00956393031993624 + }, + { + "window_id": "w08_2012m08_2013m01", + "fit_rows": 6238, + "fit_prevalence": 0.0971465213209362, + "fit_residual_quantile": 0.1118010883671265, + "coverage_lower": 0.8225359596189609, + "coverage_upper": 0.8547066934861538, + "mean_width": 0.2076312400549422, + "set_empty_share": 0.002618695206649209, + "set_zero_only_share": 0.9973813047933507, + "set_both_share": 0.0 + } + ] + }, + "portfolio": { + "c2_cells": 1080, + "c2_match_residual_abs_max": 8.326672684688674e-17, + "c2_point_minus_guardrail_objective_min": -1.4551915228366852e-10, + "broad_stress_all_envelopes_cross_zero": true, + "broad_stress_cells": 216, + "development_direction_counts": [ + { + "metric": "funded_miscoverage", + "direction": "crosses_zero", + "cells": 45 + }, + { + "metric": "funded_miscoverage", + "direction": "guardrail_higher", + "cells": 27 + }, + { + "metric": "standardized_payoff", + "direction": "crosses_zero", + "cells": 66 + }, + { + "metric": "standardized_payoff", + "direction": "guardrail_lower", + "cells": 6 + }, + { + "metric": "terminal_default", + "direction": "crosses_zero", + "cells": 72 + } + ], + "w8_development_all_envelopes_cross_zero": true, + "named_direction_counts": [ + { + "comparator_rule": "c0_same_numeric_cap", + "metric": "payoff", + "guardrail_lower": 0, + "crosses_zero": 54, + "guardrail_higher": 18 + }, + { + "comparator_rule": "c0_same_numeric_cap", + "metric": "default", + "guardrail_lower": 72, + "crosses_zero": 0, + "guardrail_higher": 0 + }, + { + "comparator_rule": "c0_same_numeric_cap", + "metric": "miscoverage", + "guardrail_lower": 15, + "crosses_zero": 52, + "guardrail_higher": 5 + }, + { + "comparator_rule": "c1_development_mean", + "metric": "payoff", + "guardrail_lower": 28, + "crosses_zero": 40, + "guardrail_higher": 4 + }, + { + "comparator_rule": "c1_development_mean", + "metric": "default", + "guardrail_lower": 3, + "crosses_zero": 35, + "guardrail_higher": 34 + }, + { + "comparator_rule": "c1_development_mean", + "metric": "miscoverage", + "guardrail_lower": 3, + "crosses_zero": 24, + "guardrail_higher": 45 + }, + { + "comparator_rule": "c2_contemporaneous", + "metric": "payoff", + "guardrail_lower": 35, + "crosses_zero": 31, + "guardrail_higher": 6 + }, + { + "comparator_rule": "c2_contemporaneous", + "metric": "default", + "guardrail_lower": 7, + "crosses_zero": 58, + "guardrail_higher": 7 + }, + { + "comparator_rule": "c2_contemporaneous", + "metric": "miscoverage", + "guardrail_lower": 5, + "crosses_zero": 20, + "guardrail_higher": 47 + } + ], + "evaluated_point_cap_solver_stability": { + "scope": "evaluated_point_caps_only_not_continuous_uniqueness", + "point_cap_rows": 7297, + "named_unique_cap_months": 2204, + "minimum_absolute_nonbasic_reduced_cost": 0.00038757301081204787, + "near_zero_bases": 0, + "primal_degenerate_bases": 2941, + "reversed_order_reruns": 2941, + "tie_sensitive_rows": 0, + "maximum_allocation_distance": 1.449778343358131e-14, + "maximum_absolute_objective_difference": 1.7171259969472885e-09, + "continuous_frontier_uniqueness_claim": false + } + }, + "decision_challenger": { + "scope": "finite_two_ruler_three_interior_coordinate_diagnostic", + "continuous_frontier_claim": false, + "tracks_are_independent_replications": false, + "primary_ruler": "objective_matched", + "secondary_ruler": "normalized_score", + "endpoint_contrast": "gamma_1_minus_gamma_0", + "run_tag": "ijds-normalized-objective-frontier-2026-07-15-v5", + "protocol_tag": "protocol/ijds-normalized-objective-frontier-2026-07-15-v5", + "protocol_commit": "e2bba580a0b07c145bd64ff61440973d6e31349b", + "manifest": { + "path": "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/verified_evaluation_manifest.json", + "bytes": 18231, + "sha256": "9ee55a2522349c8520f308bc69273774dd48964847dfd340b78a7be46474cd7f" + }, + "counts": { + "evaluated_portfolios": 6240, + "joined_funded_rows": 622455, + "window_endpoint_contrasts": 48, + "monthly_endpoint_contrasts": 720, + "metric_direction_cells": 144, + "outcome_audit_rows": 8 + }, + "endpoint_recovery_audit": { + "float_atol": 5e-14, + "float_rtol": 5e-14, + "maximum_absolute_drift": 0.0, + "maximum_relative_drift": 0.0 + }, + "primary_oot_unresolved": 12076, + "metric_directions": { + "funded_binary_miscoverage": { + "cells": 48, + "direction_counts": { + "crosses_zero": 8, + "gamma_1_higher": 40 + }, + "universal_nonzero_direction": null, + "eligible_for_separate_rolling_origin_challenger": false + }, + "funded_default": { + "cells": 48, + "direction_counts": { + "crosses_zero": 15, + "gamma_1_higher": 33 + }, + "universal_nonzero_direction": null, + "eligible_for_separate_rolling_origin_challenger": false + }, + "standardized_payoff": { + "cells": 48, + "direction_counts": { + "crosses_zero": 16, + "gamma_1_lower": 32 + }, + "universal_nonzero_direction": null, + "eligible_for_separate_rolling_origin_challenger": false + } + }, + "objective_matched_coordinate_025_repetition": { + "allocations_identical_across_windows_to_cents": true, + "changed_loan_month_positions_min": 44, + "changed_loan_month_positions_max": 44, + "one_way_turnover_usd_min": 155937.26968238514, + "one_way_turnover_usd_max": 155937.26968417346 + }, + "rows": [ + { + "ruler": "objective_matched", + "coordinate": 0.25, + "ruler_semantics": "common_plugin_objective_floor", + "window_cells": 8, + "active_months_per_window_min": 4, + "active_months_per_window_max": 4, + "expected_objective_difference_usd_min": 0.0, + "expected_objective_difference_usd_max": 0.0, + "payoff_bound_usd_lower_min": -9134.339201705214, + "payoff_bound_usd_upper_max": 5603.660798333496, + "payoff_identification_width_usd_min": 14738.0, + "payoff_identification_width_usd_max": 14738.0, + "default_bound_pp_lower_min": -0.006789927307658072, + "default_bound_pp_upper_max": 0.12654340602615935, + "default_identification_width_pp_min": 0.13333333333333333, + "default_identification_width_pp_max": 0.13333333333333333, + "miscoverage_bound_pp_lower_min": -0.006789927307658072, + "miscoverage_bound_pp_upper_max": 0.12654340602615935, + "miscoverage_identification_width_pp_min": 0.13333333333333333, + "miscoverage_identification_width_pp_max": 0.13333333333333333, + "payoff_direction_pattern": "crosses_zero:8", + "default_direction_pattern": "crosses_zero:8", + "miscoverage_direction_pattern": "crosses_zero:8" + }, + { + "ruler": "objective_matched", + "coordinate": 0.5, + "ruler_semantics": "common_plugin_objective_floor", + "window_cells": 8, + "active_months_per_window_min": 15, + "active_months_per_window_max": 15, + "expected_objective_difference_usd_min": 0.0, + "expected_objective_difference_usd_max": 4.656612873077393e-10, + "payoff_bound_usd_lower_min": -82616.17453934051, + "payoff_bound_usd_upper_max": -27958.36861160415, + "payoff_identification_width_usd_min": 53530.42261970967, + "payoff_identification_width_usd_max": 54370.44762289172, + "default_bound_pp_lower_min": 0.457176922370002, + "default_bound_pp_upper_max": 1.0972986534287477, + "default_identification_width_pp_min": 0.6281743811120005, + "default_identification_width_pp_max": 0.6375359246695269, + "miscoverage_bound_pp_lower_min": 1.0153517160736112, + "miscoverage_bound_pp_upper_max": 1.9321319867620808, + "miscoverage_identification_width_pp_min": 0.6281743811120005, + "miscoverage_identification_width_pp_max": 0.6375359246695269, + "payoff_direction_pattern": "gamma_1_lower:8", + "default_direction_pattern": "gamma_1_higher:8", + "miscoverage_direction_pattern": "gamma_1_higher:8" + }, + { + "ruler": "objective_matched", + "coordinate": 0.75, + "ruler_semantics": "common_plugin_objective_floor", + "window_cells": 8, + "active_months_per_window_min": 15, + "active_months_per_window_max": 15, + "expected_objective_difference_usd_min": 0.0, + "expected_objective_difference_usd_max": 4.656612873077393e-10, + "payoff_bound_usd_lower_min": -179484.66459580814, + "payoff_bound_usd_upper_max": 92558.17998965128, + "payoff_identification_width_usd_min": 109550.45677067185, + "payoff_identification_width_usd_max": 271561.8753013419, + "default_bound_pp_lower_min": -0.4352498618487788, + "default_bound_pp_upper_max": 2.494842541954273, + "default_identification_width_pp_min": 1.2006435014285781, + "default_identification_width_pp_max": 2.9246052959625377, + "miscoverage_bound_pp_lower_min": 1.3252372459918247, + "miscoverage_bound_pp_upper_max": 4.184767620139838, + "miscoverage_identification_width_pp_min": 1.1339768347619112, + "miscoverage_identification_width_pp_max": 2.645938629295871, + "payoff_direction_pattern": "gamma_1_lower:1;crosses_zero:7", + "default_direction_pattern": "gamma_1_higher:1;crosses_zero:7", + "miscoverage_direction_pattern": "gamma_1_higher:8" + }, + { + "ruler": "normalized_score", + "coordinate": 0.25, + "ruler_semantics": "common_relative_score_relaxation", + "window_cells": 8, + "active_months_per_window_min": 15, + "active_months_per_window_max": 15, + "expected_objective_difference_usd_min": 425196.4593238123, + "expected_objective_difference_usd_max": 557294.0146438954, + "payoff_bound_usd_lower_min": -626374.6147907167, + "payoff_bound_usd_upper_max": -195967.628254245, + "payoff_identification_width_usd_min": 340845.41518178507, + "payoff_identification_width_usd_max": 373705.3081783684, + "default_bound_pp_lower_min": 8.482852571867097, + "default_bound_pp_upper_max": 13.424612654302775, + "default_identification_width_pp_min": 3.7599027571533226, + "default_identification_width_pp_max": 4.101884671392359, + "miscoverage_bound_pp_lower_min": 8.390955538535708, + "miscoverage_bound_pp_upper_max": 13.753589682046982, + "miscoverage_identification_width_pp_min": 3.639902757153319, + "miscoverage_identification_width_pp_max": 3.9432180047256917, + "payoff_direction_pattern": "gamma_1_lower:8", + "default_direction_pattern": "gamma_1_higher:8", + "miscoverage_direction_pattern": "gamma_1_higher:8" + }, + { + "ruler": "normalized_score", + "coordinate": 0.5, + "ruler_semantics": "common_relative_score_relaxation", + "window_cells": 8, + "active_months_per_window_min": 15, + "active_months_per_window_max": 15, + "expected_objective_difference_usd_min": 152030.9877972738, + "expected_objective_difference_usd_max": 226847.9705495569, + "payoff_bound_usd_lower_min": -259658.18448523604, + "payoff_bound_usd_upper_max": -54025.8191528258, + "payoff_identification_width_usd_min": 146499.38968489924, + "payoff_identification_width_usd_max": 198657.05281574934, + "default_bound_pp_lower_min": 3.2213520803827533, + "default_bound_pp_upper_max": 6.563699817357987, + "default_identification_width_pp_min": 1.5345657876974395, + "default_identification_width_pp_max": 2.0382675456422694, + "miscoverage_bound_pp_lower_min": 2.1069714896837946, + "miscoverage_bound_pp_upper_max": 5.21415906991543, + "miscoverage_identification_width_pp_min": 1.5345657876974395, + "miscoverage_identification_width_pp_max": 1.9182675456351144, + "payoff_direction_pattern": "gamma_1_lower:8", + "default_direction_pattern": "gamma_1_higher:8", + "miscoverage_direction_pattern": "gamma_1_higher:8" + }, + { + "ruler": "normalized_score", + "coordinate": 0.75, + "ruler_semantics": "common_relative_score_relaxation", + "window_cells": 8, + "active_months_per_window_min": 15, + "active_months_per_window_max": 15, + "expected_objective_difference_usd_min": 28263.397028705105, + "expected_objective_difference_usd_max": 51201.887608815916, + "payoff_bound_usd_lower_min": -135781.22300933156, + "payoff_bound_usd_upper_max": 9812.591895865517, + "payoff_identification_width_usd_min": 83832.63333388747, + "payoff_identification_width_usd_max": 97897.24126478202, + "default_bound_pp_lower_min": 1.4391554644757423, + "default_bound_pp_upper_max": 2.3807174501084845, + "default_identification_width_pp_min": 0.8041480359892984, + "default_identification_width_pp_max": 0.9415619856327422, + "miscoverage_bound_pp_lower_min": 0.34472769373846956, + "miscoverage_bound_pp_upper_max": 1.6990552129348044, + "miscoverage_identification_width_pp_min": 0.7256480359893016, + "miscoverage_identification_width_pp_max": 0.9415619856327422, + "payoff_direction_pattern": "gamma_1_lower:7;crosses_zero:1", + "default_direction_pattern": "gamma_1_higher:8", + "miscoverage_direction_pattern": "gamma_1_higher:8" + } + ], + "interpretation": { + "coordinate_one_is_structural_null": true, + "objective_matched_equalizes_plugin_objective_floor": true, + "normalized_score_equalizes_relative_score_relaxation": true, + "normalized_score_equalizes_opportunity_cost": false, + "objective_matched_coordinate_025_is_one_repeated_allocation_contrast": true, + "preferred_gamma": null, + "preferred_ruler": null, + "preferred_coordinate": null, + "policy_winner": null, + "permitted_conclusion": "Within the predeclared finite grid, the gamma endpoint allocation contrast is not invariant to the outcome-free ruler or interior coordinate." + } + }, + "audit_thesis": "Binary absolute-residual conformal coverage does not transport to the later archive under five declared credit-risk model specifications, recurs in the only additional feasible origin, persists under three missing-value encodings, and remains below nominal under four declared fit-label scenarios. A prevalence-threshold crossing explains one observed geometry change but is not invariant to every fit-label scenario. Portfolio direction is not identified without outcome-free comparator support and is not invariant to the declared ruler or interior coordinate; USD 25 floor rounding produces only negligible rate perturbations in the evaluated archive.", + "source_artifacts": { + "active_source_registry": { + "path": "configs/ijds_active_evidence_sources.yaml", + "bytes": 15659, + "sha256": "3055bf813907f61d1c9d09bc8a3f32bc8410d11cb2dafbad187427a54a91797e" + }, + "claim_ledger_contract": { + "path": "configs/ijds_claim_ledger.yaml", + "bytes": 10976, + "sha256": "b5fe8e07f8663bf9e213fc3e0e012206201a34f92c92ed2cd6a32ceedbbe1f10" + }, + "publication_targets_contract": { + "path": "configs/crpto_publication_targets.yaml", + "bytes": 8568, + "sha256": "6eb526fb1e0bbcc184147e8d9a9375fbd58bed6f064eb2de1a5242308d0d638f" + }, + "evidence_builder": { + "path": "scripts/build_ijds_binary_geometry_frontier_v4_evidence.py", + "bytes": 101078, + "sha256": "c7507936b3796dfd97b46247f8127c8b9b43709f31779b76624bdaede483e4d6" + }, + "publication_integrity_checker": { + "path": "scripts/check_publication_integrity.py", + "bytes": 16709, + "sha256": "79b486122a114c8100ad327e13bbd404639ab804687a61cbba30e95d2b86fccc" + }, + "paper_pdf_auditor": { + "path": "scripts/inspect_ijds_pdfs.py", + "bytes": 6866, + "sha256": "d2c81df82e112ba9a3a2afd147a51f9abcd20ae40cd3d3a02ba677aae60c9890" + }, + "publication_generation_helper": { + "path": "src/ijds_audit/publication_generation.py", + "bytes": 7099, + "sha256": "b7065e4c61d8bf68bd2972c5e0b44844414fae06e6a29b9baa748c999a21981c" + }, + "v4_config_loader": { + "path": "src/ijds_audit/config.py", + "bytes": 25008, + "sha256": "a13cadec6a16267cfe80c3bfce986e6d6bbd6461b61d605036d3a917c4308f8e" + }, + "grid_contracts": { + "path": "src/ijds_audit/grid_contracts.py", + "bytes": 5258, + "sha256": "439e08405a4c1e58e21fe2fe84e02cdfd1d8a88bbbc88b550ae395ace5948a8b" + }, + "endpoint_availability_sensitivity/loader": { + "path": "src/ijds_audit/sensitivity_evidence.py", + "bytes": 17942, + "sha256": "1a2fc17e58fc660e7455aa721cd2180d73fa86ab916b9bfe1682e073d43f4ab2" + }, + "portfolio_structure_sensitivity/loader": { + "path": "src/ijds_audit/structural_evidence.py", + "bytes": 23110, + "sha256": "6027a6612241d9e8ae213099066886f77a5231b6b828679fec30aa59ce07c7a6" + }, + "robustness_sensitivities/loader": { + "path": "src/ijds_audit/robustness_evidence.py", + "bytes": 21066, + "sha256": "afda7c916742e8257bb6efe515aabaa6ef823b2b594df7d212e1fbd1d5ce5074" + }, + "claim_ledger_loader": { + "path": "src/ijds_audit/claim_ledger.py", + "bytes": 9426, + "sha256": "de79a56205f6f23e99e5b2978e628601ec2d9c7f0f38331e3cf2e239134d3bee" + }, + "source_registry_loader": { + "path": "src/ijds_audit/publication_sources.py", + "bytes": 18075, + "sha256": "99bbad2ca6445c284398102ed60ed1d33e64e1f63bc610acc5dc6b14f80eaa4c" + }, + "artifact_descriptor_helper": { + "path": "src/utils/artifact_descriptor.py", + "bytes": 1705, + "sha256": "ef8f6df7cf311e2173659518370cfde5ed5de4380b444aaad35dcd794cd620f8" + }, + "pipeline_runtime_helper": { + "path": "src/utils/pipeline_runtime.py", + "bytes": 4402, + "sha256": "7c350fdf6736d210418e8cefec5dc8d85c7f1bc2a1bf94aa6ae44a12d4948cde" + }, + "config": { + "path": "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml", + "bytes": 1371, + "sha256": "c749befbd0ab7e0f8d6fcded7e7c730cae998032f897f65c7e7673d2a12c3715" + }, + "outcome_free/source_protocol_freeze": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json", + "bytes": 20362, + "sha256": "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + }, + "freeze": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/protocol_freeze.json", + "bytes": 22714, + "sha256": "b3ddeb31dee3e94cd06ea5b9a4e4f41aa837d8e813ee2db190e92f9283f69752" + }, + "summary": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/binary_geometry_frontier_v4_summary.json", + "bytes": 58584, + "sha256": "07e793df7e457c6afa946e04d3e95d30307faa48424249659ae8d921ea060a3f" + }, + "execution_receipt": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/execution_receipt.json", + "bytes": 1142, + "sha256": "b67cf57d200e71417f4a0b51ac2a2978b8b5105396ebfe3b7647cc1554d2aced" + }, + "two_ruler/outcome_free/freeze": { + "path": "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/protocol_freeze.json", + "bytes": 15192, + "sha256": "7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185" + }, + "two_ruler/manifest": { + "path": "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/verified_evaluation_manifest.json", + "bytes": 18231, + "sha256": "9ee55a2522349c8520f308bc69273774dd48964847dfd340b78a7be46474cd7f" + }, + "two_ruler/summary": { + "path": "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/normalized_objective_frontier_v3_summary.json", + "bytes": 13175, + "sha256": "c9ac243e43ae30f31b60738f6fdd48db46499b9549b2b08ad8c6209fc3dce8f7" + }, + "two_ruler/execution_receipt": { + "path": "models/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/execution_receipt.json", + "bytes": 636, + "sha256": "cb876f7b1d714ea910154653eb68a6e899cb07567ed4b9c585f21c2a59050840" + }, + "credit_controls/summary": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/credit_risk_controls_summary.json", + "bytes": 23359, + "sha256": "37c0a5c224b62ee3f47259aea4231df26ea184d3666e7fab80431ad26d611715" + }, + "credit_controls/execution_receipt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/execution_receipt.json", + "bytes": 826, + "sha256": "179016a1c064318502cc11a7f197db0656eff8cf3f896e0e1905ddc1a9f994db" + }, + "credit_controls/freeze": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/protocol_freeze.json", + "bytes": 25908, + "sha256": "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35" + }, + "raw_data_audit/manifest": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/evidence.json", + "bytes": 3654, + "sha256": "ac3cda119d919006210435eb0ff2234ffd6281d46edf8d29f04157b5f64ba6a5" + }, + "label_lag_sensitivity/manifest": { + "path": "reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/evidence.json", + "bytes": 2476, + "sha256": "6078892647c0f041631ebdb47c63b5613619fd61a82c2d13d9fb8c9c84e0adc7" + }, + "label_lag_sensitivity/table": { + "path": "reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/label_lag_phase_sensitivity.csv", + "bytes": 6476, + "sha256": "ddafa5977bb3ae3e1698caf412c3824708633bb17310d88d43deb301e5f389ea" + }, + "endpoint_availability_sensitivity/summary": { + "path": "models/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/endpoint_sensitivity_summary.json", + "bytes": 10692, + "sha256": "f8192d9b97289ea49480b92975258801c2f9e6640acdee7606307d0368575ff6" + }, + "portfolio_structure_sensitivity/config": { + "path": "configs/experiments/ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml", + "bytes": 5016, + "sha256": "600d3332da280f4329603b4c5e72df2077711137de403b0236ab92e7285122e4" + }, + "portfolio_structure_sensitivity/freeze": { + "path": "models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/protocol_freeze.json", + "bytes": 176170, + "sha256": "2fb1ac688cc702c47b26552c2af9d953ad6103eb1df17190f388dfeeb04de951" + }, + "portfolio_structure_sensitivity/summary": { + "path": "models/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/structural_sensitivity_summary.json", + "bytes": 8091, + "sha256": "240aef92e9eb204d6a9dd3eda388544face71118c63509476087c6baa461b853" + }, + "rolling_origin/summary": { + "path": "models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/binary_geometry_frontier_v4_summary.json", + "bytes": 21830, + "sha256": "ab1bf51d5a6f54592276ec33de4affda6ff5327e6c2c362327f08089b09ff4a8" + }, + "rolling_origin/execution_receipt": { + "path": "models/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/execution_receipt.json", + "bytes": 1134, + "sha256": "d64ef87ce6e1a2f7f0493ed17e009bf8349f68546ed137d0cafd74a6292edb0a" + }, + "missingness_encoding/summary": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/missingness_sensitivity_summary.json", + "bytes": 10170, + "sha256": "33d4c760746b8c860d441fc5a461fd94affbe06c0bcfa47a69aa5a7aef077ebe" + }, + "missingness_encoding/execution_receipt": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/execution_receipt.json", + "bytes": 574, + "sha256": "85f1da84b2961d1fcbb2bc3c45f0e4b970d1ed011e8023e248db410230b1186d" + }, + "missingness_encoding/freeze": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/protocol_freeze.json", + "bytes": 7793, + "sha256": "0e3087268492f4fb27af3fecac2524a60514518441bfbf1f61d6311366144552" + }, + "fit_label_completion/freeze": { + "path": "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/protocol_freeze.json", + "bytes": 7348, + "sha256": "f4b887f3a08e453e39be6b461887449e89c7882bff8b0918b9849cd29a8426db" + }, + "fit_label_completion/summary": { + "path": "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/fit_label_completion_summary.json", + "bytes": 4644, + "sha256": "8b9f7fde7f8776809350d4921b4221a9df0325f957ffbbfc94686bf1463b9c38" + }, + "allocation_granularity/freeze": { + "path": "models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/protocol_freeze.json", + "bytes": 4677, + "sha256": "cb66f9eedcd130d5d8f57cd182bcb83603df3fb2bdd8292244ec6a5f77d78c1d" + }, + "allocation_granularity/summary": { + "path": "models/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/allocation_granularity_summary.json", + "bytes": 2226, + "sha256": "7e8d9609367ae2cb07e9d0d56e676d7af5486258f7f36a2680d66d2f90d4b51a" + }, + "solver_tie_audit/manifest": { + "path": "reports/crpto/ijds_policy_support_tie_evidence.json", + "bytes": 12845, + "sha256": "2903694ee502e6144008bbbc5891e5932f88f8527e12686363c85a28fa78684e" + }, + "outcome_free/scores": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/prediction/scores.parquet", + "bytes": 14212211, + "sha256": "4053efdbf13066355bb772233231e5fe4ccb436ca75aaf873fb14e213c1e319c" + }, + "outcome_free/recipes": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/prediction/residual_recipes.json", + "bytes": 74218, + "sha256": "0874a5e9eea37adce302f4a059d4ccde5570230a7fdabcc29ceab410988f207a" + }, + "outcome_free/fit_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/prediction/residual_fit_audit.parquet", + "bytes": 18436852, + "sha256": "a80efcedfe749c52c0624536d361a8b1e9e5121901ee9b91ad4c746c68da107c" + }, + "outcome_free/outcome_free_geometry": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/prediction/outcome_free_geometry.parquet", + "bytes": 153841, + "sha256": "7763c91887fa558e021b386e33335f8a9c82742554cd26263be5a5b06571e588" + }, + "outcome_free/availability_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/data/label_availability_audit.parquet", + "bytes": 5281, + "sha256": "c0273ff3ff37894b17d9afacfb27e992d05ee76908e69754be474e30dc4209aa" + }, + "outcome_free/monthly_residual_availability": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/data/monthly_residual_availability.parquet", + "bytes": 5534, + "sha256": "a21ecfcdbe48b953051ce86e00d238e587461c3ec591ca793932dabbca044961" + }, + "outcome_free/solve_records": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/portfolio/outcome_free_solve_records.parquet", + "bytes": 1613402, + "sha256": "00f7ff7d7b17407670e42956b580a2b6a1681356a56b7b38e534be6c2a9ae8fe" + }, + "outcome_free/allocations": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/portfolio/outcome_free_funded_allocations.parquet", + "bytes": 22686519, + "sha256": "18e3bfc1054fd6ac36549cefd4d235d22f39bd955c203305d21b68f8d3a408a8" + }, + "outcome_free/comparator_support": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/portfolio/development_comparator_support.parquet", + "bytes": 6376, + "sha256": "530f014fce59576c83b2898353d7f06d808d2cf4aee6d1590b0f9c15715b4dae" + }, + "outcome_free/frontier_breakpoints": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/portfolio/exact_frontier_breakpoints.parquet", + "bytes": 867552, + "sha256": "0433a66c7d9e689108158c980d1451af57b6bb93beb9b8b64134c1b7ea9414eb" + }, + "evaluation/temporal_coverage": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/temporal_coverage.parquet", + "bytes": 215583, + "sha256": "25be1d2e065a6eab632618127cb21ccfb99be6f582ef557e80122c184f28dfe2" + }, + "evaluation/monthly_evaluation": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/monthly_evaluation.parquet", + "bytes": 498741, + "sha256": "f4c043fa5f4e10f9d86adfcbd775b14334fa3a075acbd1cd55fd6c40767be5ab" + }, + "evaluation/funded_allocations_with_outcomes": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/funded_allocations_with_outcomes.parquet", + "bytes": 9184776, + "sha256": "923ee1326145ad9ed2e14f556bf9fe2de380dffbc6148ab296e5fb35fae6ca48" + }, + "evaluation/shared_frontier_allocations_with_outcomes": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/shared_frontier_allocations_with_outcomes.parquet", + "bytes": 12965316, + "sha256": "93da70a7c7d73655e4f62c1555e9f6e9e3b845f53363bf64a81f511d040703d1" + }, + "evaluation/aggregate_evaluation": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/aggregate_evaluation.parquet", + "bytes": 43595, + "sha256": "11d0eb671baeac24dbfe78b79962689eabe079ed0b42f7ae94163c9298aad4c0" + }, + "evaluation/paired_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/paired_sharp_contrasts.parquet", + "bytes": 25471150, + "sha256": "bf2ffeae1f9223dc2abf7d833cec6a3f2ed6776d3fed52e2b91467783ce7773d" + }, + "evaluation/comparator_envelopes": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/comparator_envelopes.parquet", + "bytes": 19052, + "sha256": "756b948f3c19de5f51cbeeba943010ef497b5f7647bded865cf6cfd81f9220e1" + }, + "evaluation/endpoint_resolution_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-15-v5/evaluation/endpoint_resolution_audit.parquet", + "bytes": 4278, + "sha256": "fe9544b2929f4ff5ba458cbe0e3b2c7632fd0385f0c8101eac2538a343a477a8" + }, + "two_ruler/outcome_free/allocations": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/frontier_funded_allocations.parquet", + "bytes": 12532305, + "sha256": "57301665dafe2e18cd6a7abe6338067fa99adb12d680401af8f0c9c7f7f7c26d" + }, + "two_ruler/outcome_free/endpoint_diagnostics": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/frontier_endpoint_diagnostics.parquet", + "bytes": 36907, + "sha256": "f60f4d92cbf55fac3d89747f16c065d019e68f21b963995a3f4efd32c3e6eb4d" + }, + "two_ruler/outcome_free/independent_validation": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/frontier_independent_solver_validation.parquet", + "bytes": 19044, + "sha256": "77f1aa085c9797746473bfb8f89db7d5947d00c47c9560723381d7a45babae43" + }, + "two_ruler/outcome_free/objective_optimum_diagnostics": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/objective_optimum_diagnostics.parquet", + "bytes": 13092, + "sha256": "cf5539a1ce2f57a44e02253202396262e0a6d1f5a01fc911b62d63e7eddc0cf7" + }, + "two_ruler/outcome_free/order_sensitivity": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/frontier_order_sensitivity.parquet", + "bytes": 28054, + "sha256": "6abc638c90e1df79aafd1f4b140713ff67cb6460a7364111682471275c9c1b1c" + }, + "two_ruler/outcome_free/solve_records": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-13-v1c/frontier/frontier_solve_records.parquet", + "bytes": 386546, + "sha256": "b44a1435bf501bcab4b8dfa6563ca1411ad4ec4912437053913c44d39354b487" + }, + "two_ruler/evaluation/evaluated_portfolios": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/evaluated_portfolios.parquet", + "bytes": 658119, + "sha256": "7cf4ecbb86d427852793bc2da08c3ac95fb123970b00784580b9995a1bec4464" + }, + "two_ruler/evaluation/joined_funded_allocations": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/joined_funded_allocations.parquet", + "bytes": 15289032, + "sha256": "2c95e0c8cec52f5be8d52084e7842f253e6a297a70cd6f413acde4065ec95d04" + }, + "two_ruler/evaluation/window_endpoint_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/window_endpoint_contrasts.parquet", + "bytes": 29942, + "sha256": "9460e557bf2b414e8306133b434e9911b5bc4046eda7906de008d76309ef4fd0" + }, + "two_ruler/evaluation/monthly_endpoint_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/monthly_endpoint_contrasts.parquet", + "bytes": 76167, + "sha256": "ca6a9e043f3242cd632c52b57a9233380855568442b302c2abb7de7546966cca" + }, + "two_ruler/evaluation/metric_direction_census": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/metric_direction_census.parquet", + "bytes": 7939, + "sha256": "00c1a58b169792f54799a911b77d4879940876eec0b1c9502538e985473f1d00" + }, + "two_ruler/evaluation/outcome_join_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-normalized-objective-frontier-2026-07-15-v5/evaluation/outcome_join_audit.parquet", + "bytes": 3988, + "sha256": "4884c29e5edcae71abd33d2ecb27effd1cc8d048dfa247f54ab5cc28caf4d375" + }, + "credit_controls/outcome_free/scores": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/scores.parquet", + "bytes": 30407406, + "sha256": "5795bc0a75be90e86d37cf7d297f4b4fd6e6604b38f8179bc5042c024a53a8dc" + }, + "credit_controls/outcome_free/recipes": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/residual_recipes.json", + "bytes": 187689, + "sha256": "969ecbefe46bec4893a03be57385eda29b33dd291d73e7c0120f6d488a9e9936" + }, + "credit_controls/outcome_free/fit_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/residual_fit_audit.parquet", + "bytes": 45955332, + "sha256": "396c30d9bec7d222220cfe6f9870ab4994cf5c33e6da8c9e4ebbd99153155353" + }, + "credit_controls/outcome_free/outcome_free_geometry": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/outcome_free_geometry.parquet", + "bytes": 362075, + "sha256": "7264ef5b076d62ec642c541abd376149a98c4ec4d33250454bce9c971716893b" + }, + "credit_controls/outcome_free/feature_variation": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/active_feature_variation.parquet", + "bytes": 11844, + "sha256": "9f7e6a6b59887f9b8a41d5168897f5d57e38eea17f6c7d43e48cf151595ef0e6" + }, + "credit_controls/outcome_free/score_psi": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/score_psi.parquet", + "bytes": 6933, + "sha256": "eb5e392446f0b9939de80b8ff2c7d78fe6a6a60b3f450e04dab6d51e8879bae1" + }, + "credit_controls/outcome_free/scorecard_feature_psi": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/scorecard_feature_psi.parquet", + "bytes": 8216, + "sha256": "7293c44c9b54a966ac86769cec377d45e776a4568db6eed6e1d1cbc79dbeab58" + }, + "credit_controls/outcome_free/woe_summary": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/woe_summary.parquet", + "bytes": 7442, + "sha256": "0366397f5d94e277366c3fa2d53fd7a4d654b6ad6aec90a53d63fcef765a6584" + }, + "credit_controls/outcome_free/woe_coefficients": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/woe_coefficients.parquet", + "bytes": 3923, + "sha256": "a58b2b4d8c7de5ce03fbfc8fdcd5909d7a279cd6d9a41a35306f085aad199d76" + }, + "credit_controls/outcome_free/woe_binning_table": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/diagnostics/woe_binning_table.parquet", + "bytes": 20130, + "sha256": "8506032f4155a899ac16810f834f412ef40e8438307ef76e4e1b4af6fff82a47" + }, + "credit_controls/models/catboost": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/catboost_seed42.cbm", + "bytes": 836640, + "sha256": "26d658e64f1b7192fe74481e45cfa2af838798887d3e5360ebd466f5fe151ec5" + }, + "credit_controls/models/catboost_platt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/catboost_platt.pkl", + "bytes": 678, + "sha256": "9391c1c918e0707f61ae59c433c312f38668661edd90af5a44cfb29588a07594" + }, + "credit_controls/models/numeric_logistic": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/numeric_logistic.pkl", + "bytes": 3799, + "sha256": "3aef902d4ac475942f39ba93e4447bf2ee5653175c1b06c0511949ac684fd806" + }, + "credit_controls/models/numeric_logistic_platt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/numeric_logistic_platt.pkl", + "bytes": 678, + "sha256": "c3b1c562d75dc65f14bb53201f221ca390a07703cb3805533cd69cd337ac9421" + }, + "credit_controls/models/catboost_monotonic": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/catboost_monotonic_seed42.cbm", + "bytes": 883944, + "sha256": "3808b60815633c95fcaa19345863168730ad533bce01c168d1d981f2798bd4e6" + }, + "credit_controls/models/catboost_monotonic_platt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/catboost_monotonic_platt.pkl", + "bytes": 678, + "sha256": "93e38202f47fd06665ccb0e5d7e3ed996c71976a5b38d8acd00b09b3571b49ce" + }, + "credit_controls/models/woe_scorecard_platform": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/woe_scorecard_platform.pkl", + "bytes": 48526, + "sha256": "525946161cb2c67454b99ee611d7e62d2c9b0fe7667563b8adbe591a8973745e" + }, + "credit_controls/models/woe_scorecard_platform_platt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/woe_scorecard_platform_platt.pkl", + "bytes": 678, + "sha256": "924522ee4d26cdf663b402f52dd8319809265a459b69e457c38fdd742039cf2d" + }, + "credit_controls/models/woe_scorecard_borrower": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/woe_scorecard_borrower.pkl", + "bytes": 34904, + "sha256": "2b8aef2bdd4d1c9353357376ea23b26dbf230c976778d026d9f99ccb418340ea" + }, + "credit_controls/models/woe_scorecard_borrower_platt": { + "path": "models/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-13-v1b/prediction/woe_scorecard_borrower_platt.pkl", + "bytes": 678, + "sha256": "66ab15c9cde1f94ba444aaac24d8c52695016db76f111064a1305b626bfbe2ba" + }, + "credit_controls/evaluation/temporal_coverage": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/evaluation/temporal_coverage.parquet", + "bytes": 502111, + "sha256": "968872d3dd119cf89cebceb969d4d155fb3a75e010026e023ae10d1cede9e58c" + }, + "credit_controls/evaluation/prediction_metrics": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/evaluation/prediction_metrics.parquet", + "bytes": 14998, + "sha256": "8848cdffbaa01ff211976a13b2820a42d9ce8d93b4c5c7e3bfd90da3de517768" + }, + "credit_controls/evaluation/endpoint_resolution_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-credit-risk-controls-2026-07-15-v5/evaluation/endpoint_resolution_audit.parquet", + "bytes": 4278, + "sha256": "fe9544b2929f4ff5ba458cbe0e3b2c7632fd0385f0c8101eac2538a343a477a8" + }, + "raw_data_audit/archive_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/archive_inventory.csv", + "bytes": 400, + "sha256": "b6788da7d4d33c43f34b5bed5ca944a9ff4d349983df041bbc9eef8fd971d6e2" + }, + "raw_data_audit/status_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/status_inventory.csv", + "bytes": 470, + "sha256": "8d81783e3854efcf5ccd88f3342984cb44ed6d0cbef774b3bfab3382b4220845" + }, + "raw_data_audit/cohort_inventory": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cohort_inventory.csv", + "bytes": 194, + "sha256": "c5bd8d225a20c53d5865473a1550e2b0b506504f4d6e777b6d4ba28d3da645bb" + }, + "raw_data_audit/feature_coverage": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_coverage.csv", + "bytes": 67423, + "sha256": "c00b84d14e26cbba9b46f96fc22cba4f52565f3bc361b5ab24596b80cbc12e04" + }, + "raw_data_audit/feature_contract": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/raw_feature_contract.csv", + "bytes": 21744, + "sha256": "9444486e9139b5a2b5af53d0bd153f5153d618fceea95e9831f79f8d01477d97" + }, + "raw_data_audit/amount_alignment": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/loan_amount_alignment.csv", + "bytes": 729, + "sha256": "d0b2d8f6198e35b1d96739175c3d40f75584f090c67496a23295a853abbc4824" + }, + "raw_data_audit/cutoff_label_availability": { + "path": "reports/crpto/data_audit/ijds-raw-data-contract-2026-07-14-v2/cutoff_label_availability.csv", + "bytes": 287, + "sha256": "a1463a347603fe4d6a72d69c797810e0ab72ae8ffe36258988215e8ffb35f689" + }, + "endpoint_availability_sensitivity/endpoint_census": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/endpoint_census.parquet", + "bytes": 5381, + "sha256": "0bba3e50961bd5d0293da57b4e34e06f9f0271e1119e6d0b31e891fa07068669" + }, + "endpoint_availability_sensitivity/coverage_cells": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/coverage_cells.parquet", + "bytes": 49351, + "sha256": "2dc769f8fc350dbc961e9249fddfb7d97ffc47971ce7c65ca010532547054505" + }, + "endpoint_availability_sensitivity/coverage_summary": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/coverage_summary.parquet", + "bytes": 5148, + "sha256": "cd88d0cb8d76629d7b4216ced0d6e4f99f2d08303467948da73ad35927d1830e" + }, + "endpoint_availability_sensitivity/two_ruler_window_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/two_ruler_window_contrasts.parquet", + "bytes": 29560, + "sha256": "301a74915340de77a0e3479bb0096f00d07815fec27ac038af2d2a13658e5836" + }, + "endpoint_availability_sensitivity/two_ruler_directions": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/two_ruler_directions.parquet", + "bytes": 10957, + "sha256": "32eda9aeb6c7375794dff6e9b9b5b81ef6fae6cbc557e6a9182f657f67ac68fa" + }, + "endpoint_availability_sensitivity/two_ruler_direction_census": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/two_ruler_direction_census.parquet", + "bytes": 3209, + "sha256": "6826a0caff6d227ab7c9851f9f7400066796cb6e888060f9c5a2099e0750ad89" + }, + "endpoint_availability_sensitivity/exact_support_envelopes": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/exact_support_envelopes.parquet", + "bytes": 36417, + "sha256": "3c8dd8da77beb8f8cc7c6e394c8d85fce52fea7078fc86077a00a276068aecda" + }, + "endpoint_availability_sensitivity/exact_support_census": { + "path": "data/processed/experiments/ijds_audit/ijds-endpoint-availability-sensitivity-2026-07-14-v1/evaluation/exact_support_census.parquet", + "bytes": 3887, + "sha256": "11cfc06b80a967c72aafbfc944bc835176a0b94986d920724179497513fa29f9" + }, + "portfolio_structure_sensitivity/scenario_summary": { + "path": "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/evaluation/scenario_summary.parquet", + "bytes": 34408, + "sha256": "fcaa31ab27e69b8b121d443c63402a28b47dbb2641806a1f7dbc78c6d43d2ad4" + }, + "portfolio_structure_sensitivity/allocation_activity": { + "path": "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/evaluation/allocation_activity.parquet", + "bytes": 17090, + "sha256": "5b7b84bdc3ec8f8db5a85192936c10a20afbdf044fecf7eebf17769d427acb1f" + }, + "portfolio_structure_sensitivity/window_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/evaluation/window_contrasts.parquet", + "bytes": 212637, + "sha256": "ad6c3c9dad37301cb20bea931b41c36f86466f3df30ca0af67e14e08d365bdb6" + }, + "portfolio_structure_sensitivity/metric_directions": { + "path": "data/processed/experiments/ijds_audit/ijds-portfolio-structure-sensitivity-2026-07-15-v6/evaluation/metric_directions.parquet", + "bytes": 104159, + "sha256": "fdbee01642869671bfd27071d22374735504d0e243406ac3fe5d67b2b83bd885" + }, + "rolling_origin/temporal_coverage": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/temporal_coverage.parquet", + "bytes": 214187, + "sha256": "47096ab92be201827efdcb330aea77b65b6c31233fd11b5127d6202c4cd087bf" + }, + "rolling_origin/monthly_evaluation": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/monthly_evaluation.parquet", + "bytes": 181455, + "sha256": "89444bfdd650cf2ee9d862333cdd6064ac0ea94b13420ba60b4b2cdc0256a049" + }, + "rolling_origin/funded_allocations_with_outcomes": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/funded_allocations_with_outcomes.parquet", + "bytes": 2572605, + "sha256": "0908613e2d08f45fcdc1eadafb0ad9886e1036def54a2e922df8acca8284b9f7" + }, + "rolling_origin/shared_frontier_allocations_with_outcomes": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/shared_frontier_allocations_with_outcomes.parquet", + "bytes": 618802, + "sha256": "3aff256e6f7cc1587ae66fcf19a043b90d9d0514df01faf1f92d3984c0e74613" + }, + "rolling_origin/aggregate_evaluation": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/aggregate_evaluation.parquet", + "bytes": 42643, + "sha256": "a60eefd0670126d404875f4647226e49f6ecc0f710b9456405d7b64f7ce04379" + }, + "rolling_origin/paired_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/paired_sharp_contrasts.parquet", + "bytes": 4942885, + "sha256": "2fb92b57bff0fcdd15d6a76e139ed26367b2752bd55c1655203f1cab167fbe03" + }, + "rolling_origin/comparator_envelopes": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/comparator_envelopes.parquet", + "bytes": 17884, + "sha256": "7f276896a24ccbcf258d8a1532a87929cdc51c57a9ff98f1243b708e9e1eea00" + }, + "rolling_origin/endpoint_resolution_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-rolling-origin-2017-2026-07-15-v4/evaluation/endpoint_resolution_audit.parquet", + "bytes": 4275, + "sha256": "8ad75f1397125ab0648eacce04471f85008f72393eb095670647f36eb9ea3f1b" + }, + "missingness_encoding/evaluation/temporal_coverage": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/evaluation/temporal_coverage.parquet", + "bytes": 28313, + "sha256": "a44ef4a5af35ec55c969b1a856687f6bab3ac31e61c643b663028aebee83509c" + }, + "missingness_encoding/evaluation/prediction_metrics": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/evaluation/prediction_metrics.parquet", + "bytes": 11641, + "sha256": "8d08584b605c1398acfa7a47c86f026b3e661b5114f1d9635292c2bec9265f3e" + }, + "missingness_encoding/evaluation/endpoint_resolution_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/evaluation/endpoint_resolution_audit.parquet", + "bytes": 3924, + "sha256": "78542f4eb7eb6d851e8f14177b9f626959d8cb2be47b1c18e673b231fd286df4" + }, + "missingness_encoding/outcome_free/scores": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/scores.parquet", + "bytes": 19610910, + "sha256": "25d23f163ade1f13e5677c0df08306eec7ffbea47915bc6fd2f035239170a7d3" + }, + "missingness_encoding/outcome_free/recipes": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/residual_recipes.json", + "bytes": 112610, + "sha256": "3f2977009a4abb619dac8de0c1b10c0902bbb8c8bc7ebb5cce50ab4281696178" + }, + "missingness_encoding/outcome_free/fit_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/residual_fit_audit.parquet", + "bytes": 27709888, + "sha256": "2e768411de2ddc0c4afdc04e0efc513ba91198ea130c7d422faf76d03f3a9583" + }, + "missingness_encoding/outcome_free/missingness_census": { + "path": "data/processed/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/data/missingness_census.parquet", + "bytes": 3813, + "sha256": "687e7b9df0765e97fe55ea065a00f3ab9d69e35d42a8f1d177bf45a0e9a02a3c" + }, + "missingness_encoding/models/catboost_missing_indicators": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/catboost_missing_indicators.cbm", + "bytes": 811196, + "sha256": "a3e28362366164c35d1358946e2d8206edbdd34127924550a02647a2105b5637" + }, + "missingness_encoding/models/catboost_missing_indicators_platt": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/catboost_missing_indicators_platt.pkl", + "bytes": 678, + "sha256": "9ed10e1a78767c437c9f47750688cd54610d4a3fdfa009f77150ad32a1f0e272" + }, + "missingness_encoding/models/catboost_native_missing": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/catboost_native_missing.cbm", + "bytes": 811676, + "sha256": "25a69b32a60c01f19e4e49353ab32f75e825f041cd0bedaeef0981fcd81b63c8" + }, + "missingness_encoding/models/catboost_native_missing_platt": { + "path": "models/experiments/ijds_audit/ijds-missingness-sensitivity-2026-07-15-v3/prediction/catboost_native_missing_platt.pkl", + "bytes": 678, + "sha256": "8cc9523794408e67eea61daa5a1e7e7e8da590cbdff78c073e44aa405f5e84bc" + }, + "fit_label_completion/outcome_free/scores": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/outcome_free/scores.parquet", + "bytes": 25009623, + "sha256": "4b27af315ed986ab81466ca634239bf9f9b7bf3bcfa3b594fe89deba98cf4569" + }, + "fit_label_completion/outcome_free/fit_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/outcome_free/fit_audit.parquet", + "bytes": 11441868, + "sha256": "576d554f748587d53b4769a5f049923ac454b32aaba1c046b99de486adebfe60" + }, + "fit_label_completion/outcome_free/scenario_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/outcome_free/scenario_audit.parquet", + "bytes": 6674, + "sha256": "eb6968fa28a1a579d915c3635896e762ff19bb97d5a6780d751d42f1f0c551f0" + }, + "fit_label_completion/outcome_free/recipes": { + "path": "models/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/outcome_free/residual_recipes.json", + "bytes": 40364, + "sha256": "22e300f9e5ad3fddb94f888e8b82c9d89c4a93ade11f0d5f9dd8ce2910f99a13" + }, + "fit_label_completion/evaluation/coverage": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/evaluation/temporal_coverage.parquet", + "bytes": 35483, + "sha256": "67b716cc89e2dc5ed76c33fb9536d94f746b4df1b4eecf9d80b0d4e6ce609750" + }, + "fit_label_completion/evaluation/summary_table": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/evaluation/scenario_summary.parquet", + "bytes": 5903, + "sha256": "97bec811a433f14c4ee652474206b5cdd2d969a6fdcddbbefc45c73077382194" + }, + "fit_label_completion/evaluation/phase_stratum": { + "path": "data/processed/experiments/ijds_audit/ijds-fit-label-completion-sensitivity-2026-07-16-v2/evaluation/phase_stratum.parquet", + "bytes": 30003, + "sha256": "a6b9f17fde90057aea320f809e22176124fc8b1ca502bdbd5d3ac2d76594a81a" + }, + "allocation_granularity/outcome_free/rounded_allocations": { + "path": "data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/outcome_free/rounded_allocations.parquet", + "bytes": 3209311, + "sha256": "16b1e6496f86c3ea6f5cab7d6e32c68601cbcc448ab1fccfb5e83f75c610b7a2" + }, + "allocation_granularity/outcome_free/rounded_solve_records": { + "path": "data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/outcome_free/rounded_solve_records.parquet", + "bytes": 138352, + "sha256": "d70ca566875ee369e691a32ceec5e2096a035036142aabdd80df369ab373702c" + }, + "allocation_granularity/outcome_free/granularity_audit": { + "path": "data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/outcome_free/granularity_audit.parquet", + "bytes": 11648, + "sha256": "d932ee027921585f68af6fc7caa56d1ec059435dd484f2d3f04debb6371a2cea" + }, + "allocation_granularity/evaluation/granularity_contrasts": { + "path": "data/processed/experiments/ijds_audit/ijds-allocation-granularity-sensitivity-2026-07-16-v3/evaluation/granularity_contrasts.parquet", + "bytes": 33656, + "sha256": "12a91dbc69b4f666a16a6fe51524f13fb192a1122f8eab2f40fe38b50cd191a4" + } + }, + "paper_artifacts": { + "table/coverage": { + "path": "reports/crpto/tables/crpto_ijds_v4_table1_coverage_windows.csv", + "bytes": 9210, + "sha256": "4d6b72d01ca8074bf8463ebc3e9bd2d22aae0f89e7c41625df4df708e7453196" + }, + "table/phase_transition": { + "path": "reports/crpto/tables/crpto_ijds_v4_table2_phase_transition.csv", + "bytes": 1429, + "sha256": "da1c61c4c7c529a061d35711b6e6d3bd701a2509a3c4493b5555768426b6cb00" + }, + "table/development_envelopes": { + "path": "reports/crpto/tables/crpto_ijds_v4_table3_development_envelopes.csv", + "bytes": 32236, + "sha256": "949138cd932607d75b746dbc721879c03334ad825742e9f129766d912d4da52b" + }, + "table/direction_summary": { + "path": "reports/crpto/tables/crpto_ijds_v4_table4_direction_summary.csv", + "bytes": 204, + "sha256": "e92abbfb42d625db7111b43465e6fda6b064684aedced85d492a5907cd59d610" + }, + "table/two_ruler_tracks": { + "path": "reports/crpto/tables/crpto_ijds_v4_table5_two_ruler_tracks.csv", + "bytes": 2874, + "sha256": "f6b31b94345041fc2df644795738ab7a06f0578a4b95a5d5eb4d71b0c7de4865" + }, + "table/named_comparators": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS1_named_comparators.csv", + "bytes": 397, + "sha256": "c1575fb5b09ac56c358b67e85b8f2a9084cdebed83516f3ebf890a08aef8d9d9" + }, + "table/credit_controls": { + "path": "reports/crpto/tables/crpto_ijds_v4_table6_credit_controls.csv", + "bytes": 1883, + "sha256": "7c978c4e893ad16f8f3a99cd8a139066819dabbb43cd2e9e00507379487632f7" + }, + "table/credit_prediction_metrics": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS2_credit_prediction_metrics.csv", + "bytes": 9649, + "sha256": "52ac8e3c9d1707ba596e9cb3e6b66cdb41081104eb674c8b6c76ef980bfe83f0" + }, + "table/woe_iv_psi": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS3_woe_iv_psi.csv", + "bytes": 7731, + "sha256": "b5fef8582e2c181fc678642db839019e34c0ca0262573bca08151b2313e15793" + }, + "table/score_psi": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS4_score_psi.csv", + "bytes": 8181, + "sha256": "e26e44608a95f83fd67777c3fa438c3cb5f2fdde22f4392c23e4ef2dd23463e7" + }, + "table/label_lag_sensitivity": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS5_label_lag_sensitivity.csv", + "bytes": 6365, + "sha256": "7488224317a6e06af77dff5203261cabc6b73a7fda6b294cd88b2be868be352d" + }, + "table/endpoint_availability_sensitivity": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv", + "bytes": 2886, + "sha256": "df753518b2865dc034601f5bc14ac0ba65c01331eaf7450af14703881b9a3d8e" + }, + "table/portfolio_structure_sensitivity": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv", + "bytes": 4384, + "sha256": "664347e71a723c44388f458e88a1af80c45bcc774c0d16af058b3629eca6f8f6" + }, + "table/endpoint_resolution": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS8_endpoint_resolution.csv", + "bytes": 367, + "sha256": "435e313836f3c703242fad44d28aa1dc31a943d3bcbf6e3c3d4974543e2ff732" + }, + "table/missingness_encoding": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv", + "bytes": 666, + "sha256": "abc9c7eea485ca61c0914a7d7992b9061e6b7dab9bba80b29f88453a99b17d15" + }, + "table/rolling_origin": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv", + "bytes": 2105, + "sha256": "5d675c4d65567e31b1d41db8947eb369a65b14b08aa21f5963c2559ce260de5e" + }, + "table/fit_label_completion": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS11_fit_label_completion.csv", + "bytes": 625, + "sha256": "52f73ba5498d4307cc111d2291a4446247e8f1c919523fe914130610f815cbe8" + }, + "table/allocation_granularity": { + "path": "reports/crpto/tables/crpto_ijds_v4_tableS12_allocation_granularity.csv", + "bytes": 418, + "sha256": "a2fb823bb0f93ddda5f80f172f52acd0acb2e2788ccfa58e4d6de85e2d3ed183" + }, + "figure/coverage/png": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig1_coverage.png", + "bytes": 142518, + "sha256": "aff4aac92a7a04a9df2771a28c219fb140633ada0fa48d47df1f3c2ac932e174" + }, + "figure/coverage/pdf": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig1_coverage.pdf", + "bytes": 22772, + "sha256": "6c4725454f98f0f0097fdd8ee24d9957f0167c00572e19f00acdaf3de3192e2e" + }, + "figure/phase_transition/png": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.png", + "bytes": 163281, + "sha256": "605cbe30d35247cd2f9f06277f9b5ea22ad02e13846251a076b9c8721f41164f" + }, + "figure/phase_transition/pdf": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig2_phase_transition.pdf", + "bytes": 20005, + "sha256": "894ba508f5e8ffba52baeb2639fafd07c5d92c2eb6b712bf1bf9ab16c3fb4b5c" + }, + "figure/development_envelopes/png": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.png", + "bytes": 197839, + "sha256": "47b985822749585ec5540c37ad7d66c914f9901d023704c67ccb5ca021d8caa8" + }, + "figure/development_envelopes/pdf": { + "path": "reports/crpto/figures/crpto_ijds_v4_fig3_envelopes.pdf", + "bytes": 22715, + "sha256": "a5428ae8c537528a06d902ded93ba0094eadfe73ff7230a23214159f980ff5ca" + } + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + "claim_ledger": { + "schema_version": "2026-07-15.3", + "status": "active_ijds_claim_contract", + "numeric_authority": "parent_evidence_manifest_only", + "contract": { + "path": "configs/ijds_claim_ledger.yaml", + "bytes": 10976, + "sha256": "b5fe8e07f8663bf9e213fc3e0e012206201a34f92c92ed2cd6a32ceedbbe1f10" + }, + "claims": [ + { + "id": "data.exhaustive_status_independent_population", + "status": "active", + "kind": "empirical", + "lineages": [ + "raw_data_audit", + "binary_geometry" + ], + "scope": "declared_36_month_temporal_design", + "rule": "equals", + "result_pointer": "/data_contract/sampling", + "result": "none_all_eligible_rows_within_each_declared_temporal_role", + "forbidden_inference": [ + "raw_archive_census_is_one_common_estimand", + "external_validity" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "endpoint.not_verified_snapshot", + "status": "active", + "kind": "boundary", + "lineages": [ + "raw_data_audit", + "binary_geometry.evaluation" + ], + "scope": "reconstructed_status_availability_at_2020_09_30", + "rule": "equals", + "result_pointer": "/design/archive_is_verified_point_in_time_snapshot", + "result": false, + "forbidden_inference": [ + "verified_point_in_time_snapshot", + "observed_operational_chargeoff_date" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "endpoint.reason_census_partitions_candidates", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.evaluation" + ], + "scope": "primary_oot_reconstructed_endpoint_reason_taxonomy", + "rule": "equals", + "result_pointer": "/evaluation_endpoint/reason_census_partitions_primary_candidates", + "result": true, + "forbidden_inference": [ + "identified_missingness_mechanism", + "observed_operational_event_dates" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "coverage.five_models_all_windows_below_nominal", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.evaluation", + "credit_controls.evaluation" + ], + "scope": "all_candidate_sharp_bounds_under_six_month_endpoint", + "rule": "equals", + "result_pointer": "/credit_risk_controls/all_five_all_eight_upper_below_nominal", + "result": true, + "forbidden_inference": [ + "universal_conformal_invalidity", + "selected_model_winner", + "funded_set_validity" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "geometry.prevalence_sensitive_mechanism", + "status": "active", + "kind": "theorem", + "lineages": [ + "binary_geometry.outcome_free", + "label_lag_sensitivity" + ], + "scope": "constant_score_binary_threshold_and_observed_stratum_path", + "rule": "documented", + "result_pointer": null, + "result": null, + "forbidden_inference": [ + "finite_sample_causal_explanation", + "universal_threshold_transition" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "timing.fit_label_crossing_retained", + "status": "active", + "kind": "empirical", + "lineages": [ + "label_lag_sensitivity" + ], + "scope": "lags_passing_locked_monthly_retention_rule", + "rule": "equals", + "result_pointer": "/binary_phase_transition/label_lag_sensitivity/w7_to_w8_threshold_crossing_at_all_admissible_lags", + "result": true, + "forbidden_inference": [ + "arbitrary_label_lag_invariance", + "causal_attribution" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "timing.endpoint_six_month_reconciles_active", + "status": "active", + "kind": "empirical", + "lineages": [ + "endpoint_availability", + "binary_geometry.evaluation", + "two_ruler.evaluation" + ], + "scope": "complete_endpoint_availability_lag_grid", + "rule": "equals", + "result_pointer": "/sensitivity/evaluation_endpoint_availability/six_month_endpoint_reconciles_to_active_evaluation", + "result": true, + "forbidden_inference": [ + "endpoint_selection", + "independent_replications" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "sensitivity.second_origin_coverage_recurrence", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.evaluation", + "rolling_origin" + ], + "scope": "two_retrospective_origins_eight_windows_each", + "rule": "equals", + "result_pointer": "/sensitivity/rolling_origin/all_sixteen_upper_below_nominal", + "result": true, + "forbidden_inference": [ + "independent_replication", + "temporal_invariance", + "prospective_validation" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "sensitivity.missingness_encoding_recurrence", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.evaluation", + "missingness_encoding" + ], + "scope": "three_feature_semantics_preserving_catboost_encodings", + "rule": "equals", + "result_pointer": "/sensitivity/missingness_encoding/all_three_all_eight_upper_below_nominal", + "result": true, + "forbidden_inference": [ + "identified_missingness_mechanism", + "encoding_winner", + "portfolio_robustness" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "sensitivity.fit_label_completion_coverage_recurrence", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.evaluation", + "fit_label_completion" + ], + "scope": "observed_only_plus_three_declared_fit_label_completion_rules", + "rule": "equals", + "result_pointer": "/sensitivity/fit_label_completion/all_scenarios_all_windows_upper_below_nominal", + "result": true, + "forbidden_inference": [ + "sharp_completion_bounds", + "arbitrary_label_assignment_robustness", + "preregistration" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "geometry.fit_label_completion_crossing_not_universal", + "status": "active", + "kind": "boundary", + "lineages": [ + "binary_geometry.outcome_free", + "fit_label_completion" + ], + "scope": "stratum_2_w7_w8_path_over_four_fit_label_scenarios", + "rule": "equals", + "result_pointer": "/sensitivity/fit_label_completion/w7_w8_crossing_in_all_scenarios", + "result": false, + "forbidden_inference": [ + "universal_threshold_transition", + "causal_explanation", + "geometry_invariance" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "timing.fit_and_endpoint_lags_not_factorial", + "status": "active", + "kind": "boundary", + "lineages": [ + "label_lag_sensitivity", + "endpoint_availability" + ], + "scope": "two_separate_one_factor_sensitivities", + "rule": "equals", + "result_pointer": "/sensitivity/evaluation_endpoint_availability/fit_label_lag_crossed_factorially", + "result": false, + "forbidden_inference": [ + "joint_lag_robustness", + "full_factorial_robustness" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "decision.no_selected_policy", + "status": "active", + "kind": "boundary", + "lineages": [ + "two_ruler.outcome_free", + "two_ruler.evaluation", + "binary_geometry.evaluation" + ], + "scope": "all_reported_rulers_coordinates_windows_and_supports", + "rule": "equals", + "result_pointer": "/decision_challenger/interpretation/policy_winner", + "result": null, + "forbidden_inference": [ + "policy_winner", + "preferred_ruler", + "preferred_coordinate", + "universal_direction" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "comparator.broad_support_all_cross_zero", + "status": "active", + "kind": "empirical", + "lineages": [ + "binary_geometry.outcome_free", + "binary_geometry.evaluation" + ], + "scope": "declared_point_cap_support_0_05_to_0_12", + "rule": "equals", + "result_pointer": "/portfolio/broad_stress_all_envelopes_cross_zero", + "result": true, + "forbidden_inference": [ + "universal_comparator_quantification", + "policy_equivalence" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "theory.sharp_common_outcome_bounds", + "status": "active", + "kind": "theorem", + "lineages": [ + "binary_geometry.evaluation", + "two_ruler.evaluation" + ], + "scope": "binary_completion_bounds_for_candidates_and_paired_allocations", + "rule": "documented", + "result_pointer": null, + "result": null, + "forbidden_inference": [ + "sampling_confidence_interval", + "missing_at_random_identification" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "theory.binary_identification_width", + "status": "active", + "kind": "theorem", + "lineages": [ + "binary_geometry.evaluation", + "two_ruler.evaluation" + ], + "scope": "unrestricted_binary_completion_for_common_outcome_policy_contrasts", + "rule": "documented", + "result_pointer": null, + "result": null, + "forbidden_inference": [ + "missing_at_random_identification", + "sampling_uncertainty", + "causal_effect" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "theory.basis_endpoint_sufficiency", + "status": "active", + "kind": "theorem", + "lineages": [ + "binary_geometry.outcome_free", + "binary_geometry.evaluation" + ], + "scope": "declared_piecewise_linear_point_cap_support", + "rule": "documented", + "result_pointer": null, + "result": null, + "forbidden_inference": [ + "continuous_joint_frontier_uniqueness", + "arbitrary_comparator_support" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "boundary.no_selected_set_validity", + "status": "active", + "kind": "boundary", + "lineages": [ + "binary_geometry", + "two_ruler" + ], + "scope": "candidate_conformal_audit_followed_by_outcome_free_allocation", + "rule": "documented", + "result_pointer": null, + "result": null, + "forbidden_inference": [ + "selected_set_conformal_guarantee", + "post_selection_coverage" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "sensitivity.structure_no_universal_direction", + "status": "active", + "kind": "empirical", + "lineages": [ + "portfolio_structure", + "two_ruler.evaluation" + ], + "scope": "complete_budget_by_purpose_cap_by_lgd_cartesian_grid", + "rule": "equals", + "result_pointer": "/sensitivity/portfolio_structure/every_scenario_has_adverse_default_and_miscoverage_cells", + "result": true, + "forbidden_inference": [ + "scenario_winner", + "universal_adversity", + "structural_invariance", + "deployment" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + }, + { + "id": "optimization.allocation_granularity_is_diagnostic", + "status": "active", + "kind": "boundary", + "lineages": [ + "two_ruler.outcome_free", + "allocation_granularity" + ], + "scope": "deterministic_usd25_floor_with_residual_cash_on_active_allocations", + "rule": "equals", + "result_pointer": "/sensitivity/allocation_granularity/integer_policy_or_reoptimization_claim_authorized", + "result": false, + "forbidden_inference": [ + "integer_optimality", + "discrete_policy_robustness", + "reoptimization" + ], + "surfaces": { + "required": [ + "body", + "supplement", + "registry", + "claim_matrix" + ], + "allowed": [ + "body", + "supplement", + "registry", + "claim_matrix" + ] + } + } + ] + } +} diff --git a/reports/crpto/ijds_policy_support_tie_evidence.json b/reports/crpto/ijds_policy_support_tie_evidence.json new file mode 100644 index 0000000..d7aeccc --- /dev/null +++ b/reports/crpto/ijds_policy_support_tie_evidence.json @@ -0,0 +1,403 @@ +{ + "schema_version": "2026-07-12.1", + "status": "complete_prefreeze_structural_evidence", + "active_claim_status": "not_active_until_family_redesign_decision", + "run_tag": "ijds-policy-support-tie-audit-2026-07-12-v1", + "protocol_tag": "protocol/ijds-policy-support-tie-audit-2026-07-12-v1", + "protocol_commit": "115eaf1b81ed5f05ffe601e9c20079059c40c200", + "claim_boundary": "Outcome-free structural evidence only. No empirical metric, direction, policy promotion, universal comparator support, or selected-set claim.", + "outcome_columns_passed": [], + "results": { + "family": { + "rows": 3120, + "inherited_rows": 1872, + "inherited_infeasible": 0, + "inherited_decision_active": 1846, + "gamma_one_infeasible": 0, + "gamma_one_decision_active": 624, + "gamma_one_objective_slack": 0, + "classification_counts": [ + { + "role": "policy_development", + "gamma": 0.0, + "risk_tolerance": 0.15, + "cap_classification": "objective_slack", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.0, + "risk_tolerance": 0.17, + "cap_classification": "objective_slack", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.0, + "risk_tolerance": 0.19, + "cap_classification": "objective_slack", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 85 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.15, + "cap_classification": "objective_slack", + "cells": 3 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 80 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.17, + "cap_classification": "objective_slack", + "cells": 8 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 78 + }, + { + "role": "policy_development", + "gamma": 0.25, + "risk_tolerance": 0.19, + "cap_classification": "objective_slack", + "cells": 10 + }, + { + "role": "policy_development", + "gamma": 0.5, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.5, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.5, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.75, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.75, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 0.75, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 1.0, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 1.0, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "policy_development", + "gamma": 1.0, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 88 + }, + { + "role": "primary_oot", + "gamma": 0.0, + "risk_tolerance": 0.15, + "cap_classification": "objective_slack", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.0, + "risk_tolerance": 0.17, + "cap_classification": "objective_slack", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.0, + "risk_tolerance": 0.19, + "cap_classification": "objective_slack", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.25, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.25, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 119 + }, + { + "role": "primary_oot", + "gamma": 0.25, + "risk_tolerance": 0.17, + "cap_classification": "objective_slack", + "cells": 1 + }, + { + "role": "primary_oot", + "gamma": 0.25, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 116 + }, + { + "role": "primary_oot", + "gamma": 0.25, + "risk_tolerance": 0.19, + "cap_classification": "objective_slack", + "cells": 4 + }, + { + "role": "primary_oot", + "gamma": 0.5, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.5, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.5, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.75, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.75, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 0.75, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 1.0, + "risk_tolerance": 0.15, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 1.0, + "risk_tolerance": 0.17, + "cap_classification": "decision_active", + "cells": 120 + }, + { + "role": "primary_oot", + "gamma": 1.0, + "risk_tolerance": 0.19, + "cap_classification": "decision_active", + "cells": 120 + } + ], + "maximum_absolute_parent_score_difference": 2.220446049250313e-16, + "maximum_absolute_parent_objective_difference": 8.731149137020111e-11, + "gamma_zero_objective_slack": 624, + "inherited_slack": 26, + "gamma_one_minus_075_objective_mean": -2519.4381117036437, + "gamma_one_minus_075_objective_min": -8337.854696241615, + "gamma_one_minus_075_objective_max": -579.9765933432209 + }, + "point_cap_census": { + "rows": 7297, + "periods": 15, + "named_unique_cap_months": 2204, + "minimum_absolute_nonbasic_reduced_cost": 0.00038757301081204787, + "near_zero_bases": 0, + "primal_degenerate_bases": 2941, + "maximum_dual_sign_violation": 0.0, + "maximum_objective_reconciliation_error": 5.820766091346741e-10, + "support_sources": [ + { + "source": "broad_support_lower", + "cap_month_rows": 15, + "decision_active": 15, + "objective_slack": 0, + "near_zero_bases": 0, + "primal_degenerate_bases": 0 + }, + { + "source": "broad_support_upper", + "cap_month_rows": 15, + "decision_active": 0, + "objective_slack": 15, + "near_zero_bases": 0, + "primal_degenerate_bases": 0 + }, + { + "source": "development_support_lower", + "cap_month_rows": 1065, + "decision_active": 1065, + "objective_slack": 0, + "near_zero_bases": 0, + "primal_degenerate_bases": 14 + }, + { + "source": "development_support_upper", + "cap_month_rows": 1080, + "decision_active": 1074, + "objective_slack": 6, + "near_zero_bases": 0, + "primal_degenerate_bases": 0 + }, + { + "source": "named_c0", + "cap_month_rows": 45, + "decision_active": 0, + "objective_slack": 45, + "near_zero_bases": 0, + "primal_degenerate_bases": 0 + }, + { + "source": "named_c1", + "cap_month_rows": 1080, + "decision_active": 1079, + "objective_slack": 1, + "near_zero_bases": 0, + "primal_degenerate_bases": 7 + }, + { + "source": "named_c2", + "cap_month_rows": 1079, + "decision_active": 1075, + "objective_slack": 0, + "near_zero_bases": 0, + "primal_degenerate_bases": 4 + }, + { + "source": "period_basis_breakpoint", + "cap_month_rows": 2952, + "decision_active": 2922, + "objective_slack": 15, + "near_zero_bases": 0, + "primal_degenerate_bases": 2920 + } + ] + }, + "order_sensitivity": { + "triggered_rows": 2941, + "tie_sensitive_rows": 0, + "maximum_allocation_distance": 1.449778343358131e-14, + "maximum_absolute_objective_difference": 1.7171259969472885e-09 + } + }, + "source_summary": { + "path": "models/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/policy_support_tie_audit_summary.json", + "bytes": 18692, + "sha256": "f0b7a938d2e11671cb0216a35d02d3d3c00ec9c484064ed201e6a740178c29da" + }, + "source_artifacts": { + "models/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/protocol_freeze.json": { + "path": "models/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/protocol_freeze.json", + "bytes": 7105, + "sha256": "decc2f03b4fd6bdc358ab2b47c9d8c52c188def5c341fbc1da8e640cc982b79d" + }, + "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/policy_family_feasibility.parquet": { + "path": "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/policy_family_feasibility.parquet", + "bytes": 110455, + "sha256": "6127eac96d4d2860136473c812368ea7360db22d78b8d4487667028cab062538" + }, + "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/point_cap_basis_diagnostics.parquet": { + "path": "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/point_cap_basis_diagnostics.parquet", + "bytes": 428056, + "sha256": "809ce7a295c8794765439b82f329ef1cf8c5c1f37e789bd874f8b485288ae4e0" + }, + "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/point_cap_order_sensitivity.parquet": { + "path": "data/processed/experiments/ijds_audit/ijds-policy-support-tie-audit-2026-07-12-v1/point_cap_order_sensitivity.parquet", + "bytes": 64487, + "sha256": "d8b03aa46041af0346bc6a3c369621682f6157c96be490c4f9e00007f783d474" + } + }, + "tables": { + "family": { + "path": "reports/crpto/tables/crpto_ijds_policy_family_domain.csv", + "bytes": 4249, + "sha256": "fd6e1b9bcdc4560291410cc4e421c38dacf074cd78e3d8da2ee12eaf2fa9e027" + }, + "gamma_endpoint": { + "path": "reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv", + "bytes": 642, + "sha256": "da638fc1f9d684779e67739fd0b6237694f354fee724730f7b2aa166a36178cf" + }, + "comparator_support": { + "path": "reports/crpto/tables/crpto_ijds_comparator_support_domain.csv", + "bytes": 394, + "sha256": "081eec38e541a3fe64c8fdeb680d230a0dbe4b32093a00e2b22425814dbd3ac4" + } + }, + "protected_stages_run": [], + "protected_artifacts_written": [] +} diff --git a/reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv b/reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv deleted file mode 100644 index 27d068e..0000000 --- a/reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv +++ /dev/null @@ -1,15 +0,0 @@ -relative_path,caption_type,caption_index,caption_text,editorial_sink,why_useful,use_rule,claim_boundary -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,1,"Figure 1 compares the conventional CRO framework with our proposed method CRC, at the nominal",agenda extendida CRPTO/tesis robustness-certificate appendix,Shows the conceptual contrast between conventional CRO and CRC-style robustness control.,Use as a schematic inspiration only; do not reproduce the figure without permission.,"Future gate over V/violation, not current CRPTO evidence." -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,2,"Figure 2: Simulation results showing mean prediction interval coverage and width, averaged",agenda extendida CRPTO/tesis source/shift conformal appendix,Coverage-width caption is useful for explaining why non-exchangeability gates must report both validity and efficiency.,Use for assumptions table and reviewer response framing; no new run.,Requires declared weighting or shift structure. -supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf,figure,1,"Figure 1: Empirical coverages of conformal prediction intervals, computed using 5000 different random splits",agenda extendida CRPTO/tesis source/shift conformal appendix,Canonical weighted-conformal covariate-shift coverage caption for density-ratio caveats.,Use only to motivate weight-estimation assumptions.,No source-shift deployment claim without credible weights. -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,3,"Figure 3: [Lei and Candès, 2021]’s WCP coverage guarantee (3), compared to the new guarantee",agenda extendida CRPTO/tesis group/source governance appendix,Compares older weighted CP guarantees to the new group-weighted guarantee.,"Use as candidate-experiment design input, not as evidence of LC performance.",Needs target/source group weights before any run. -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,1,Figure 1: Illustration of the proposed robust algorithm for multi-source conformal prediction. Each θb represented by a,agenda extendida CRPTO/tesis multi-source conformal appendix,Illustrates multi-source calibration structure and source-combination logic.,Use for taxonomy only.,"LC source families are retrospective, not validated external sources." -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,1,Figure 1: Illustration of how proposed methods compare to the standard formulation of conformal prediction.,agenda extendida CRPTO/tesis utility-directed CP future gate,Contrasts standard CP with utility-directed CP in a decision-aware frame.,Use to specify future decision-loss screen.,No selector change without fixed loss and coverage gate. -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,4,Figure 4: Measured impact of base model accuracy on downstream optimization values on all losses considered,agenda extendida CRPTO/tesis utility-directed CP future gate,Connects base model accuracy to downstream optimization value.,Use as appendix motivation for utility reporting.,Not evidence that CRPTO currently optimizes utility-directed sets. -supplement/Guan 2023 - Localized Conformal Prediction.pdf,figure,1,Figure 1: Conformal bands (blue) and localized conformal bands (red) using regression score (dashed) and quantile,agenda extendida CRPTO/tesis localized conformal candidate appendix,Visualizes global versus localized conformal bands.,Use to frame a localized-vs-Mondrian challenger only if reviewer asks.,No current localized calibration design. -paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,figure,2,Figure 2: Performance comparison.,CRPTO/agenda extendida CRPTO/tesis applied robust-credit related work,Performance-comparison caption anchors the P2P robust portfolio context.,Use to motivate domain fit; do not treat as comparable champion evidence.,"Different data, objective and uncertainty construction." -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,figure,4,Figure 1 plots the Gini coefficient for the credit score and our model prediction over time.,agenda extendida CRPTO/tesis score/equity governance appendix,Gini-over-time caption is a useful template for score-vs-model governance reporting.,Use as dashboard/table design inspiration only.,Not a legal fair-lending claim. -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,1,Figure 1 gives an instructive example of possible group-specific effects when borrow-,agenda extendida CRPTO/tesis equity/noise governance appendix,Clarifies how better prediction technology can create group-specific effects.,Use to frame equity caveats in prose.,No protected-attribute causal claim in LC. -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,4,Table 6 shows that 46%-53% of the overall AUC difference is accounted for by the fact that disadvantaged,agenda extendida CRPTO/tesis data-quality/equity governance appendix,Links predictive-performance gaps to disadvantaged consumers and data quality.,Use as motivation for source/noise limitations.,Project lacks the paper's protected-group and lender data. -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,5,TABLE 3: CORRELATIONS BETWEEN PROXY PROBABILITY AND REPORTED RACE AND ETHNICITY,agenda extendida CRPTO/tesis fairness-proxy boundary appendix,Shows proxy-probability validation needs richer race/ethnicity methodology.,Use only to justify the stop rule on legal fair-lending claims.,LC lacks surname/fine geography and protected labels. -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,2,Figure 2: Main trial screen shown to participants for Ob-,agenda extendida CRPTO/tesis governance/committee communication appendix,Human-facing uncertainty display can inspire committee explanation design.,"Use for communication patterns, not as credit decision evidence.","Different task, subjects and utility function." diff --git a/reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv b/reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv deleted file mode 100644 index 445300d..0000000 --- a/reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv +++ /dev/null @@ -1,588 +0,0 @@ -relative_path,caption_type,caption_index,caption_text,caption_index_truncated -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,1,"Table 1 reports basic demographic information on our sample, including age, household",false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,2,Table 1: Descriptive Statistics,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,3,Table 2: Yti Frequency and Transitions,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,figure,4,Figure 1 plots the Gini coefficient for the credit score and our model prediction over time.,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,figure,5,"Figure 1: Gini Coefficient, Credit Score and Prediction Model",false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,6,Table 3: Credit Score Risk Profiles,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,7,Table 4: Credit Score and Model Risk Profile Comparison,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,8,Table 5 also reports our model-based prediction for the default rate of borrowers in,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,9,Table 5: Default Risk Variation by Credit Score Profile,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,10,Table 6: Feature Attribution Differences,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,11,Table 14 in Appendix C reports summary statistics on indicators of credit demand and,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,figure,12,Figure 5 in Appendix C displays the time effects by risk profile. The time variation,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,13,Table 7: Access to Credit by Risk Profile,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,table,14,Table 8 reports our baseline regression results for both the Experian (Panel A) and,false -paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,truncation_note,15,26 additional figure/table captions omitted from this compact index.,true -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,1,Figure 1: Prediction set examples on Imagenet. We show three progressively more difficult examples,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,2,Figure 2: Illustration of conformal prediction with matching Python code.,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,3,Figure 3: Python code for adaptive prediction sets.,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,4,"Figure 3 shows Python code to implement this method. As usual, these uncertainty sets (with tie-breaking)",false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,5,Figure 4: A visualization of the adaptive prediction sets algorithm in Eq. (3). Classes are included,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,6,Figure 5: Python code for conformalized quantile regression.,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,7,Figure 6: A visualization of the conformalized quantile regrssion algorithm in Eq. (4). We adjust,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,8,Figure 7: Python code for conformalized uncertainty scalars.,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,9,Figure 8: A visualization of the uncertainty scalars algorithm in Eq. (5). We produce the set by,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,10,Figure 9: A visualization of the conformalized Bayes algorithm in Eq. (6). The prediction set is a,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,11,"Figure 10: Prediction sets with various notions of coverage: no coverage, marginal coverage, or",false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,12,Figure 11: The distribution of coverage with an infinite validation set is plotted for different values of,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,table,13,Table 1: Calibration set size n() required for coverage slack  with δ = 0.1 and α = 0.1.,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,figure,14,Figure 12: Python code for computing coverage with efficient score caching. Notice that from the,false -paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,truncation_note,15,15 additional figure/table captions omitted from this compact index.,true -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,1,Figure 1: FNR control in tumor segmentation. The top figure shows examples of our procedure,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,2,Figure 2: FNR control on MS COCO. The top figure shows examples of our procedure with correct,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,3,Figure 3: Control of graph distance on hierarchical ImageNet. The top figure shows examples,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,4,Figure 4: F1-score control on Natural Questions. The top figure shows examples of our procedure,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,5,Figure 5: Comparison of RCPS/LTT with conformal risk control on the polyp dataset.,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,6,Figure 6 shows results.,false -paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,figure,7,Figure 6: Risk control results on a synthetic covariate shift dataset. Running standard conformal,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,1,Figure 1.1: The conformal score function determines the shape of the sets. The shaded band is a,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,2,Figure 2.1: Illustration of a permutation test for the equality of two real-valued distribu-,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,3,Figure 2.2 illustrates parts (iii) and (iv) of this last fact. When there are no ties between values,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,4,Figure 2.2: An illustration of two quantiles chosen on the CDF. The figure illustrates the,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,5,"Figure 2.3: This figure depicts a single time step of Pólya’s urn model, for the case c = 2. We begin",false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,6,Figure 3.1: Illustration of notation for a single hypothesized response y. This figure,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,7,Figure 3.2: An illustration of full conformal prediction with the residual score function. On the,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,table,8,Table 3.1: Comparison of full and split conformal methods.,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,9,"Figure 3.3: A visualization of the split conformal quantile q̂ (as defined in Algorithm 3.6),",false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,10,"Figure 3.2, is inspired by illustrations developed by Ryan Tibshirani in various lectures and tutorials.",false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,11,Figure 4.1: An illustration of marginal and conditional coverage. The interval on the left,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,12,Figure 4.2: The distribution of coverage of split conformal prediction conditional on the,false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,13,"Figure 4.3: Illustration of the event Eunif , which is defined in the proof of Theorem 4.3. Each plot",false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,figure,14,"Figure 4.4: Illustration of the hardness of conditional coverage, and the idea of the proof of",false -paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,truncation_note,15,31 additional figure/table captions omitted from this compact index.,true -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,1,"FIG. 1. Object detection with simultaneous distribution-free guarantees on recall, intersection-over-union",false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,2,Figure 2 gives a visual description of the object detection pipeline.,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,3,FIG. 2. Our detection pipeline annotated with the formal mathematical notation to aid the reader.,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,4,FIG. 3. Multilabel prediction set examples on MS COCO using fixed-sequence testing. Black classes are true,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,5,FIG. 4. Numerical results of our multilabel classification procedure. The risk is plotted as a violin plot over,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,6,FIG. 5. Numerical results of selective classification on ImageNet. The violins plot the selective error over 100,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,7,FIG. 6. Numerical results of selective regression on the MEPS dataset. The MSE is plotted as a violin plot over,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,8,FIG. 7. Numerical performance of methods for simultaneous OOD type-1 error and coverage control on CIFAR-,false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,9,"FIG. 8. Instance segmentation examples on MS COCO. The input images are on the left, and the output instance",false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,10,"FIG. 9. Numerical results for our instance segmentation algorithm with α1 = 0.5, α2 = 0.5, α3 = 0.25, and",false -paper/Angelopoulos et al 2025 - Learn Then Test.pdf,figure,11,Figure 9 are conservative. This is because we optimized the choice of λ̂ ∈ Λ,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,1,"Figure 1: A stylized example of risk-controlling prediction sets. Here, “P” gives the estimated",false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,2,Figure 2: Sample-splitting setup. The training data is used to fit a predictive model fˆ. The remaining,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,3,Figure 3: Visualization of UCB calibration.,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,4,Figure 4: Numerical evaluations of concentration results for bounded losses. We show the simple,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,5,Figure 5: Numerical evaluations of the PU bound. We compare the bound from (9) with the estimated,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,table,6,Table 1: Distributions considered for the unbounded case.,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,7,"Figure 6: Optimal prediction sets. In the special case where ρ̂x (y, S) does not depend on S, Tλ (x) from",false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,8,"Figure 6. (The negative on λ comes from the definition of nesting, which asks sets to grow as λ grows.)",false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,9,Figure 7: Prediction set results on Imagenet. The risk and set sizes for an RCPS are plotted as,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,10,Figure 8: Multi-label prediction set examples on MS COCO. Black classes are correctly identified,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,11,Figure 9: Multi-label prediction set results on MS COCO. The risk and set sizes are plotted as,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,12,Figure 10: Hierarchical predictions. We show randomly selected examples of hierarchical prediction sets,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,13,Figure 11: The risk and height of RCPS for hierarchical classification. We show histograms of,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,figure,14,Figure 12: Polyp segmentations. We show examples of polyps along with prediction sets that capture,false -paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,truncation_note,15,10 additional figure/table captions omitted from this compact index.,true -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,table,1,"Table 1 summarizes the a priori structural assumptions, hypothesis tests, and resulting uncertainty",false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,table,2,Table 1 Summary of data-driven uncertainty sets proposed in this paper.,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,3,"Figure 1 The left panel shows the sets Uχ and UG , α =  = 10%. When N = 0, the smallest set which implies a",false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,4,Figure 1 illustrates the sets Uχ and UG with a particular numerical example. The true distribution,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,5,Figure 1 illustrates an example. The true distribution is a standard normal whose cumulative,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,6,Figure 2 illustrates the sets UI and UF B numerically. The marginal distributions of P∗ are inde-,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,7,Figure 2 The left panel shows the marginal densities. The right panel shows UI (dashed black line) and UF B,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,table,8,"Table 2 Comparing Thresholds with and without bootstrap using NB = 10, 000",false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,9,"Figure 3 Comparing UM , ULCX , UCS and UDY for the example from Fig. 2,  = 10%, α = 20%. The left panel",false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,table,10,Table 3 Portfolio statistics for each of our methods. UDY and UCS ∩ supp(P∗ ) perform identically to UM .,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,11,Figure 4 Portolio performance by method: α =  = 10%. Top left: Cross-validation results. Top right: Out-of-,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,12,Fig. 4 shows the distribution of the 10% worst-case return. (Additional plots are also available in,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,13,Figure 5 The left panel shows various bounds on the median waiting time ( = .5) for n = 10 and various values,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,figure,14,Fig. 5. We have included the empirical CDF of the waiting time and the sampled version of the,false -paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,truncation_note,15,1 additional figure/table captions omitted from this compact index.,true -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,1,"Figure 1 A regression tree is trained on data (x1 , y 1 ), . . . , (x10 , y 10 ) and partitions the X data into regions",false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,2,"Figure 2 Performance of various prescriptions with respect to true distributions, averaged over samples and",false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,3,Figure 3 Performance of various prescriptions in the two-stage shipment planning with pricing problem.,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,4,"Figure 4 The coefficient of prescriptiveness P in the example from Section 1.1, measured out of sample. The",false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,5,"Figure 5 The percentage of all sales in the German state of Berlin taken up by each of 13 selected titles,",false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,6,Figure 6 Scatter plots of various data from IMDb and RT (horizontal axes) against total European sales during,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,7,Figure 7 Weekly search engine attention for two unnamed films in the world and in two populous German,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,8,Figure 9 The performance of our prescription over time. Blue vertical dashes indicate major release dates. The,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,9,Figure 10 The distribution of coefficients of prescriptiveness P over retail locations.,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,10,Figure 11 Network data for shipment planning example.,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,11,Figure 12 Results of the experiment in Section 13.2.,false -paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,figure,12,"Figure 13 The graph of actors, connected via common movies where both are first-billed. Colored nodes",false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,1,Figure 3 we compare Bounds 1 and 3 and the approximate bound for n = 2000. Bound 3 is identical,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,2,Figure 1: Comparison of probability bounds for n = |Ji | = 10.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,3,Figure 2: Comparison of probability bounds for n = |Ji | = 100.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,4,Figure 3: Comparison of probability bounds for n = |Ji | = 2000.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,table,5,Table 1: Choice of Γi as a function of n = |Ji | so that the probability of constraint violation is less than,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,6,Figure 4 illustrates the effect of the protection level on the objective function value. In the absence of,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,7,Figure 4: Optimal value of the robust knapsack formulation as a function of Γ.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,8,Figure 5: Optimal value of the robust knapsack formulation as a function of the probability bound of constraint,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,table,9,Table 2: Results of Robust Knapsack Solutions.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,10,Figure 6: The return and the objective function value (risk adjusted return) as a function of the protection level,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,11,Figure 7: The solution of the portfolio for various protection levels.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,12,Figure 8: Simulation study of the probability of underperforming the nominal return as a function of Γ.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,table,13,Table 3: Simulation results given by the robust solution.,false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,figure,14,"Figure 9: Empirical Result of expected, maximum and minimum yield.",false -paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,truncation_note,15,4 additional figure/table captions omitted from this compact index.,true -paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,figure,1,Figure 1: The curve of CV (h).,false -paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,figure,2,Figure 2: Performance comparison.,false -paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,table,3,Table 2: Rate of return from the optimal portfolio on the Prosper Table 3: Investors’ choices of input parameters for portfolio selec-,false -paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,table,4,Table 4: Investment performances of input parameters for portfolio selection.,false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,figure,1,"Figure 1: Features x, model predictions y, and policy z for the three experiments.",false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,figure,2,Figure 2: Inventory problem results for 10 runs over a representative instantiation of true parameters,false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,figure,3,Figure 4: Results for 10 runs of the generation-scheduling problem for representative decision,false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,figure,4,"Figure 3: 2-hidden-layer neural network to programming model that minimizes just the RMSE,",false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,figure,5,"Figure 4 shows the performance of the three models on the testing dataset. As expected, the RMSE",false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,table,6,"Table 1: Task loss results for 10 runs each of the battery storage problem, given a lithium-ion battery",false -paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,table,7,Table 1 shows the performance of the two models on the testing dataset. As energy prices are difficult,false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,1,Figure 1 Geometric Illustration of SPO Loss,false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,2,Figure 2 Difference between prediction and decision residuals,false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,3,Figure 3 Illustrative Example.,false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,4,"Figure 4 summarizes our findings, and note that the box plot for each configuration of the",false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,5,"Figure 4 Normalized test set SPO loss for the SPO+, least squares, absolute loss, and random forests methods on",false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,6,Figure 5 displays our results for this experiment. Generally we observe similar patterns,false -paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,figure,7,"Figure 5 Normalized test set SPO loss for the SPO+, least squares, absolute loss, and random forests methods on",false -paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,figure,1,Fig. 1. We estimate the investment return of L3 by computing the weights between L3 and baseline models for experimental comparison.,false -paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,figure,2,Fig. 2. Predicted rate of return (top row) and predicted risk (bottom row) versus probability of default for Lending Club (left column) and Prosper (right column).,false -paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,table,3,Table 2 shows the investment return rate on each testing subset 4 0.0236 0.0398 0.0481,false -paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,figure,4,Fig. 3. Sensitivity analysis.,false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,1,"Figure 1 compares the conventional CRO framework with our proposed method CRC, at the nominal",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,2,"Figure 1: Comparison of CRO and our method CRC. Portfolio optimization problem with ϕ(y, z) =",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,3,"Figure 2: The results of risk certificate, decision loss, robustness, and coverage on synthetic data",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,4,"Figure 3: The results of risk certificate, decision loss, robustness, and coverage on synthetic data",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,5,"Table 1: The results of risk certificate, decision loss, and robustness under nominal levels α = 0.1",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,6,"Figure 4: The results of risk certificate, decision loss, and robustness when varying nominal level α",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,7,"Figure 5: The risk certificate, decision loss and robustness when varying nominal level α on battery",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,8,Table 2: The results of different smoothing parameters sensitivity of CRC at the nominal level,false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,9,"Table 3: The results of lagrange multiplier update schedule of CRC at the nominal level α = 0.1,",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,10,Figure 6: The densities of risk certificate on synthetic data when α = 0.15 and n = 1500.,false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,11,"Table 4: The simulation results of CRC and RAC at the nominal level α = 0.1, where the sample",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,figure,12,"Figure 7: The results of risk certificate, decision loss, robustness, and coverage on synthetic data",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,13,"Table 5: The simulation results under polyhedral prediction set with nominal level α = 0.1, where",false -paper/Hu et al 2026 - Conformal Robustness Control.pdf,table,14,"Table 6: The results of CRC ablation experiments with nominal level α = 0.1, where the sample",false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,table,1,Table 1 shows the comparison of interest rate spreads that borrowers are charged on,false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,table,2,"Table 2. For example, we observe a significantly positive relationship between interest rate spreads",false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,figure,3,Figure 1. Correlation Between Origination FICO and,false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,figure,4,Figure 4. Average Spread by Rating Grades — Cards and,false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,figure,5,Figure 5. LendingClub Loans,false -paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,figure,6,Figure 7. LendingClub vs. Y-14M Cards Figure 8. LendingClub Loans,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,1,Figure 1: Uncertainty set constructed with q = 1 (left) and q = 2 (right) for random,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,2,"Figure 2: Illustration of (7), showing the feasible region (left) and uncertainty set (right).",false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,3,Figure 3: Example of nonconvexity for full conformal prediction region with sample co-,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,4,Figure 4 compares the overall robust optimization performance of the three uncertainty,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,5,Figure 4: α worst-case objective function value across n test cases with 100 experiment,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,6,Figure 5: Empirical coverage across n test cases with 100 experiment repetitions for nor-,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,7,Figure 6: Empirical coverage across n test cases with 100 experiment repetitions for nor-,false -paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,figure,8,Figure 7: log of α worst-case objective function value across n test cases with 100 experi-,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,1,"Figure 1: CPO leverages informative, non-convex conformal prediction regions for robust predict-then-optimize",false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,table,2,"Table 1: Coverages across tasks for α = 0.05 are shown in the left table, where coverage was assessed over a batch of",false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,3,Figure 2. Inflection points were around K = 10 for most setups.,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,4,Figure 2: Average volume estimates ℓ({B ck )}) over x(i) ∈ DC2 across SBI benchmarks.,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,table,5,Table 2: Coverage was assessed over 128 i.i.d. test samples and average objective optima over 10 i.i.d. test samples,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,6,"Figure 3: Suboptimality of the approximate representative points ∆(Ξ, Ξ)",false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,7,Figure 3 demonstrates the suboptimality of Ξ b decreases with increasing samples. Of note is that this convergence is,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,8,Figure 4: Solutions for the RTFP under the Box (left) and CPO (right) uncertainty regions.,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,9,Figure 5: Two RPs for C(x) for travel time prediction (left) and the extents of their Voronoi cells (right).,false -paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,figure,10,Figure 6: Precipitation maps (left) are converted to edge weights (right) as per Equation 19. Solving the shortest paths,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,1,Figure 1: Illustration of robust optimization (RO) and distributionally robust optimization (DRO) (here,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,2,Figure 2: Performance under different prediction and quantile regression models in PTC-B. The results,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,3,Figure 3: Value of better prediction Figure 4: Sample efficiency,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,table,4,Table 1: Average VaR and coverage on shortest path problems under different risk level settings. The,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,table,5,"Table 1. The Ellipsoid method does not utilize the covariates information, so although it has a coverage",false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,6,"Figure 5 shows the performance of different algorithms when d increases (d = 1, 4, and 16), and the plot",false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,7,Figure 5: Comparison in performance deterioration with increasing dimensions,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,8,Figure 6: Box plot of scaled VaR on shortest path problems under different risk level settings,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,table,9,Table 2 associated with Figure 7 displays the comparison results under varying risk levels and demon-,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,table,10,Table 2: Average VaR and coverage on fractional knapsack problems under different risk level settings.,false -paper/Sun et al 2024 - Predict-then-Calibrate.pdf,figure,11,Figure 7: Box plot of scaled VaR on knapsack problems under different risk level settings,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,1,Figure 1 clarifies the transition from data prepro­,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,2,Figure 1. The proposed model.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,3,Table 1. Description of independent variables used in this study.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,4,Figure 2. Schematic illustrating the variables used in this study.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,5,Figure 3. Box plot of research variables.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,6,Table 2. Number of samples in fully paid and failed classes.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,7,Figure 4. Scaling of studied characteristics. well-liked technique for balancing out imbalanced,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,8,Figure 5. Data distribution plot for charged-off and fully,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,9,"Table 3. Summary of data partitioning of the proposed Loan A’s NPV will be $1,200 and Loan B’s NPV will be",false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,10,Table 4. Comparison results of models.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,figure,11,Figure 6. The features of importance from random forest model.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,12,"Table 5. The computational results of the CNN, LSTM, and",false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,13,Table 7. The results of logistic regression. Table 8. Confusion matrix.,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,table,14,Table 10. Return and risk by NSGA-II. Table 12. Comparing the performance of two investment,false -paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,truncation_note,15,5 additional figure/table captions omitted from this compact index.,true -paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf,table,1,Table 1: CRO performance comparison. 100 CRO-CRS Mapping,false -paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf,figure,2,Figure 1: (a) Parameter mapping between CRS target,false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,1,"Figure 1: Cumulative average error Ēj for three scenarios alongside the shrinking interval (α + (1 − α)/j, α +",false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,2,Figure 2: Selective classification results on Imagenet. The left panel shows the choices of thresholds,false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,3,Figure 3: FDR control results in polyp segmentation. The top left and bottom left panels show,false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,4,Figure 3 shows empirical results from the FDR segmentation task with n = 500. Over 1000 bootstrap,false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,5,Figure 4: IOU control results in polyp segmentation. The left and middle panels show histograms of,false -supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,figure,6,"Figure 5: Results of multigroup debiasing on COMPAS dataset. Each row is a group, and the",false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,1,Figure 1: Illustration for the grid-approximated F-CROMS with Y ⊆ R2 . The red dots,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,2,Table 1: Comparison of computational cost in terms of number of model selection and,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,3,Table 2: The evaluation metrics and running time (seconds) with the 95% asymptotic,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,4,Table 2 presents the evaluation metrics of the compared methods along with their,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,5,Figure 2: The evaluation metrics with confidence intervals under the classification task.,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,6,Table 3 reports the evaluation metrics and running time of different methods. Our,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,7,Table 3: The evaluation metrics and running time (seconds) with the 95% asymptotic,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,8,"Figure 3: The average loss, worst-case conditional miscoverage, and worst-case conditional",false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,9,"Figure 4: The group conditional losses when varying sample size n in the classification task,",false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,10,"Figure 5: The average loss, marginal misrobustness, and worst-case conditional misrobustness",false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,figure,11,Figure 6: The group conditional loss on COVID-19 Radiography Database under α = 0.1.,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,12,Table 4: Summary of notations.,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,13,Table 5 and 6 present the performance of the compared methods along with their corre-,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,table,14,Table 5: The evaluation metrics and running time (seconds) on 100 test points with 95%,false -supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,truncation_note,15,6 additional figure/table captions omitted from this compact index.,true -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,1,Figure 1: Empirical results from a real data set (details will be given in Section 5.2).,false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,2,"Figure 2: Simulation results showing mean prediction interval coverage and width, averaged",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,3,"Table 1: Simulation results showing mean prediction interval coverage and width,",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,4,Figure 3: Electricity data results showing coverage and prediction interval width on the,false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,5,Table 2: Electricity data results showing coverage and prediction interval width on,false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,6,Figure 4: Left map: relative change in the number of votes for the Democratic presidential,false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,7,"Table 3: Election data results showing coverage, averaged over all test counties.",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,8,"Table 3 shows the resulting predictive coverage, averaged over the test set, for",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,9,"Table 4: Simulation results showing mean prediction interval coverage and width,",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,table,10,"Table 5: Simulation results showing mean prediction interval coverage and width,",false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,11,Figure 5: Simulation results showing mean prediction interval coverage and width for split,false -supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,figure,12,Figure 6: Simulation results showing mean prediction interval coverage and width for,false -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,1,"Figure 1: [Lei and Candès, 2021]’s WCP coverage guarantee (3) for varying numbers of groups.",false -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,2,"Figure 2: Simulation results for the setting of fixed group sizes, in three regimes for the group",false -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,3,"Figure 3: [Lei and Candès, 2021]’s WCP coverage guarantee (3), compared to the new guarantee",false -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,4,"Figure 4: Marginal coverage of the weighted conformal prediction interval (15), implemented with",false -supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,figure,5,"Figure 5: Simulation results for the setting of fixed group sizes, in three regimes for the group sizes",false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,1,TABLE 1: MORTGAGE LOAN SAMPLE,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,2,Table 2 provides the distribution of reported race and ethnicity (Reported) and the distributions,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,3,TABLE 2: DISTRUBUTION OF LOANS BY RACE AND ETHNICITY21,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,4,"Table 3 provides the correlations between reported race and ethnicity and the BISG, surname-",false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,5,TABLE 3: CORRELATIONS BETWEEN PROXY PROBABILITY AND REPORTED RACE AND ETHNICITY,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,6,"Table 4 provides the results of statistical comparisons of the geography-only, surname-only, and",false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,7,TABLE 4: LIKELIHOOD OF ASSIGNMENT OF HIGHER PROXY PROBABILITY FOR GROUP MEMBERSHIP,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,8,TABLE 5: CLASSIFICATION OVER RANGE OF BISG PROXY FOR HISPANIC,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,9,Table 5 makes clear the consequence of applying a threshold rule to the BISG proxy probability,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,10,TABLE 6: DISTRIBUTION OF RACE AND ETHNICITY FOR INDIVIDUALS IN THE U.S. POPULATION WITH THE,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,11,TABLE 7: POPULATION RESIDING IN CALIFORNIA AS A PERCENTAGE OF THE TOTAL U.S. POPULATION,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,table,12,"TABLE 8: SURNAME-ONLY AND BISG PROBABILITIES FOR ""SMITH"" IN CALIFORNIA",false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,figure,13,FIGURE 1: RECEIVER OPERATING CHARACTERISTIC (ROC) CURVES FOR NON-HISPANIC WHITE,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,figure,14,FIGURE 2: RECEIVER OPERATING CHARACTERISTIC (ROC) CURVES FOR NON-HISPANIC BLACK,false -supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,truncation_note,15,9 additional figure/table captions omitted from this compact index.,true -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,1,Figure 1: Training pipeline for task-based learning of the solution to the problem (3) with respect to the input,false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,2,Figure 2: Training pipeline for dual task based learning,false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,3,"Figure 3: Comparison of uncertainty set (α = 0.9) coverage for different ψ realizations: (a) [2.5, −0.2]T , (b) [−2.6, 0.5]T ,",false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,table,4,Table 1: Avg. CVaR and marginal coverage for α = 1 − ϵ =,false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,5,Figure 4: Average cumulative distribution of conditional,false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,6,Figure 5: Avg. CVaR of returns across 10 portfolio trajectory simulations. Error bars report 95% CI.,false -supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,figure,7,"Figure 6: Convergence comparison between 5-steps TRO sij ≥ −(ξ ij )T xi − λ ,",false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,1,Figure 1: Illustration of how proposed methods compare to the standard formulation of conformal prediction.,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,2,Figure 2: Empirical statistical coverage on the iNaturalist dataset across various loss functions. The observed,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,3,Figure 3: Comparisons between different methods for the 1) separable loss case using the sum of penalties in,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,4,Figure 4: Measured impact of base model accuracy on downstream optimization values on all losses considered,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,5,Figure 5: Mean and standard deviation from the predicted softmax score for the true class as a function of,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,6,Table 1: Conditional statistical coverage by set size for the cleaned version of the Fitzpatrick dataset and a,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,7,Table 2: Hyperparameters used to train the classifiers for each dataset. The configurations are identical except,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,8,"Table 3: Details about the datasets we use to calibrate and evaluate the sets. For the iNaturalist dataset, we use",false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,9,Figure 6: Comparison of our methods when we use a Bayesian model as the source of p̂(y|x). The plots show,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,10,Table 4: Comparison of our proposals for the separable loss case against base conformal predictions for dif-,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,figure,11,Figure 7: Comparisons between different methods for the 1) separable loss case using the sum of penalties in,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,12,Table 5: Comparison between different methods for the non-separable loss case across different datasets using,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,13,Table 6: Comparison between different methods for the coverage function loss case across different datasets,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,table,14,Table 7: Comparison between different values of the hyperparameter λ for the proposed Penalized Conformal,false -supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,truncation_note,15,8 additional figure/table captions omitted from this compact index.,true -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,1,"Figure 1: Local coverage frequencies for adaptive conformal (blue), a non-adaptive method that holds",false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,2,"Figure 2: Local coverage frequencies for adaptive conformal (blue), a non-adaptive method that holds",false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,3,"Figure 3: Local coverage frequencies of adaptive conformal (blue), a non-adaptive method that holds",false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,4,Figure 4 shows daily open prices for the four stocks considered in Section 2.2.,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,5,Figure 4: Daily open prices for the four stocks considered in Section 2.2.,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,6,Figure 5: Realized trajectories of αt for predicting stock market volatility as outlined in Section 2.2,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,7,Figure 6: Realized trajectories of αt for predicting stock market volatility as outlined in Section 2.2,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,8,Figure 7: Realized trajectory of αt for election night forecasting as outlined in Section 6 using update,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,9,Figure 8: Realized trajectory of αt for election night forecasting as outlined in Section 6 using update,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,10,Figure 9 shows the local coverage level of adaptive and non-adaptive conformal inference for the,false -supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,figure,11,"Figure 9: Local coverage frequencies for adaptive conformal (blue), a non-adaptive method that holds",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,1,Figure 1: Predicting with finite-sample guarantees: our conditionally valid pipeline vs. split conformal prediction.,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,2,"Figure 2: Comparison of split conformal prediction (blue, left-most panel) and the randomized implementation of",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,3,"Figure 2 illustrates the coverage guarantee on a simulated dataset. Here, x is univariate and we have",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,4,Figure 3: Marginal calibration-conditional miscoverage (left panel) and length (right panel) of quantile regression,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,5,"Figure 3 shows the improved finite sample coverage of these method. For simplicity, this plot only displays the",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,6,"Figure 4: Comparison of split conformal prediction (blue, left-most panel) and the randomized implementation of our",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,7,"Figure 4 illustrates a simple example of this guarantee on a synthetic dataset. Once again, the covariate",false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,8,Figure 5: Demonstration of the unrandomized implementation of our shift-agnostic method on a simulated dataset,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,9,Figure 6: Evaluation of the computational efficiency of our prediction set construction. The left panel displays the,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,10,Figure 6 displays the computational efficiency of this sensitivity analysis method on a 2020 MacBook,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,11,Figure 7: Empirical miscoverage achieved by split conformal inference (blue) and the unrandomized version of our,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,12,Figure 7 shows the empirical miscoverages obtained by our algorithm and split conformal prediction under,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,13,Figure 8: Empirical miscoverage achieved by split conformal prediction (blue) and the unrandomized version of our,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,figure,14,Figure 9: Estimated coverages for re-weightings of the data according to a Gaussian kernel centered at each of,false -supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,truncation_note,15,8 additional figure/table captions omitted from this compact index.,true -supplement/Guan 2023 - Localized Conformal Prediction.pdf,figure,1,Figure 1: Conformal bands (blue) and localized conformal bands (red) using regression score (dashed) and quantile,false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,2,Table 1: Example 4.1. Coverage and length comparison for CR and LCR (auto-tuned) across four simulation setups.,false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,3,"Table 2: Example 4.1. Comparisons of coverage, percent of infinite PI, ave.PI and ave.PI0 for different tuning",false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,figure,4,"Figure 2: Example 4.1. Confidence bands constructed using CR, LCR with different tuning parameter values for h",false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,5,"Table 3: Example 4.2. Empirical Coverage for different procedures across different simulation settings, with a targeted",false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,6,Table 4: Example 4.2. Average lengths of PIs for different procedures across four different simulation settings.,false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,7,Table 3 and 4 show the results of average coverage and average length of PI in the four simulation settings.,false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,8,"Table 5: Example 4.3. Empirical Coverage for different procedures, with a targeted level at α = 0.95. Column names",false -supplement/Guan 2023 - Localized Conformal Prediction.pdf,table,9,Table 6: Example 4.3. The upper half shows the average length using samples with finite PIs for the given procedure.,false -supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,table,1,Table 1: Empirical distribution of nonconformity scores (δa denotes a point mass at a).,false -supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,figure,2,"Figure 1: Empirical coverage of 80% prediction intervals from (W)SCPS, computed using",false -supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,figure,3,"Figure 2: Empirical CRPS of (W)SCPS, computed using 1000 different experiment trials",false -supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,figure,4,Figure 3: Post-hoc Friedman-Nemenyi test for CRPS.,false -supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,figure,5,"Figure 4: Distribution of p-values of SCPS under IID model (blue), covariate shift (orange),",false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,figure,1,"Figure 1: RAC pipeline, an interface between prediction and action for high stakes applications.",false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,figure,2,"Figure 2: Left: Illustration of how the functions θ and a are computed for a given x ∈ X and t ∈ [0, 1].",false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,figure,3,Figure 3: Results from two experiments. (a) Average realized max-min value as a function of α. (b),false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,table,4,"Table 1: Utility matrix for the four-class chest X-ray task. Rows denote true conditions, columns",false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,table,5,Table 2: Utility matrix for the MovieLens recommendation task.,false -supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,table,6,Table 3: Utility matrix for the four-class chest X-ray task.,false -supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf,figure,1,Fig. 1: Robot planner using a conformal controller on the Stanford Drone Dataset [1]. The future trajectories of humans are predicted,false -supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf,figure,2,Fig. 3: Stanford Drone Dataset: Qualitative Results. Visualization of interaction over time (left to right). (Top) With,false -supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf,figure,3,Fig. 5: Manufacturing Assembly Line Robot: Quantitative Results. (Left) Illustrative example: Robot must adjust the,false -supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf,figure,4,Fig. 6: Stock Trading: Quantitative Results. All results predictive model is actually good. Buy-and-hold also has,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,figure,1,Figure 1: Classic and conformal IO pipelines.,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,figure,2,Figure 2: Illustration of Example 1. The gray areas are the feasible region X (u). The black arrows,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,table,3,Table 1: Performance profile of classic and conformal IO in Example 1.,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,figure,4,Figure 3: Empirical coverage achieved by the learned uncertainty set (error bar = range).,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,figure,5,Figure 4: Performance profile of classic (blue) and conformal IO (green).,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,table,6,Table 2: Average (std) computational time of classic and conformal IO in seconds.,false -supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,figure,7,Figure 5: Percentage reduction in AOG and POG when using the conformal IO vs classic IO.,false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,1,Figure 1: Illustration of the proposed robust algorithm for multi-source conformal prediction. Each θb represented by a,false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,2,"Figure 2: A: Marginal coverage, B: Prediction interval width, C: Conditional coverage, and D: Weights for our proposed",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,3,Figure 3 displays the prediction intervals for hospi- semiparametric efficiency theory and federated learning for,false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,4,Figure 3: Each panel represents the prediction intervals for hospital LOS for a randomly selected individual following a,false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,5,Figure 4: Boxplots of prediction interval widths,false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,6,"Table 1: nk = 300, homogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,7,"Table 2: nk = 1000, homogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,8,"Table 3: nk = 3000, homogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,9,"Figure 5: Boxplots of coverage probability, under homogeneous covariate distributions",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,10,"Figure 6: Boxplots of prediction interval width, under homogeneous covariate distributions",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,11,"Table 4: nk = 300, weakly heterogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,12,"Table 5: nk = 1000, weakly heterogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,table,13,"Table 6: nk = 3000, weakly heterogeneous covariate distribution",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,figure,14,"Figure 7: Boxplots of coverage probability, under weakly heterogeneous covariate distributions",false -supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,truncation_note,15,10 additional figure/table captions omitted from this compact index.,true -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,table,1,"Table 1 reports the performance metrics. Due to space limitations, the full results for control-based",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,table,2,Table 1: Quantitative Comparison on the AXP Dataset. Performance metrics for UP-OCP versus,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,3,"Figure 1: Pareto frontiers for average prediction set size on the AXP Dataset, for 50 target miscoverage",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,4,Figure 1 illustrates that UP-OCP (red) empirically achieves the best Pareto trade-off. UP-OCP consis-,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,table,5,"Table 1. We observed this at all target levels, making the mean set size infinite.",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,6,Figure 2: Realized vs. target coverage.,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,7,Figure 3: Pareto frontiers on the synthetic sinusoid. Optimal performance is the bottom-left corner,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,8,"Figure 4: Plot of the realized marginal coverage against the average prediction set size, averaged over",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,9,Figure 5: Observed nonconformity scores for AAPL stock returns.,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,10,Figure 6: UP-OCP vs. DtACI for forecasting AXP stock return.,false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,11,"Figure 7: As in Figure 6, UP-OCP vs. KT.",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,12,"Figure 8: As in Figure 6, UP-OCP vs. SFOGD (lr=25).",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,13,"Figure 9: As in Figure 6, UP-OCP vs. P Ctrl (lr=0.1).",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,figure,14,"Figure 10: As in Figure 6, UP-OCP vs. PI Ctrl (lr=0.1).",false -supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,truncation_note,15,63 additional figure/table captions omitted from this compact index.,true -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,1,Figure 1: Decision-making under uncertainty involves both predictive and prescriptive analytics. In,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,2,Figure 2: An illustrative numerical example with a knapsack problem with two items to exemplify,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,3,"Figure 3: In decision-focused learning, the neural network model is trained to minimize the task loss",false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,4,Figure 4: An overview of gradient-based DFL methodologies categorized into four classes.,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,5,"Figure 4 presents key characteristics of these four methodology classes, highlighting the types of",false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,table,6,Table 1: A concise overview of gradient modeling techniques in key DFL techniques that use gradient-,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,table,7,Table 2: Brief overview of the test problems considered for experimental evaluation. The objective,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,table,8,Table 3: The range of hyperparameters for hyperparameter tuning by grid search.,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,9,Figure 5: Comparative evaluations on the synthetic shortest path problem with noise-halfwidth,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,10,Figure 6: Comparative evaluations on the synthetic portfolio optimization problem with noise mag-,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,11,Figure 7: Comparative evaluations on the Warcraft shortest path problem instances. The boxplots,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,12,Figure 8: Comparative evaluations on the energy-cost aware scheduling problem instances. This,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,13,Figure 9: Comparative evaluations on the knapsack problem instances. This boxplot shows the,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,figure,14,Figure 10: Comparative evaluations on the diverse bipartite matching problem instances. This,false -supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,truncation_note,15,13 additional figure/table captions omitted from this compact index.,true -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,1,Table 1: Portfolio Mean Return,false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,2,Table 2: Portfolio Covariance Matrix,false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,3,Table 3: Optimal Portfolio with the Minimum Variance Approach,false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,4,Table 4: VaR and CVaR obtained with the Minimum Variance Approach,false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,5,"Table 5: Portfolio, VaR, and CVaR from Min CVaR Approach: Monte Carlo Sim-",false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,6,"Table 6: The Portfolio, VaR, and CVaR from Min CVaR Approach: Simulations",false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,7,"Table 7: NIKKEI Portfolio, reproduced from Mauser and Rosen (1999).",false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,figure,8,"Figure 1: Distribution of losses for the NIKKEI portfolio with best normal approximation,",false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,9,"Table 8: Best Hedge, Corresponding VaR and CVaR with Minimum CVaR Approach:",false -supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,table,10,Table 9: Initial Positions and Best Hedge with Minimum CVaR Approach: Simul-,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,1,"Figure 1: Visualization of the pinball loss function in (6), where z = y − ŷ.",false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,2,Figure 2: Prediction intervals on simulated heteroscedastic data with outliers (see Figure 7 for a full,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,table,3,"Table 1 summarizes our 2,200 experiments, showing the average performance across all the datasets",false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,table,4,"Table 1: Length and coverage of prediction intervals (α = 0.1) constructed by various methods,",false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,5,Figure 4 shows the results for: blog feedback (blog_data) [41]; physicochemical properties of,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,6,"Figure 3: Average length (left) and coverage (right) of prediction intervals (α = 0.1), averaged over",false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,7,Figure 4: Refer to the caption of Figure 3 for details.,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,8,Figure 5: Refer to the caption of Figure 3 for details.,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,9,Figure 6: Refer to the caption of Figure 3 for details.,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,10,Figure 7: Full range scatter plot of the test data used in the synthetic simulation of Figure 2.,false -supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,figure,11,"Figure 8: Prediction intervals on simulated data constructed by locally adaptive conformal prediction,",false -supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,figure,1,Figure 1: Visualization of the empirical regret loss in DFL (left) and the robust losses in comparison (predictive pipeline is equal). The robust,false -supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,figure,2,Figure 2: Observed coefficients c (profit) per feature value z (temperature) and optimal linear predictors ĉ = fθ∗ (z) according to different loss,false -supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,figure,3,Figure 2 gives an illustrative example in which PFL using Var(min c) ̸= min Var(c) for some random variable c.,false -supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,figure,4,Figure 3: Test set mean normalized empirical regret in % (y-axes) on 3 experimental problems with different noise (ϵ̄) and training size (t),false -supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,table,5,Table 1: Test set mean normalized empirical regret in % (standard deviation) on 3 experimental problems with different noise (ϵ̄) and training,false -supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf,figure,1,"Figure 1: Empirical coverages of conformal prediction intervals, computed using 5000 different random splits",false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,1,Figure 1: Prediction sets with uniform coverage need to balance the coverage across multiple distributions. (a),false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,2,"Figure 2: Performance of MDCP and baselines in the classification Linear experiments, where the bars represent",false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,3,Figure 3 presents the results in the Nonlinear settings. While single-source calibration severely under-,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,4,Figure 3: Performance of MDCP and baselines in the classification Nonlinear experiments. The x-axis is the setting,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,5,Figure 4: Performance of MDCP and baselines in the classification Temperature experiments. The x-axis is the,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,6,Figure 5: Evaluation with regression Linear suites; details are otherwise the same as Figure 2.,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,7,Figure 6: Evaluation with regression Nonlinear suites; details are otherwise the same as in Figure 3.,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,8,Figure 7: Evaluation with regression Temperature suites; details are otherwise the same as in Figure 4.,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,9,"Figure 8: Performance of MDCP, Baseline-agg, and Baseline-src-k with each source region on the FMoW",false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,10,Figure 9: Results of MDCP and baselines in the PovertyMap dataset.,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,11,Figure 10: Results of MDCP and baselines in the MEPS dataset evaluation across three panels and two sensitive,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,12,Figure 10 reports the performance of the competing methods. The single-source baseline trained and,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,13,"Figure 11: Evaluation on the classification Linear suites, where MDCP with data-driven γ is labeled as “MDCP",false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,figure,14,Figure 12: Evaluation on the classification Nonlinear suites. Experimental settings are identical to Figure 3. The,false -supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,truncation_note,15,12 additional figure/table captions omitted from this compact index.,true -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,figure,1,Figure 1: Results for the tumor image segmentation problem (Section 5.1) across different FNR,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,figure,2,Figure 2: Results from the battery storage problem (Section 5.2) across different CVaR quantile levels,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,table,3,Table 1: Sensitivity of task loss and tail risk to relative changes in hyperparameter t on the,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,figure,4,Figure 3: Predictions on 8 randomly selected images from the test set for the tumor image segmen-,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,table,5,"Table 2: Comparison of cross-entropy loss, false negative rate (FNR), and false positive rate (FPR)",false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,figure,6,Figure 4: Values of the decision scaling factor λ for the battery storage problem (Section 5.2) across,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,figure,7,Figure 5: Comparison of financial tail risk (top) and task loss (bottom) as a function of calibration set,false -supplement/Yeh et al 2025 - Conformal Risk Training.pdf,table,8,Table 3: Sensitivity of task loss and tail risk to absolute changes in hyperparameter t on the battery,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,1,"Figure 1: Whereas prior “estimate-then-optimize” (ETO, top) approaches separate the model training",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,2,"Figure 1 (bottom) illustrates the key parts of our framework to answer these questions. First, we use a",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,3,"Figure 2: Consider a robust portfolio optimization problem with 2 assets, where y ∈ R2 is a random vector of",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,4,Figure 3: Task loss performance (mean ±1 stddev across 10 runs) for the battery storage problem with no,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,5,"Figure 3 (top) compares task loss performance for different uncertainty levels (α ∈ {.01, .05, .1, .2}) and the",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,6,Figure 4: Coverage (mean ±1 stddev across 10 runs) for the battery storage problem with no distribution,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,table,7,"Table A1: Time per epoch of pretraining, optimization, and end-to-end training, measured in seconds",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,table,8,Table A1 counterintuitively reports lower E2E training times for Ellipse and PICNN than optimization times.,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,9,"Figure A1: Similar to Figure 3, except that the value plotted is the VaR1−α [task loss] (mean ±1 stddev",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,10,"Figure A2: Similar to Figure 3, except that the value plotted is the CVaR1−α [task loss] (mean ±1 stddev",false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,table,11,Table A2: Task loss performance (mean ± 1 stddev across 10 runs) for the portfolio optimization problem.,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,table,12,Table A3: Coverage (mean ±1 stddev across 10 runs) for the portfolio optimization problem. Our E2E,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,13,Figure A3: This figure plots the density of the conditional distribution P(y | x) for x = −1.167 0.024,false -supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,figure,14,Figure A4: Visualization of data from the battery storage problem under distribution shift. The train/val,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,1,Figure 1 Our proposed framework (CREDO) compared against conventional predict-then-optimize (PTO).,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,2,Figure 2 Example of the CREDO procedure applied to a linear programming problem.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,3,Figure 3 Comparison of risk estimation using a point model and generative models (𝐾 = 3 and 𝐾 = 4) in CREDO.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,4,Figure 4 Illustration of settings I and II.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,table,5,"Table 1 summarizes the results. For the first three baselines (SA, LR, NN), we observe low estimation",false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,table,6,Table 1 Evaluated metrics for different risk estimation methods across different optimization settings.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,table,7,Table 2 presents the comparison results. It can be seen that CREDO achieves the smallest ranking metric,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,table,8,Table 2 Evaluated empirical confidence ranking (↓) for different methods across different optimization settings.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,9,Figure 5 Comparison of CREDO performance for three different calibrated radii.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,10,"Figure 5 presents the evaluation results comparing different types of calibrated radii ( 𝑅 𝑝 , 𝑅𝑒 and 𝑅∞ )",false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,11,Figure 6 Comparison of CREDO performance for four different generative model configurations.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,12,Figure 5 presents the evaluation results comparing different types of calibrated radii. We observe that,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,table,13,Table 3 Evaluated metrics for different optimization procedures solving (13b) across different settings.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,figure,14,Figure 6 presents the comparison results of CREDO risk estimation using different generative models.,false -supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,truncation_note,15,5 additional figure/table captions omitted from this compact index.,true -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,1,Figure 2. Illustration of the problem setup with a simple robust linear optimization problem. Panel (a) shows the miscoverage-regret,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,2,Figure 3. Validity-accuracy tradeoff curves under four optimization settings. Connected dots trace estimator performance as the number of,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,3,"Figure 4. Miscoverage-regret tradeoff Pareto frontiers. Given some prespecified preference function, there is an optimal tradeoff point",false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,4,Figure 3 presents the evaluation results comparing CREME,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,5,"Figure 3 also provides insights into the finite-sample be- N (µ̂, Σ̂), with µ̂ and Σ̂ estimated from the data. (ii) Em-",false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,table,6,Table 1. Sensitivity analysis of CREME against number of samples n and the robustness level index set size |Λ|.,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,table,7,Table 1 summarizes the experimental results. As n and |Λ|,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,8,"Figure 4 visualizes the evaluated Pareto frontier and tradeoff larger n and |Λ| lead to increased runtime, which peaks",false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,9,Figure 5. An example illustration of a network with three paths considered in our shortest path optimization.,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,10,Figure 6. Validity-accuracy tradeoff curves under four optimization settings. First row: validity versus robustness parameter λ. Second,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,11,Figure 7. Miscoverage–regret tradeoff Pareto frontiers with attained solutions from each model under different preference weights.,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,12,Figure 6 provides a detailed breakdown of the validity–accuracy performance evaluation originally presented in Figure 3,false -supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,figure,13,"Figure 7 parallels Figure 4 in the main paper, but presents additional trade-off points attained by different methods under",false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,figure,1,Fig. 1. The sorting method in NSGA2.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,table,2,Table 4 shows MSEs of MLPs with different number of neurons and theory and on the following two objectives.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,table,3,Table 5 presents the results of the fitted logistic regression. The Pareto front generated by the proposed algorithm is depicted,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,table,4,Table 7 shows the 10-fold cross validation results. The overall To understand more about what happened in the experimental,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,table,5,Table 3 Logistic regression results.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,figure,6,Fig. 3. Pareto front generated by NSGA2.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,table,7,"Table 9 model for lenders. In this single-objective optimization problem,",false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,figure,8,Fig. 5. Comparison of the risk of the best portfolio of models.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,figure,9,Fig. 4. Comparison of the return of the best portfolio of models.,false -tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,figure,10,"Fig. 7. The portfolio risk sensitivity to M. dinality constrained portfolio optimisation. Computers & Operations Research,",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,1,Table 1. Relative violation of constraint 372 in PILOT4,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,2,Table 1.1 Data for Example 1.1.1.,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,3,Figure 2.1 Robust solutions to portfolio selection problem from Example 2.3.6. Along,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,4,Table 2.1 Critical noise levels for two instances of the Signal Recovery problem.,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,5,Table 2.2 Experiments with A being near-projector onto the hyperplane {s ∈ R32 : si =,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,6,"Figure 2.2 Graphs of Σ(κ) , κ = 1, 2, 3, 4.",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,7,Table 2.3 Summary of Examples 2.4.6 – 2.4.9 and 2.4.11 – 2.4.13. In the R table for a,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,8,Table 3.1 Performance of various designs. In the table: Opt is the optimal value in,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,9,"Figure 3.1 Nominal, RC and GRC antenna designs. First rows: sample plots of |D(φ)|",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,10,Figure 3.2 “Circle hat” Γk (ABC) and triangle ∆k (ABD).,false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,table,11,"table 3.1, and illustrated in figure 3.1. We see that “immunizing” the design against",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,12,"Figure 4.1 Plots of OptI (²) — OptIII (²) and Opt+ (²). Top to bottom: Opt+ (²),",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,13,"Figure 4.2 Generating function γ11,8 (s) (middle curve) vs. exp{s} (top curve) and max[1+",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,figure,14,"Figure 4.3 Intersection of uncertainty sets, underlying various approximation schemes in",false -tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,truncation_note,15,58 additional figure/table captions omitted from this compact index.,true -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,1,Table 1 shows summary statistics for our sample of originated mortgages across the data merges. The first,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,2,Table 2 shows default rates across different loan products for the sample of applicants by loan approval status,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,3,Table 5 shows results from the estimated models. Our baseline models attain predictive performance that is,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,4,Table 6 shows that 46%-53% of the overall AUC difference is accounted for by the fact that disadvantaged,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,5,"Table 7 shows our second stage estimates, separately for different consumer types. The coefficients can be",false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,6,Table 9 presents a summary of model parameter estimates. We find that mortgage applicants differ substan-,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,7,Table 10 presents results from the two counterfactuals described above and also characterizes efficiency in the,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,figure,8,Figure 1: Receiver Operating Characteristic Curves by Consumer Type,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,figure,9,Figure 2: Plot of VantageScore 3.0 Credit Score and Predicted Default (XGBoost),false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,figure,10,Figure 3: SHAP Values for XGBoost Models by Consumer Type,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,figure,11,Figure 4: AUC and Compositional Differences in Credit Files,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,figure,12,Figure 5: Model Identification,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,13,Table 1: Summary Statistics: Originated Mortgages,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,table,14,Table 2: Summary Statistics: Default Measures,false -tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,truncation_note,15,8 additional figure/table captions omitted from this compact index.,true -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,1,Table 1: Count of Excluded Records by Reason,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,2,Table 2: Age Groups of Explanatory Census Variables,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,3,Table 3: Sample Summary Statistics,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,4,Figure 1: Incidence of Limited Credit History by Age,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,5,Figure 2: Distribution and Incidence of Limited Credit History by Age,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,6,Figure 3: Distribution and Incidence of Limited Credit History by Race or,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,7,"Figure 4: Coefficient Estimates, Limited Credit History",false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,8,Table 4: Incidence of Performance for Scored and Unscored Consumers,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,9,Table 4 shows the performance measures calculated for the population of,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,table,10,table 7 shows the results for stale unscored.,false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,11,"Figure 5: Coefficient Estimates, Credit Invisible",false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,12,"Figure 6: Coefficient Estimates, Insufficient Unscored",false -tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,figure,13,"Figure 7: Coefficient Estimates, Stale Unscored",false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,1,"Table 1: Model Performance For the image below, select the most appropriate type.",false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,2,Figure 2: Main trial screen shown to participants for Ob-,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,3,Table 2: Accuracy – p-values and Effect Sizes Prediction set,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,4,Table 3: Response Time – p-values and Effect Sizes Prediction Set Size,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,5,"Figure 7: Comparison of ObjectNet results between two models. The ViT-L/14 model achieved 83.3% top-1 accuracy,",false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,6,"Figure 10: Per-class accuracy on ObjectNet of Human-only (control), Model-only (top-1 accuracy), and Human-Model",false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,7,"Table 4: Adoption Rate (%) by Treatment Group not fool humans, making a human-in-the-loop perhaps the",false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,8,Table 5: Dataset Statistics,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,9,Table 6: Hyperparameter Settings and Empirical Coverage,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,table,10,Table 7: Demographics and Performance of Participants,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,11,Figure 11: Consent screen shown to participants at the start of the experiment.,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,12,Figure 12: Main trial screen shown to participants for GoEmotions with conformal set treatment. The correct answer is,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,13,Figure 13: Screens displayed to participants during our experiment using conformal sets on Few-NERD. Other experiments,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,figure,14,Figure 14: Histograms of accuracy observations across three treatments for each task. Vertical lines indicate the mean of,false -tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,truncation_note,15,5 additional figure/table captions omitted from this compact index.,true -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,1,Figure 1: Effect of label noise on CIFAR-10. Left: distribution of average coverage on,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,2,Figure 2 portrays the marginal coverage and average interval length achieved using,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,3,Figure 2: Results for real-data regression experiment: predicting aesthetic visual,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,4,"Figure 3 illustrates the idea behind Theorem 1, demonstrating that when the noisy score",false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,5,Figure 3: Clean (green) and noisy (red) non-conformity scores under dispersive corruption.,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,6,Figure 4: Clean (green) and noisy (red) class probabilities under dispersive corruption.,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,table,7,Table 1: Summary of coverage bounds for different scores and noise models,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,8,"Figure 5: FNR on MS COCO data set, achieved over noisy (red) and clean (green) test",false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,table,9,Table 2: Summary of coverage bounds for different risk control tasks and different noise,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,10,Figure 6: Effect of label noise on synthetic multi-class classification data. Per-,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,11,Figure 7: Response-independent noise. Performance of conformal prediction intervals,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,12,Figure 8: Dispersive versus contractive noise regression experiment. Performance,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,13,Figure 9: Visualization of the multi-label CIFAR-100N data.,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,figure,14,Figure 10: FNR achieved over noisy (red) and clean (green) test sets of the on CIFAR-100N,false -tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,truncation_note,15,17 additional figure/table captions omitted from this compact index.,true -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,1,Figure 1 gives an instructive example of possible group-specific effects when borrow-,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,2,Figure 1. Unequal Effects of Better Technology.,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,3,Figure 2 provides an example where unequal effects are generated exclusively by tri-,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,4,Figure 2. Triangulation.,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,5,Figure 3. ROC and Precision-Recall Curves.,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,6,Figure 4. Example of Predicted Default Probabilities Across Models. Figure,false -tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,figure,7,"Figure 5. Comparison of Predicted Default Probabilities Across Models, by",false -tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,table,1,Table 1: US stock data,false -tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,table,2,Table 2: Japanese stock data,false -tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,figure,3,"Figure 1: Experimental results for US stocks. The y-axis indicates cumulative returns, and",false -tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,figure,4,"Figure 2: Experimental results for Japanese stocks. The y-axis indicates cumulative returns,",false -tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,figure,5,Figure 2 presents results for the Japanese market and again confirms the strong per-,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,1,Figure 1: Construction of an integrated database of transactions and credit bureau statistics,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,2,"Figure 2 depicts the relationship between CScore for the months of January of 2005,",false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,3,Figure 2: Relationship between CScore and subsequent default frequency (over the next 6,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,4,"Figure 5: Characteristics of credit-card balances, payments and their relationship to monthly",false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,5,Figure 6: Characteristics of payments of all forms of debt to monthly income and total,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,6,Figure 7: Delinquency rates of customers with high credit-card-payment-to-income ratios.,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,7,Figure 8: Delinquency rates for customers with high credit-card-payment-to-income ratios.,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,8,Figure 9: Delinquency rates for customers experiencing negative income shocks. For each,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,9,Figure 10: A summary of the machine-learning algorithm used to construct the consumer,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,table,10,Table 4 lists the data elements from the three sources discusses in Section 2.1 that were,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,11,Figure 11 compares the machine-learning forecasts of 90-days-or-more delinquencies of the,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,figure,12,"Figure 11: Comparison of December 2008 CScore (x-axis, and where higher values are asso-",false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,table,13,"Table 1: Inflows and outflows into accounts are collected by the Bank. Direction, amount,",false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,table,14,Table 2: Data items extracted from credit bureaus. Not all fields extracted were used because,false -tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,truncation_note,15,16 additional figure/table captions omitted from this compact index.,true diff --git a/reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv b/reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv deleted file mode 100644 index 4c7dbdf..0000000 --- a/reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv +++ /dev/null @@ -1,62 +0,0 @@ -folder,file,relative_path,title,authors,year,status,primary_domain,bib_key,bib_status,canonical_url,verification_status,pages,word_count,figure_caption_count,table_caption_count,core_concepts,key_claims,method_or_evidence,conclusions,figures_tables_useful,limitations,crpto_value,extended_lab_value,quarto_destination,decision,action_required,evidence_gate,artifact_sink,stop_rule,implementation_or_experiment,notes,tags -paper,Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf,Credit Scores: Performance and Equity,Stefania Albanesi; Domonkos F. Vamossy,2024,NBER working paper,credit scoring / equity,albanesi2024credit,existing,https://doi.org/10.3386/w32917,local_pdf_text,59,21439,10,30,"Score performance, equity, misclassification, rank disagreement, observable vulnerable groups.",Better scores can improve accuracy while also changing error allocation across groups.,Empirical credit-score comparison and equity diagnostics.,"Useful for metric governance: report discrimination, calibration, utility and equity together.",Use tables on score performance/equity as template for FICO proxy vs champion diagnostics.,Not Lending Club CRPTO evidence and not a legal fair-lending protocol.,Light motivation: CRPTO is not only AUC; calibrated decision inputs require governance.,Strong appendix support for score-proxy vs champion ML and equity boundary.,agenda extendida CRPTO/tesis appendix; Paper CRPTO motivation only.,append_extended_governance,completed_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;equity;governance -paper,Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf,Conformal Prediction: A Gentle Introduction,Anastasios N. Angelopoulos; Stephen Bates,2023,Foundations and Trends in Machine Learning,conformal foundations,angelopoulos2023,existing,https://arxiv.org/abs/2107.07511,local_pdf_text,51,25093,28,1,"Exchangeability, split conformal, prediction sets, finite-sample marginal coverage.",Distribution-free uncertainty can be wrapped around black-box predictors.,"Tutorial/survey with proofs, examples and diagnostics.",Best pedagogical citation for explaining why CRPTO's intervals are valid before optimization.,Use tutorial figures only conceptually; the paper needs its own CRPTO diagram.,Introductory; not a portfolio optimizer or credit-risk paper.,Core theory support for the uncertainty layer.,Proof-audit reference for lab gates.,CRPTO theory and glossary.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;foundation -paper,Angelopoulos et al 2024 - Conformal Risk Control.pdf,paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf,Conformal Risk Control,Anastasios N. Angelopoulos; Stephen Bates; Adam Fisch; Lihua Lei; Tal Schuster,2024,ICLR 2024,conformal risk control,angelopoulos2024risk,existing,https://openreview.net/forum?id=33XGfHLtZg,local_pdf_text,21,11236,7,0,"Risk control, monotone bounded losses, post-hoc calibration, user-defined loss.",Conformal calibration can control expected loss beyond ordinary set coverage.,Finite-sample risk-control framework.,Direct ancestor of CRPTO's funded-set weighted miscoverage framing.,Use conceptual loss-control diagrams as related-work inspiration.,Does not itself define a portfolio-funded-set bound.,Core theory support for mapping uncertainty to a decision-relevant loss.,CRC/LTT decision-loss gates for future experiments.,CRPTO theory; agenda extendida CRPTO/tesis gate register.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;risk_control -paper,Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf,Theoretical Foundations of Conformal Prediction,Anastasios N. Angelopoulos; Rina Foygel Barber; Stephen Bates,2026,Cambridge University Press pre-publication / monograph,conformal foundations,angelopoulos2024foundations,existing,https://arxiv.org/abs/2411.11824,local_pdf_text,266,136226,44,1,"Exchangeability, exact finite-sample coverage, randomized quantiles, modern CP theory.","Conformal validity is a theorem about a procedure and assumptions, not a model-quality claim.",Long-form theoretical reference.,Use as the canonical proof reference for split conformal language and limitations.,"Use definitions/proof structure, not figures, in CRPTO.",Too broad for the paper body; cite selectively.,Canonical theory reference for the proof and notation.,Proof-audit anchor for future online/source claims.,CRPTO theory; glossary.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;foundation -paper,Angelopoulos et al 2025 - Learn Then Test.pdf,paper/Angelopoulos et al 2025 - Learn Then Test.pdf,Learn then Test: Calibrating Predictive Algorithms to Achieve Risk Control,Anastasios N. Angelopoulos; Stephen Bates; Emmanuel J. Candes; Michael I. Jordan; Lihua Lei,2025,Annals of Applied Statistics,risk control / post-hoc testing,angelopoulos2025ltt,existing,https://doi.org/10.1214/24-AOAS1998,local_pdf_text,22,11696,11,0,"Learn-then-test, finite-sample risk guarantees, post-hoc risk calibration.",Predictive algorithms can be calibrated after learning to satisfy risk constraints.,Statistical testing/calibration framework.,Supports predeclared gates and reviewer-safe validation language.,Use algorithmic workflow conceptually for gate diagrams.,Does not solve downstream optimization by itself.,Supports exact risk-gate language and post-selection caveats.,Lane-gate template for CRC/LTT decision-loss checks.,CRPTO theory; agenda extendida CRPTO/tesis future gate register.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;risk_control;gates -paper,Barber et al 2021 - Limits of Distribution-Free Conditional Predictive Inference.pdf,paper/Barber et al 2021 - Limits of Distribution-Free Conditional Predictive Inference.pdf,The Limits of Distribution-Free Conditional Predictive Inference,Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas; Ryan J. Tibshirani,2021,Information and Inference,conditional coverage limits,barber2021limits,existing,https://doi.org/10.1093/imaiai/iaaa017,local_pdf_text,34,13440,0,0,"Conditional coverage impossibility, approximate conditional validity, subgroup limits.",Exact distribution-free conditional coverage is impossible without assumptions or restrictions.,Theoretical impossibility/boundary results.,This is a claim-boundary source: it prevents overclaiming subgroup guarantees.,"Use theorem statements, not figures.","Boundary paper, not an implementation.",Keeps Mondrian coverage language honest.,Source/fairness coverage caveat.,CRPTO theory; agenda extendida CRPTO/tesis online/source caveats.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;conditional_coverage;claim_boundary -paper,Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf,"Distribution-Free, Risk-Controlling Prediction Sets",Stephen Bates; Anastasios N. Angelopoulos; Lihua Lei; Jitendra Malik; Michael I. Jordan,2021,arXiv / risk-control foundation,risk-controlling prediction sets,bates2021rcps,existing,https://arxiv.org/abs/2101.02703,local_pdf_text,34,17464,23,1,"RCPS, set-valued prediction, expected loss control, black-box models.",Prediction sets can be calibrated to control a user-defined risk distribution-free.,Post-hoc calibration framework.,Provides the bridge from coverage to risk-control vocabulary.,Use algorithmic schematic as conceptual support only.,General prediction sets; no credit portfolio objective.,Core CRPTO theory lineage.,Decision-risk gate template.,CRPTO theory and related work.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;risk_control -paper,Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf,Data-Driven Robust Optimization,Dimitris Bertsimas; Vishal Gupta; Nathan Kallus,2018,Mathematical Programming,data-driven robust optimization,bertsimas2018datadriven,needs_bib_if_cited,https://doi.org/10.1007/s10107-017-1125-8,local_pdf_text,53,23925,10,5,"Uncertainty sets learned from data, robustness guarantees, finite-sample feasibility.",Robust sets can be constructed data-driven rather than manually specified.,Optimization theory and computational experiments.,Supports the historical progression from ad-hoc RO to calibrated/data-driven uncertainty sets.,Useful as related-work table contrasting source of uncertainty sets.,Not conformal and not Lending Club-specific.,Background for why conformal sets are a natural next RO step.,Context only.,CRPTO related work if a longer RO paragraph is added.,context_crpto_related_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,robust_optimization -paper,Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf,From Predictive to Prescriptive Analytics,Dimitris Bertsimas; Nathan Kallus,2020,Management Science,prescriptive analytics,bertsimas2020prescriptive,needs_bib_if_cited,https://doi.org/10.1287/mnsc.2018.3253,local_pdf_text,57,27603,12,0,"Prescriptive analytics, decision quality, predictive-to-prescriptive bridge.",Prediction alone is insufficient; decision policies should be evaluated by downstream outcomes.,Framework and empirical prescriptive methods.,"Useful framing for why CRPTO evaluates funded-set outcomes, not AUC only.",Related-work schematic only.,Not conformal and not a risk-control guarantee.,Strengthens introduction/positioning.,Supports governed decision-analytics framing.,CRPTO related work or thesis framing.,context_crpto_related_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,prescriptive_analytics;decision -paper,Bertsimas Sim 2004 - The Price of Robustness.pdf,paper/Bertsimas Sim 2004 - The Price of Robustness.pdf,The Price of Robustness,Dimitris Bertsimas; Melvyn Sim,2004,Operations Research,robust optimization,bertsimas2004,existing,https://doi.org/10.1287/opre.1030.0065,local_pdf_text,34,9963,13,5,"Budgeted uncertainty, protection level, tractable robust LP/IP, price of robustness.","Robustness trades objective value for protection in a tunable, tractable way.",Classical robust optimization formulation.,Use as the language ancestor for alpha -> Gamma_CP -> return trade-off.,Use price-of-robustness framing in CRPTO figures.,"Budget is chosen, not conformally calibrated.",Core RO foundation.,Tail/challenger context.,CRPTO theory and portfolio chapter.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,robust_optimization -paper,Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf,Data-Driven Robust Credit Portfolio Optimization for Investment Decisions in P2P Lending,Guangyu Chi; Lei Ding; Xiangyu Peng,2019,Applied Soft Computing / Elsevier article,P2P credit portfolio optimization,chi2019p2p,added_2026_06_06,,local_pdf_text,10,8038,2,2,"P2P lending, robust credit portfolio, data-driven scoring, risk-return allocation.",Robust optimization can improve P2P investment decisions under model uncertainty.,Empirical P2P lending optimization study.,Closest credit-domain precedent for robust P2P allocation.,Use experimental design/tables as comparator template for Lending Club portfolio papers.,No conformal guarantees and no funded-set miscoverage bound.,Important applied-credit related work.,Context for digital-lending robust allocation.,CRPTO related work; thesis context.,append_crpto_related_work,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;robust_optimization;p2p -paper,Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf,Task-based End-to-end Model Learning in Stochastic Optimization,Priya L. Donti; Brandon Amos; J. Zico Kolter,2017,NeurIPS 2017,decision-focused learning,donti2017,existing,https://arxiv.org/abs/1703.04529,local_pdf_text,15,9069,5,2,"End-to-end learning, differentiable optimization, downstream task loss.",Training against task loss can outperform prediction-error training for decisions.,Decision-focused learning experiments.,DFL lineage for SPO+/PyEPO comparator.,"Use as historical DFL lineage, not as CRPTO method figure.",No conformal coverage or credit-specific auditability.,Comparator lineage only.,Supports PyEPO/DFL lane.,CRPTO related work; agenda extendida CRPTO/tesis DFL appendix.,append_comparator,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,dfl -paper,Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf,"Smart Predict, then Optimize",Adam N. Elmachtoub; Paul Grigas,2022,Management Science,SPO+ / predict-then-optimize,elmachtoub2022,existing,https://doi.org/10.1287/mnsc.2020.3922,local_pdf_text,46,19271,7,0,"SPO loss, SPO+ surrogate, regret-oriented training, downstream decision quality.",Models can be trained to reduce decision regret rather than prediction error.,Theory and computational experiments for SPO/SPO+.,Primary DFL comparator: SPO+ may reduce regret but does not provide conformal auditability.,Use benchmark/regret tables as comparator template.,No coverage guarantee and no uncertainty-set audit trail.,"Core comparator, not replacement.",Formal PyEPO lane support.,CRPTO related work and SPO appendix.,append_comparator,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,dfl;spo -paper,Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf,Instance-based Credit Risk Assessment for Investment Decisions in P2P Lending,Yanhui Guo; Wenjun Zhou; Chunyu Luo; Chuanren Liu; Hui Xiong,2016,European Journal of Operational Research,P2P credit decision support,guo2016p2p,needs_bib_if_cited,https://doi.org/10.1016/j.ejor.2015.05.050,local_pdf_text,10,10638,3,1,"Instance-based learning, P2P loan investment, risk-return decision support.",Loan-level decision support can improve P2P investment performance.,Empirical P2P lending recommender/credit assessment.,Useful historical Lending Club/P2P context.,Portfolio decision tables can inspire thesis background.,No conformal or robust bound.,Applied setting support.,Digital-lending decision context.,Related work if credit-domain section expands.,context_credit_domain,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;p2p -paper,Hu et al 2026 - Conformal Robustness Control.pdf,paper/Hu et al 2026 - Conformal Robustness Control.pdf,Conformal Robustness Control,Yang Hu; Jieren Tan; Changliang Zou; Yajie Bao; Haojie Ren,2026,ICLR 2026,conformal robustness / decision risk,hu2026crc,added_2026_06_06,https://openreview.net/forum?id=xRjOrcj08o,web_primary_and_local_pdf,29,15751,8,6,"Robustness control, conformal calibration, robust decision criteria.",Conformal machinery can certify robustness-oriented decision properties.,Recent conference method with robust-control framing.,"Confirms CRPTO is timely, but should remain future-work unless reproduced.","Use as frontier map figure/table in related work, not main evidence.",Not yet implemented in the project; no Lending Club funded-set result.,Future-work frontier; supports not overclaiming current method.,Candidate future gate for robustness certificates.,agenda extendida CRPTO/tesis future gate; CRPTO related-work frontier.,park_future_work,completed_bib_quarto_patch,Only run if a reviewer asks for robustness-control comparison or it changes C4/C6.,agenda extendida CRPTO/tesis future experiment register.,Park if it cannot be evaluated on frozen CRPTO artifacts without retraining a new method.,Possible robustness-control audit over V/violation; not executed.,,conformal;robustness;future_work -paper,Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf,The Roles of Alternative Data and Machine Learning in Fintech Lending,Julapa Jagtiani; Catharine Lemieux,2019,Financial Management,fintech lending / LendingClub,jagtiani2019altdata,existing,https://doi.org/10.1111/fima.12295,local_pdf_text,32,11827,4,2,"LendingClub, alternative data, credit grades, FICO relationship, fintech underwriting.",Fintech grades and alternative data provide empirical credit information beyond traditional scores.,Empirical LendingClub study.,Best setting citation for why Lending Club is a meaningful credit-risk lab.,"Use tables about grade/FICO/performance as context, not CRPTO evidence.",Does not validate CRPTO or fairness claims.,Core empirical-setting support.,Metric governance and FICO proxy context.,CRPTO introduction and dataset context.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;lendingclub -paper,Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf,Conformal Uncertainty Sets for Robust Optimization,Chancellor Johnstone; Bruce Cox,2021,COPA / PMLR,conformal robust optimization,johnstone2021,existing,https://proceedings.mlr.press/v152/johnstone21a.html,local_pdf_text,19,7385,8,0,"Conformal regions as uncertainty sets, robust optimization, finite-sample validity.",Conformal prediction regions can feed robust optimization with validity.,Method paper with simulation evidence.,Closest foundational CP -> RO bridge.,Use as conceptual predecessor for CRPTO pipeline figure.,No credit-specific funded-set weighted bound.,Core method foundation.,Context only.,CRPTO theory and related work.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;robust_optimization -paper,Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf,Conformal Contextual Robust Optimization,Neel Patel; Omar Rayan; Ambuj Tewari,2024,AISTATS 2024,conformal contextual robust optimization,patel2024,existing,https://arxiv.org/abs/2310.10003,local_pdf_text,22,8949,8,2,"Contextual robust optimization, conformal regions, generative uncertainty.",Informative conformal uncertainty regions can improve robust contextual decisions.,Method and benchmark experiments.,Strong neighboring method but with different uncertainty geometry and domains.,Use related-work contrast table.,Not credit-specific and no funded-set PD constraint.,Direct neighbor in related work.,Future high-dimensional uncertainty context.,CRPTO related work.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;robust_optimization -paper,Sun et al 2024 - Predict-then-Calibrate.pdf,paper/Sun et al 2024 - Predict-then-Calibrate.pdf,Predict-then-Calibrate: A New Perspective of Robust Contextual LP,Chunlin Sun; Linyu Liu; Xiaocheng Li,2024,arXiv / robust contextual LP,predict-then-calibrate,sun2024ptc,existing,https://arxiv.org/abs/2305.15686,local_pdf_text,33,15922,7,4,"Predict-then-calibrate, robust contextual LP, box/ellipsoid uncertainty sets.",Post-hoc calibration can build valid uncertainty sets for robust contextual optimization.,Theory and experiments for robust LP calibration.,Closest conceptual comparator to post-hoc CRPTO.,Use contrast table: PtC set membership vs CRPTO funded-set weighted risk.,No Lending Club and no Mondrian funded-set bound.,Core closest-neighbor contrast.,Selector/calibration context.,CRPTO related work and theory.,promote_crpto_body,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;robust_optimization;ptc -paper,Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf,Integrating AI and OR for investment decision-making in emerging digital lending businesses,Vajiheh Torkian; Shahrooz Bamdad; Amir Homayoun Sarfaraz,2026,Journal of the Operational Research Society; online 2025,AI/OR digital lending,aior2025lendingclub,added_2026_06_06,https://doi.org/10.1080/01605682.2025.2498652,web_primary_and_local_pdf,21,14792,9,10,"Digital lending, multi-objective risk-return optimization, AI + OR pipeline.",Combining ML and OR improves investment recommendations in digital lending.,Applied multi-objective optimization paper.,Useful applied competitor/context; not a conformal method.,Use risk-return frontier and experimental setup as agenda extendida CRPTO/tesis comparator inspiration.,"No conformal coverage, no CRPTO bound, no Lending Club champion evidence.",Related-work context only.,agenda extendida CRPTO/tesis digital-lending architecture context.,agenda extendida CRPTO/tesis synthesis; CRPTO related work if space allows.,append_extended_context,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;optimization;digital_lending -paper,Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf,paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf,Conformal Robust Optimization and Satisficing for Prescriptive Analytics with Black-Box Predictors,Lingjie Zhao; Hansheng Jiang; Wei Qi,2026,AISTATS workshop / SSRN working paper,conformal robust optimization / satisficing,zhao2025robust,existing,https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5338354,local_pdf_text,5,3097,1,1,"Conformal robust optimization, satisficing, black-box predictors, parameter mapping.",CRO and conformal robust satisficing can certify robust prescriptive decisions.,Workshop paper with optimization experiments.,Valuable frontier for satisficing margins and policy acceptance thresholds.,"Use satisficing margin idea for appendix, not body.",Preprint/workshop and not credit-specific.,Related-work frontier.,Satisficing/governance candidate.,CRPTO related work; agenda extendida CRPTO/tesis future gates.,append_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;robust_optimization;satisficing -supplement,Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf,Conformal Risk Control for Non-Monotonic Losses,Anastasios N. Angelopoulos,2026,arXiv preprint,non-monotonic conformal risk control,angelopoulos2026nonmonotonic,existing,https://arxiv.org/abs/2602.20151,local_pdf_text,22,10455,6,0,"Non-monotonic loss, risk control, multi-objective loss families.",CRC ideas can extend beyond monotone losses under new machinery.,Theory preprint.,Useful for future compound loss: return + V + CVaR + source risk.,Use loss examples for future gate design.,Not implemented; would change current CRPTO claim.,Future-work only.,Strong future gate for composite risk.,agenda extendida CRPTO/tesis future experiment register.,park_future_work,none_now,Only reopen with a fixed compound loss and prospective split.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;risk_control;future_work -supplement,Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf,Optimal Model Selection for Conformalized Robust Optimization,Yajie Bao; Yang Hu; Haojie Ren; Peng Zhao; Changliang Zou,2025,arXiv preprint,CROMS / model selection,bao2025croms,existing,https://arxiv.org/abs/2507.04716,local_pdf_text,104,40342,6,14,"Conformalized robust optimization, decision-aware model selection, downstream risk.",Selecting conformal models should account for robust decision performance.,Theory/algorithm and experiments.,Justifies CRPTO's CROMS-lite selector but not a full implementation claim.,Use selection diagrams/tables to structure A5/A10 appendix.,"Project only has a selector screen over artifacts, not CROMS training.",Appendix selector support.,Lane 2 selector context.,CRPTO appendix and agenda extendida CRPTO/tesis selector lane.,append_crpto_selector,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;selector;robust_optimization -supplement,Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf,Conformal Prediction Beyond Exchangeability,Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas; Ryan J. Tibshirani,2023,Annals of Statistics,beyond exchangeability / weighted conformal,barber2023beyond,added_2026_06_06,https://doi.org/10.1214/23-AOS2276,web_primary_and_local_pdf,63,23681,6,6,"Weighted exchangeability, distribution shift, non-exchangeable conformal validity.",Conformal validity can be extended with weights under structured departures from exchangeability.,Theory paper.,Important boundary/future source for source/shift lanes.,Use assumptions table for source governance appendix.,Requires weight/shift structure not currently guaranteed.,Caveat/future work.,Strong source/shift conformal lane reference.,agenda extendida CRPTO/tesis online/source caveat.,append_extended_source_shift,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;shift;source -supplement,Ben-Tal Teboulle 2007 - Optimized Certainty Equivalent.pdf,supplement/Ben-Tal Teboulle 2007 - Optimized Certainty Equivalent.pdf,An Old-New Concept of Convex Risk Measures: The Optimized Certainty Equivalent,Aharon Ben-Tal; Marc Teboulle,2007,Mathematical Finance,OCE / convex risk measures,bental2007oce,existing,https://doi.org/10.1111/j.1467-9965.2007.00311.x,local_pdf_text,30,11477,0,0,"Optimized certainty equivalent, convex risk measures, CVaR relation.",OCE provides a principled convex risk-measure family.,Mathematical risk-measure theory.,Grounds OCE/CVaR tail-risk appendix.,"Use definitions, not figures.",Does not choose a credit champion by itself.,Appendix tail-risk foundation.,Tail challenger foundation.,CRPTO appendix; agenda extendida CRPTO/tesis tail lane.,append_tail_risk,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,tail_risk;oce -supplement,Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf,Group-Weighted Conformal Prediction,Aabesh Bhattacharyya; Rina Foygel Barber,2026,Electronic Journal of Statistics,group-weighted conformal prediction,bhattacharyya2026groupweighted,added_2026_06_06,https://doi.org/10.1214/26-EJS2506,web_primary_and_local_pdf,27,13333,5,0,"Group-weighted CP, group-based shift, weighted conformal prediction.","When groups drive covariate shift, group-weighted calibration can improve guarantees.",Theory paper with group-weighted guarantees.,"Natural candidate for agenda extendida CRPTO/tesis source/grade reweighting, but not current CRPTO.",Use group-weighting assumptions for source-governance table.,Needs target/source group weights and prospective design.,Future robustness/context only.,Potential source/shift experiment over grade/period/source families.,agenda extendida CRPTO/tesis source/shift appendix.,append_mixed_diagnostic_source_not_better,experiment_completed_appendix_diagnostic,"Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; absolute gate passed, but worst-source coverage fell 0.0111 below Mondrian.",docs/research/papers_tesis_deep_audit_2026-06-06.md,Append as source/shift diagnostic only; do not claim group-weighted source improvement over Mondrian.,"Group-weighted source-max replay: coverage 0.9387, avg width 0.9394, worst defended source 0.8602.",,conformal;group_weighted;shift -supplement,CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf,Using Publicly Available Information to Proxy for Unidentified Race and Ethnicity,Consumer Financial Protection Bureau,2014,official methodology report,fair-lending proxy methodology,cfpb2014bisg,needs_bib_if_cited,https://files.consumerfinance.gov/f/201409_cfpb_report_proxy-methodology.pdf,local_pdf_text,37,9880,6,17,"BISG-style proxy, surname/geography, proxy validation, protected-attribute limitation.",Race/ethnicity proxying requires richer inputs and careful validation.,Regulatory/official methodology report.,Supports why Lending Club zip3/state is insufficient for legal fair-lending claims.,Use proxy-method workflow as governance appendix reference.,Project lacks surname and fine geography.,Fairness limitation only.,Strong claim-boundary source for proxy governance.,agenda extendida CRPTO/tesis fairness/source governance.,append_extended_governance,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,fairness;governance -supplement,Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf,End-to-End Conditional Robust Optimization,Abhilash Chenreddy; Erick Delage,2024,UAI 2024 / PMLR,end-to-end conditional robust optimization,chenreddy2024,existing,https://arxiv.org/abs/2403.04670,local_pdf_text,13,9034,6,1,"Conditional robust optimization, differentiable optimization, end-to-end training.",End-to-end conditional robust training can improve conditional coverage/objective tradeoffs.,Method with differentiable robust optimization experiments.,"Frontier contrast: CRPTO is post-hoc and auditable, not end-to-end.",Use as frontier comparison table.,Different stack and conditional guarantees; not current project method.,Future-work contrast.,End-to-end lane context.,agenda extendida CRPTO/tesis future work; CRPTO related-work frontier.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,robust_optimization;end_to_end -supplement,Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf,Utility-Directed Conformal Prediction,Santiago Cortes-Gomez; Carlos Patino; Yewon Byun; Steven Wu; Eric Horvitz; Bryan Wilder,2025,ICLR 2025,decision-aware conformal prediction,cortesgomez2025utility,added_2026_06_06,https://openreview.net/forum?id=iOMnn1hSBO,web_primary_and_local_pdf,21,11008,8,14,"Utility-directed CP, decision loss, actionable uncertainty, coverage preservation.",Prediction sets can incorporate downstream utility while retaining standard coverage.,Decision-aware conformal method and experiments.,Highly relevant future step: optimize CP usefulness without abandoning coverage.,Use framework diagram for future decision-loss conformal selector design.,Current CRPTO does not implement this training/calibration objective.,Future-work extension; supports A5 selector direction.,Candidate decision-loss gate or conformal utility experiment.,agenda extendida CRPTO/tesis future gates; CRPTO future work.,park_width_fail,experiment_completed_parked,Executed with fixed width/violation/tail-miss loss on frozen v4 replay; coverage rose to 0.9954 but avg width 0.9981 failed the 0.98 width gate.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Park as negative width result; no utility-directed selector claim.,"Utility-directed loss replay: coverage 0.9954, avg width +0.2205 versus Mondrian, worst defended source 0.9499.",,conformal;utility;decision -supplement,FinRegLab 2023 - Explainability and Fairness in ML Credit Underwriting.pdf,supplement/FinRegLab 2023 - Explainability and Fairness in ML Credit Underwriting.pdf,Explainability and Fairness in Machine Learning for Credit Underwriting: Policy Analysis,FinRegLab,2023,policy report,ML credit governance,finreglab2023fairness,existing,https://finreglab.org/research/explainability-fairness-in-machine-learning-for-credit-underwriting-policy-analysis/,local_pdf_text,106,61633,0,0,"Explainability, fairness, adverse action, model governance, underwriting controls.",ML credit systems require governance controls beyond predictive performance.,Policy analysis and stakeholder evidence.,Supports MRM/fairness appendix and explains why claims must be bounded.,Use control-taxonomy tables for governance mapping.,"Policy report, not CRPTO empirical evidence.",Governance appendix support.,Strong governance/fairness boundary source.,CRPTO governance appendix; agenda extendida CRPTO/tesis governance.,append_governance,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,governance;fairness;credit -supplement,Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf,Adaptive Conformal Inference Under Distribution Shift,Isaac Gibbs; Emmanuel J. Candes,2021,NeurIPS 2021,adaptive conformal inference,gibbs2021aci,existing,https://arxiv.org/abs/2106.00170,local_pdf_text,25,12059,11,0,"ACI, online updating, distribution shift, adaptive quantiles.",Conformal thresholds can be adapted under shift to improve long-run coverage.,Algorithm and experiments.,"Future online/drift candidate, not current static CRPTO.",Use online update schematic for agenda extendida CRPTO/tesis future work.,Needs streaming/prospective feedback to claim deployment validity.,Future-work caveat.,Online/source conformal lane context.,agenda extendida CRPTO/tesis online caveat; thesis.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;online;shift -supplement,Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf,Conformal Prediction with Conditional Guarantees,Isaac Gibbs; John J. Cherian; Emmanuel J. Candes,2025,JRSS-B,conditional conformal guarantees,gibbs2024,existing,https://arxiv.org/abs/2305.12616,local_pdf_text,54,28096,22,0,"Conditional guarantees, coverage spectrum, restricted conditional targets.",Relaxed conditional goals can be achieved where exact conditional coverage cannot.,Theory paper.,Supports future conditional-tightening language and source-group caveats.,Use conceptual spectrum for appendix if conditional validity is discussed.,"Not implemented in project; current Mondrian is finite-group conditional-ish, not exact individual conditional.",Caveat and future-work support.,Source/group coverage lane context.,agenda extendida CRPTO/tesis online/source caveat; CRPTO theory appendix.,append_claim_boundary,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;conditional_coverage -supplement,Guan 2023 - Localized Conformal Prediction.pdf,supplement/Guan 2023 - Localized Conformal Prediction.pdf,Localized Conformal Prediction,Leying Guan,2023,Biometrika,localized conformal prediction,guan2023localized,existing,https://doi.org/10.1093/biomet/asac040,local_pdf_text,29,16540,2,7,"Localized conformity scores, local calibration, approximate conditional validity.",Local weighting can improve adaptivity while retaining conformal inference structure.,Generalized local conformal framework.,Candidate comparator to Mondrian if future coverage sharpness matters.,Use local-vs-global schematic for appendix only.,Would require a new localized calibration design.,Future-work comparator.,Potential source/shift experiment.,agenda extendida CRPTO/tesis future gates; thesis.,append_mixed_diagnostic_wider_than_mondrian,experiment_completed_appendix_diagnostic,"Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; coverage and worst-source coverage improved, but width increased 0.0867 versus Mondrian.",docs/research/papers_tesis_deep_audit_2026-06-06.md,Append only as reviewer-facing localized diagnostic; no localized guarantee or champion replacement claim.,"Localized score-bin replay: coverage 0.9497, avg width 0.8643, worst defended source 0.9252.",,conformal;localized -supplement,Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf,Conformal Predictive Systems Under Covariate Shift,Sander Jonkers and coauthors,2024,arXiv / preprint,covariate shift conformal systems,jonkers2024wcps,needs_bib_if_cited,,local_pdf_text,18,6825,4,1,"Conformal predictive systems, covariate shift, calibration under shift.",Conformal predictive systems can be adapted for shifted covariate distributions.,Method paper.,"Useful for source/shift caveat, not current champion.",Use shift taxonomy only.,Requires shift assumptions and is not credit-specific.,Future-work context.,Source/shift lane context.,agenda extendida CRPTO/tesis source/shift appendix if expanded.,context_source_shift,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;shift -supplement,Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf,Decision Theoretic Foundations for Conformal Prediction,Shayan Kiyani and coauthors,2025,AISTATS / ICML-era preprint,decision theory for conformal prediction,kiyani2025,existing,https://arxiv.org/abs/2502.02561,local_pdf_text,28,17605,3,3,"Decision-theoretic CP, utility, actionability, uncertainty quantification.","Conformal sets can be understood through decisions and utilities, not only coverage.",Theoretical framing.,Good conceptual support for CRPTO's decision-aware framing.,Use as theory bridge in future-work paragraph.,Does not implement CRPTO portfolio optimization.,Future/positioning support.,Decision-risk lab framing.,CRPTO discussion; agenda extendida CRPTO/tesis future gates.,append_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;decision -supplement,Lekeufack et al 2023 - Conformal Decision Theory.pdf,supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf,Conformal Decision Theory,Jordan Lekeufack and coauthors,2023,NeurIPS workshop / preprint,conformal decision theory,lekeufack2023cdt,needs_bib_if_cited,,local_pdf_text,9,7335,4,0,"Conformal prediction for decisions, calibrated decision policies, set-valued actions.",Conformal uncertainty can be integrated with decision-making objectives.,Conceptual/method paper.,Supports agenda extendida CRPTO/tesis decision-risk framing but is not a CRPTO baseline.,Use only as conceptual context.,No credit portfolio evidence.,Future-work context.,Decision analytics context.,agenda extendida CRPTO/tesis related work if expanded.,context_decision_conformal,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;decision -supplement,Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf,Conformal Inverse Optimization,Bo Lin; Erick Delage; Timothy C. Y. Chan,2024,NeurIPS 2024,conformal inverse optimization,chan2024inverse,existing,https://arxiv.org/abs/2402.01489,local_pdf_text,24,12919,5,2,"Inverse optimization, conformal uncertainty, decision ambiguity.",Conformal methods can quantify uncertainty in inverse optimization.,Theory/algorithm.,"Valuable frontier for auditing human/legacy decisions, not current CRPTO.",Use CREDO/inverse-optimization family as future audit lane.,Different problem: inferring preferences/parameters from decisions.,Future-work context.,Decision audit/source governance context.,agenda extendida CRPTO/tesis future gates.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;inverse_optimization -supplement,Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf,Multi-Source Conformal Inference Under Distribution Shift,Meichen Liu; Alexander W. Levis; Sara-Lynne Normand; Larry Han,2024,arXiv preprint,multi-source conformal inference,liu2024multisource,needs_bib_if_cited,https://arxiv.org/abs/2402.12105,local_pdf_text,39,23862,15,9,"Multi-source inference, heterogeneous source distributions, shifted test populations.",Combining calibration sources carefully can maintain valid intervals under source shift.,Theory and experiments.,Directly relevant to agenda extendida CRPTO/tesis source-family holdout but needs stronger data design.,Use source-combination taxonomy for source governance.,"Current LC source groups are retrospective, not validated external sources.",Caveat/future work only.,Source-family holdout lane.,agenda extendida CRPTO/tesis online/source caveat.,append_extended_source_shift,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;source;shift -supplement,Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf,Online Conformal Prediction via Universal Portfolio Algorithms,Tuo Liu; Edgar Dobriban; Francesco Orabona,2026,arXiv preprint,online conformal prediction,liu2026portfolio,existing,https://arxiv.org/abs/2602.03168,local_pdf_text,63,23587,66,11,"Online conformal prediction, universal portfolios, regret-to-coverage, parameter-free adaptation.",Online CP can achieve long-run coverage via universal portfolio algorithms.,Theory and experiments.,"Excellent future direction for deployment, but current project is retrospective.",Use online update process as future-work schematic.,No production feedback stream in current data.,Future-work only.,Online lane context and stop-rule support.,agenda extendida CRPTO/tesis online caveat; CRPTO future work.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;online -supplement,Mandi et al 2024 - Decision-Focused Learning Survey.pdf,supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf,"Decision-Focused Learning: Foundations, State of the Art, Benchmark and Future Opportunities",Jayanta Mandi and coauthors,2024,Journal of Artificial Intelligence Research,decision-focused learning survey,mandi2024,existing,https://arxiv.org/abs/2307.13565,local_pdf_text,76,34467,24,3,"DFL taxonomy, SPO+, gradient-based/gradient-free methods, benchmarks.",DFL is mature but no single method dominates across tasks.,Survey and benchmark synthesis.,"Use to defend treating SPO+/PyEPO as comparator, not CRPTO replacement.",Use taxonomy table for agenda extendida CRPTO/tesis DFL suite framing.,Survey does not provide conformal guarantees.,Comparator framing.,Core PyEPO/DFL suite context.,CRPTO related work; agenda extendida CRPTO/tesis PyEPO section.,append_comparator,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,dfl;survey -supplement,Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf,Optimization of Conditional Value-at-Risk,R. Tyrrell Rockafellar; Stanislav Uryasev,2000,The Journal of Risk,CVaR optimization,rockafellar2000cvar,existing,https://doi.org/10.21314/JOR.2000.038,local_pdf_text,26,8861,1,9,"CVaR, tail risk, tractable convex optimization.",CVaR is optimizable and more useful than VaR for tail-risk control.,Optimization theory.,Grounds A12 and agenda extendida CRPTO/tesis tail challenger.,"Use formulation, not figures.",Tail-risk improvement does not imply wealth champion.,Appendix tail-risk diagnostic.,Strong CVaR challenger foundation.,CRPTO appendix; agenda extendida CRPTO/tesis tail lane.,append_tail_risk,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,tail_risk;cvar -supplement,Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf,Conformalized Quantile Regression,Yaniv Romano; Evan Patterson; Emmanuel J. Candes,2019,NeurIPS 2019,CQR,romano2019,existing,https://arxiv.org/abs/1905.03222,local_pdf_text,19,8886,9,2,"Conformalized quantile regression, adaptive intervals, split conformal.",Quantile regression plus conformalization yields valid adaptive intervals.,Method and experiments.,Baseline/variant reference for interval adaptivity.,Use interval-width/adaptivity examples in conformal chapter.,"Current champion is Mondrian score-decile, not CQR.",Conformal variant/context.,Potential future variant only.,Conformal chapter; CRPTO appendix.,append_conformal_variant,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;cqr -supplement,Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf,Robust Losses for Decision-Focused Learning,Wolfgang Schutte and coauthors,2024,NeurIPS / arXiv-era DFL paper,robust DFL,schutte2024robust,needs_bib_if_cited,,local_pdf_text,10,9002,4,1,"Robust decision-focused losses, decision regret, misspecification.",Robust losses can improve DFL under uncertainty or misspecification.,DFL method and experiments.,agenda extendida CRPTO/tesis DFL challenger context; not a CRPTO body claim.,Use loss comparison table for DFL appendix only.,No conformal auditability or credit-specific bound.,Comparator/future work.,Potential DFL suite extension.,agenda extendida CRPTO/tesis PyEPO/DFL appendix if expanded.,park_future_dfl,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,dfl;robustness -supplement,Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf,supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf,Conformal Prediction Under Covariate Shift,Ryan J. Tibshirani; Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas,2019,NeurIPS 2019,weighted conformal / covariate shift,tibshirani2019covshift,needs_bib_if_cited,https://arxiv.org/abs/1904.06019,local_pdf_text,11,6021,1,0,"Covariate shift, weighted conformal prediction, likelihood-ratio weights.",Conformal coverage can adapt under covariate shift if weights are known/estimated.,Theory and experiments.,Core source for future source/shift gates.,Use assumption table for source-shift appendix.,Requires credible density-ratio/weight estimation.,Future/caveat only.,Source/shift conformal lane.,agenda extendida CRPTO/tesis online/source caveat.,append_extended_source_shift,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;shift -supplement,Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf,Multi-Distribution Robust Conformal Prediction,Yuqi Yang; Ying Jin,2026,arXiv preprint,multi-distribution robust conformal prediction,yang2026multidistribution,existing,https://arxiv.org/abs/2601.02998,local_pdf_text,45,26846,26,0,"Multiple source distributions, robust coverage, max-p aggregation.",Finite-sample coverage can be made robust over multiple distributions/mixtures.,Theory preprint.,Strong source-robustness future-work citation.,Use source distribution diagram if agenda extendida CRPTO/tesis source appendix expands.,"Current project has retrospective source proxies, not true multi-source deployment.",Future-work context.,Source-governance lane and caveat.,CRPTO future work; agenda extendida CRPTO/tesis source caveat.,append_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;source;shift -supplement,Yeh et al 2025 - Conformal Risk Training.pdf,supplement/Yeh et al 2025 - Conformal Risk Training.pdf,Conformal Risk Training,Christopher Yeh; Nicolas Christianson; Adam Wierman; Yisong Yue,2025,NeurIPS 2025,end-to-end conformal risk training,yeh2025training,existing,https://arxiv.org/abs/2510.08748,local_pdf_text,30,18537,5,3,"Conformal risk training, OCE/CVaR, differentiable conformal risk.","Conformal risk can be optimized end-to-end, including OCE-style risks.",Method and experiments.,Future direction for replacing diagnostic tail risk with optimized conformal risk.,Use risk-training diagram as future-work context only.,Current CRPTO is post-hoc; implementing CRT would change method.,Future-work contrast.,Candidate composite decision-risk training lane.,CRPTO future work; agenda extendida CRPTO/tesis future gates.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;risk_control;end_to_end -supplement,Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf,End-to-End Conformal Calibration for Optimization Under Uncertainty,Christopher Yeh; Nicolas Christianson; Alan Wu; Adam Wierman; Yisong Yue,2025,TMLR; arXiv v2 in 2026,end-to-end conformal calibration for optimization,yeh2026,existing,https://arxiv.org/abs/2409.20534,local_pdf_text,29,16242,10,4,"End-to-end conformal calibration, downstream optimization, learned uncertainty sets.",Calibration can be optimized for decision usefulness while retaining validity.,Method and experiments.,Confirms future direction but not current post-hoc CRPTO.,Use related-work contrast table only.,Different training stack and not credit-specific.,Future-work frontier.,Lane 3/end-to-end context.,CRPTO future work; agenda extendida CRPTO/tesis future gates.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;end_to_end -supplement,Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf,Conformalized Decision Risk Assessment,Wenbin Zhou; Agni Orfanoudaki; Shixiang Zhu,2026,ICLR 2026 / arXiv 2025,decision risk certificates,zhou2025credo,added_2026_06_06,https://arxiv.org/abs/2505.13243,web_primary_and_local_pdf,41,18592,9,10,"CREDO, decision risk certificates, inverse optimization, conformalized risk estimation.",A candidate decision can receive a distribution-free upper bound on probability of suboptimality.,Conformal + inverse optimization framework.,Very relevant for agenda extendida CRPTO/tesis auditability; not part of current CRPTO champion.,Use risk-certificate diagram as agenda extendida CRPTO/tesis future audit inspiration.,Different problem geometry; no Lending Club implementation.,Future auditability contrast.,Strong future lane for decision certificate rather than policy promotion.,agenda extendida CRPTO/tesis future gate register.,park_future_work,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;decision;auditability -supplement,Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf,Calibrating Decision Robustness via Inverse Conformal Risk Control,Yang Zhou; Ruohan Zhu,2026,arXiv preprint,inverse conformal risk control,zhou2026creme,existing,https://arxiv.org/abs/2510.07750,local_pdf_text,20,12634,11,2,"Inverse CRC, decision robustness, robustness calibration.",Decision robustness can be calibrated through inverse conformal risk-control ideas.,Theory/method preprint.,Relevant future lane for robustness calibration.,Use only as future-work contrast.,Not implemented and not credit-specific.,Future-work context.,Decision robustness gate candidate.,agenda extendida CRPTO/tesis future gates.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;robustness -tesis,Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf,A multi-objective instance-based decision support system for investment recommendation in peer-to-peer lending,Gholamreza Babaei; Shahrooz Bamdad,2020,Journal article,P2P investment recommendation,babaei2020p2p,needs_bib_if_cited,,local_pdf_text,11,11155,5,5,"P2P lending, multi-objective recommendation, risk-return tradeoff, NPV.",Decision-support systems can optimize investment recommendations over risk and return.,Applied recommendation/optimization experiment.,Useful thesis context for digital lending optimization.,Use risk-return recommendation setup as background only.,No conformal guarantees.,Credit-domain context.,Digital-lending optimization context.,Thesis/agenda extendida CRPTO/tesis context.,context_credit_domain,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;p2p;optimization -tesis,Basel Committee 2015 - Guidance on Credit Risk and Expected Credit Losses.pdf,tesis/Basel Committee 2015 - Guidance on Credit Risk and Expected Credit Losses.pdf,Guidance on credit risk and accounting for expected credit losses,Basel Committee on Banking Supervision,2015,official supervisory guidance,ECL / IFRS9 governance,basel2015ecl,added_2026_06_06,https://www.bis.org/bcbs/publ/d350.htm,local_pdf_text,36,19933,0,0,"Expected credit losses, credit risk governance, forward-looking information, controls.","ECL estimation needs governance, data quality, forward-looking information and controls.",Supervisory guidance.,Supports IFRS9-inspired boundary and why contractual IFRS9 is not claimed.,Use guidance checklist for agenda extendida CRPTO/tesis IFRS9 proxy appendix.,Open Lending Club data lacks contractual monthly DPD/EAD/recovery infrastructure.,Not for CRPTO body.,Strong IFRS9/SICR proxy boundary source.,agenda extendida CRPTO/tesis IFRS9 appendix; thesis.,append_extended_ifrs9,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,ifrs9;governance -tesis,Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf,Robust Optimization,Aharon Ben-Tal; Laurent El Ghaoui; Arkadi Nemirovski,2009,Princeton University Press book,robust optimization foundations,bental2009robust,needs_bib_if_cited,https://press.princeton.edu/books/hardcover/9780691143682/robust-optimization,local_pdf_text,570,232048,42,32,"Uncertainty sets, robust counterparts, tractability, convex robust optimization.",Robust optimization provides a systematic language for feasible decisions under uncertainty.,Foundational monograph.,Use as deep thesis foundation; CRPTO body can rely on shorter RO citations.,"Use formulations/definitions, not figures.",Very broad; not conformal or credit-specific.,"Theoretical background for thesis, optional paper appendix.",Context only.,Thesis robust optimization chapter.,thesis_foundation,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,robust_optimization;book -tesis,Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf,How Costly is Noise? Data and Disparities in Consumer Credit,Laura Blattner; Scott Nelson,2021,working paper / credit economics,credit data noise / disparities,blattner2021noise,needs_bib_if_cited,,local_pdf_text,86,32376,5,17,"Data noise, credit scores, disparities, unequal information quality.",Noisy credit data can create unequal access and distort credit allocations.,Empirical/economic analysis.,Important governance caveat: model performance can be constrained by source data quality.,Use disparity/noise tables as governance appendix inspiration.,Not Lending Club CRPTO evidence.,Threat-to-validity/context.,Strong metric/data governance context.,agenda extendida CRPTO/tesis governance appendix; thesis.,append_extended_governance,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;data_quality;equity -tesis,Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf,Credit Invisibles and the Unscored,Kenneth P. Brevoort; Philipp Grimm; Michelle Kambara,2016,Cityscape / CFPB research,credit invisibles / unscored consumers,brevoort2016creditinvisibles,needs_bib_if_cited,,local_pdf_text,44,10194,7,6,"Credit invisibles, unscored consumers, data availability, inclusion.",Large consumer segments lack conventional scores or have weak scoring coverage.,Consumer credit data analysis.,Supports why open accepted-loan data cannot make broad credit-access claims.,Use population breakdowns as motivation only.,Not a CRPTO method source.,Credit-market context and scope limitation.,Fairness/source governance context.,Thesis/agenda extendida CRPTO/tesis governance.,context_governance,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;inclusion;governance -tesis,Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf,Conformal Prediction Sets Improve Human Decision Making,Jesse Cresswell and coauthors,2024,preprint / empirical decision support,human decision support with CP,cresswell2024,needs_bib_if_cited,,local_pdf_text,19,11537,12,7,"Human-AI decision making, prediction sets, uncertainty communication.",Conformal sets can improve human decisions when uncertainty is communicated well.,Human-subjects/decision experiment.,Useful for discussing auditability and committee-facing uncertainty.,Use decision-support figures as inspiration for MRM/committee communication.,Not credit portfolio optimization.,Motivation for uncertainty-as-decision input.,Governance/human decision appendix context.,agenda extendida CRPTO/tesis governance; thesis discussion.,context_decision_support,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;human_decision;governance -tesis,Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf,Label Noise Robustness of Conformal Prediction,Bat-Sheva Einbinder; Shai Feldman; Stephen Bates; Anastasios N. Angelopoulos; Asaf Gendler; Yaniv Romano,2024,Journal of Machine Learning Research,label noise / conformal robustness,einbinder2024labelnoise,existing,https://jmlr.org/papers/v25/23-1549.html,local_pdf_text,66,24071,29,2,"Label noise, conformal robustness, noisy outcomes, validity under imperfect labels.",Conformal prediction has robustness properties under forms of label noise.,Theory and experiments.,Useful threat-to-validity note for default labels and hardened shift tests.,Use noise taxonomy for appendix caveat.,Does not validate all Lending Club label issues.,Threat-to-validity support.,Data/label governance appendix.,CRPTO limitations; agenda extendida CRPTO/tesis governance.,append_claim_boundary,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;label_noise -tesis,Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf,Predictably Unequal? The Effects of Machine Learning on Credit Markets,Andreas Fuster; Paul Goldsmith-Pinkham; Tarun Ramadorai; Ansgar Walther,2022,Journal of Finance,ML credit markets / equity,fuster2022predictably,added_2026_06_06,https://doi.org/10.1111/jofi.13090,web_primary_and_local_pdf,92,32099,7,0,"ML credit screening, distributional impacts, mortgage markets, disparities.",More flexible ML can change rate disparities and who benefits from credit-market technology.,Theory and empirical mortgage-market analysis.,Critical fairness/equity context: better predictive technology does not automatically imply equitable outcomes.,Use disparity mechanism figures/tables in governance discussion.,"Mortgage market, not Lending Club; no protected attributes in current project.","Motivation/caveat, not method evidence.",Strong governance/fairness boundary source.,agenda extendida CRPTO/tesis governance appendix; CRPTO ethics/motivation if space permits.,append_extended_governance,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;equity;ml -tesis,Kato 2024 - Conformal Predictive Portfolio Selection.pdf,tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf,Conformal Predictive Portfolio Selection,Masahiro Kato,2025,arXiv preprint,conformal portfolio selection,kato2025,existing,https://arxiv.org/abs/2410.16333,local_pdf_text,14,4913,3,2,"Portfolio selection, conformal predictive sets, financial decisions.",Conformal prediction can support portfolio selection under uncertainty.,Finance/portfolio method preprint.,Relevant neighbor in finance; CRPTO differs by credit PD/funded-set risk.,Use as finance-related-work table.,Not credit lending and not Lending Club.,Related work context.,Finance portfolio context.,CRPTO related work if finance CP paragraph expands.,context_finance_cp,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;portfolio -tesis,Kawasumi Kato Duan 2026 - Conformal Prediction for Ordinal Credit Scoring.pdf,tesis/Kawasumi Kato Duan 2026 - Conformal Prediction for Ordinal Credit Scoring.pdf,Conformal Prediction for Ordinal Credit Scoring,Ryotaro Kawasumi; Masahiro Kato; Yaqing Duan,2026,arXiv preprint,ordinal credit scoring / conformal prediction,kawasumi2026ordinal,needs_bib_if_cited,,local_pdf_text,8,2835,0,0,"Ordinal credit scoring, conformal prediction, rating categories.",Conformal methods can be adapted to ordinal credit-score outputs.,Short method preprint.,"Useful for future grade/rating-set extension, not current PD interval CRPTO.",Use only as future-work note.,No portfolio optimization and very short current evidence.,Future credit conformal context.,Ordinal score/rating future lane.,Thesis future work.,park_future_work,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;credit;ordinal -tesis,Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf,Consumer Credit-Risk Models via Machine-Learning Algorithms,Amir E. Khandani; Adlar J. Kim; Andrew W. Lo,2010,Journal of Banking & Finance,consumer credit ML,khandani2010consumer,added_2026_06_06,https://doi.org/10.1016/j.jbankfin.2010.06.001,web_primary_and_local_pdf,49,14412,21,9,"Consumer credit risk, nonlinear ML, transaction data, economic value of risk forecasts.",ML forecasts can materially improve consumer credit-risk management and economic decisions.,Empirical credit-card portfolio study.,"Classic support for economic evaluation of credit ML, not just classification metrics.",Use economic-benefit tables as background for decision-value framing.,"Credit cards/bank data, not LC/CRPTO.",Historical credit-ML decision-value context.,Metric/economic governance context.,CRPTO motivation or thesis background.,context_credit_domain,completed_bib_quarto_patch,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,credit;ml;economic_value -tesis,Noguer i Alonso 2024 - Conformal Portfolio Optimization.pdf,tesis/Noguer i Alonso 2024 - Conformal Portfolio Optimization.pdf,Conformal Portfolio Optimization,Miquel Noguer i Alonso,2024,SSRN preprint,conformal portfolio optimization,noguer2024portfolio,existing,https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5011129,local_pdf_text,6,1812,0,0,"Conformal prediction, portfolio optimization, finance uncertainty.",Conformal intervals can support portfolio decisions in financial assets.,Short finance preprint.,"Finance-adjacent context, but CRPTO's credit/funded-set bound is different.",Use only as related-work mention.,"Short, non-credit, not a robust credit allocation benchmark.",Related work context.,Finance portfolio context.,Related work if finance CP paragraph expands.,context_finance_cp,none_now,No run: literature integration only.,docs/research/papers_tesis_deep_audit_2026-06-06.md,Do not reopen a lane unless it changes a manuscript claim.,,,conformal;portfolio diff --git a/reports/crpto/multidataset/source/freddie_exhaustiveness_certificate.json b/reports/crpto/multidataset/source/freddie_exhaustiveness_certificate.json deleted file mode 100644 index ce44fd8..0000000 --- a/reports/crpto/multidataset/source/freddie_exhaustiveness_certificate.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "freddie_all_lp_solved": true, - "freddie_all_robust_funded": 143, - "freddie_all_robust_funded_outside_top250k": 0, - "freddie_all_robust_max_funded_rank": 551, - "freddie_available_candidates": 1396053 -} diff --git a/reports/crpto/multidataset/source/source_log.md b/reports/crpto/multidataset/source/source_log.md deleted file mode 100644 index 85b20b3..0000000 --- a/reports/crpto/multidataset/source/source_log.md +++ /dev/null @@ -1,19 +0,0 @@ -# Multidataset External Replication Source Log - -This folder contains only curated, local CSV summaries used by the paper and book. -It intentionally does not contain credentials, browser-session references, or paths to the exploratory laboratory. - -## Public Dataset Sources - -- Prosper loan-level data access documentation: https://help.prosper.com/hc/en-us/articles/210013083-Where-can-I-download-data-about-loans-through-Prosper -- Freddie Mac Single Family Loan-Level Dataset: https://www.freddiemac.com/research/datasets/sf-loanlevel-dataset -- Freddie/Mendeley processed mortgage windows: https://data.mendeley.com/datasets/bzr2rxttvz/3 -- Home Credit Default Risk page, archived only and not used in the main external claim: https://www.kaggle.com/competitions/home-credit-default-risk/data - -## Editorial Decision - -Prosper final-status loans and Freddie FM48 are reported as external economic replications. Home Credit is discarded from the IJDS main claim because it lacks a clean investment-return and exposure contract comparable to Lending Club, Prosper, and Freddie. - -## Extended Audit Layer - -A28 solves the Freddie FM48 LP on the full OOT candidate universe and documents that the all-candidate optimum funds only loans inside the top-return screen. A29 isolates sparse Mondrian groups. A30--A33 report confidence intervals, OOT subperiods, Prosper default-definition sensitivity, and Freddie red/green segment sensitivity. diff --git a/reports/crpto/multidataset/source/table_external_baselines_compact.csv b/reports/crpto/multidataset/source/table_external_baselines_compact.csv deleted file mode 100644 index d899a65..0000000 --- a/reports/crpto/multidataset/source/table_external_baselines_compact.csv +++ /dev/null @@ -1,5 +0,0 @@ -dataset,candidate_cap,n_candidates,robust_objective,nonrobust_objective,price_of_robustness_pct,robust_n_funded,robust_max_funded_rank -prosper,all,10531,199419.0887756553,220259.86083869895,0.0946190194785686,234,508 -freddie_fm48,500000,500000,1291227.5801824057,1305409.0552784826,0.0108636254963404,143,551 -freddie_fm48,1000000,1000000,1291227.5801824057,1305409.0552784826,0.0108636254963404,143,551 -freddie_fm48,all,1396053,1291227.5801824057,1305409.0552784826,0.0108636254963404,143,551 diff --git a/reports/crpto/multidataset/source/table_external_lp_exhaustiveness.csv b/reports/crpto/multidataset/source/table_external_lp_exhaustiveness.csv deleted file mode 100644 index b6cff77..0000000 --- a/reports/crpto/multidataset/source/table_external_lp_exhaustiveness.csv +++ /dev/null @@ -1,5 +0,0 @@ -dataset,candidate_cap,available_candidates,n_candidates,robust_objective,nonrobust_objective,price_of_robustness_abs,price_of_robustness_pct,robust_objective_boot_low,robust_objective_boot_high,nonrobust_objective_boot_low,nonrobust_objective_boot_high,robust_total_allocated,robust_n_funded,robust_max_funded_rank,robust_solver_success,nonrobust_solver_success -prosper,all,10531,10531,199419.0887756553,220259.86083869895,20840.772063043667,0.0946190194785686,186546.04708826495,213330.69865323557,205795.96550527308,236437.5902917046,1000000.0,234,508,True,True -freddie_fm48,500000,1396053,500000,1291227.5801824057,1305409.0552784826,14181.47509607696,0.0108636254963404,1062741.9024937386,1535012.3146356042,1072985.5163668434,1549941.0256744812,25000000.0,143,551,True,True -freddie_fm48,1000000,1396053,1000000,1291227.5801824057,1305409.0552784826,14181.47509607696,0.0108636254963404,1062741.9024937386,1535012.3146356042,1073319.0027516496,1546644.5177656687,25000000.0,143,551,True,True -freddie_fm48,all,1396053,1396053,1291227.5801824057,1305409.0552784826,14181.47509607696,0.0108636254963404,1062741.9024937386,1535012.3146356042,1072985.5163668434,1549941.0256744812,25000000.0,143,551,True,True diff --git a/reports/crpto/multidataset/source/table_external_metric_intervals.csv b/reports/crpto/multidataset/source/table_external_metric_intervals.csv deleted file mode 100644 index 1b503be..0000000 --- a/reports/crpto/multidataset/source/table_external_metric_intervals.csv +++ /dev/null @@ -1,9 +0,0 @@ -dataset,metric,estimate,ci_low,ci_high,method,n -Prosper,auc_roc,0.7072982821805249,0.6955733258816219,0.719023238479428,Hanley-McNeil large-sample approximation,10531 -Prosper,coverage_90,0.920425410692242,0.9151019142826232,0.9254422965261512,Wilson binomial interval,10531 -Prosper,coverage_alpha01,0.9943025353717596,0.9926737771462196,0.995570805432002,Wilson binomial interval,10531 -Prosper,robust_objective,199419.0887756553,186546.04708826495,213330.69865323557,Funded-loan contribution bootstrap; solver input uncertainty is not resampled,234 -Freddie FM48,auc_roc,0.7838634726333499,0.7799265563524038,0.7878003889142959,Hanley-McNeil large-sample approximation,1396053 -Freddie FM48,coverage_90,0.974523173547136,0.9742604888155064,0.974783246836056,Wilson binomial interval,1396053 -Freddie FM48,coverage_alpha01,0.9907116706887202,0.9905511894862964,0.9908694513583008,Wilson binomial interval,1396053 -Freddie FM48,robust_objective,1291227.5801824057,1062741.9024937386,1535012.3146356042,Funded-loan contribution bootstrap; solver input uncertainty is not resampled,143 diff --git a/reports/crpto/multidataset/source/table_external_subperiod_metrics.csv b/reports/crpto/multidataset/source/table_external_subperiod_metrics.csv deleted file mode 100644 index 715ce3f..0000000 --- a/reports/crpto/multidataset/source/table_external_subperiod_metrics.csv +++ /dev/null @@ -1,7 +0,0 @@ -dataset,period,n,defaults,default_rate,auc_roc,pr_auc,coverage_90,coverage_alpha01,economic_candidates -Prosper,2012,8010,2577,0.3217228464419475,0.6988646610585748,0.4888868352024403,0.9116104868913858,0.9933832709113608,8010 -Prosper,2013,2521,316,0.1253470844902816,0.687683343379546,0.221384282793829,0.9484331614438716,0.9972233240777468,2521 -Freddie FM48,2015Q1,346682,3906,0.0112668093526632,0.7962807462700461,0.0507110650457581,0.9785480642202364,0.991992661863033,346682 -Freddie FM48,2015Q2,389075,4804,0.0123472338238128,0.7823109517507605,0.0491288479707842,0.9774413673456276,0.9910043050825676,389075 -Freddie FM48,2015Q3,345167,4975,0.0144133129760376,0.7736149116384292,0.0506352703987166,0.9707793618741072,0.990080164094482,345167 -Freddie FM48,2015Q4,315129,4867,0.0154444687731056,0.7791377151096768,0.0573174166539526,0.9705929952495644,0.9896328170368326,315129 diff --git a/reports/crpto/multidataset/source/table_freddie_mondrian_eligible_summary.csv b/reports/crpto/multidataset/source/table_freddie_mondrian_eligible_summary.csv deleted file mode 100644 index 59a9c6a..0000000 --- a/reports/crpto/multidataset/source/table_freddie_mondrian_eligible_summary.csv +++ /dev/null @@ -1,4 +0,0 @@ -scope,n_groups,min_coverage_90,min_coverage_alpha01,test_rows -all_groups,29,0.5,0.5,1396053 -eligible_cal_and_test_ge_500,25,0.8853734439834025,0.9565383149065956,1396010 -sparse_or_fallback_groups,4,0.5,0.5,43 diff --git a/reports/crpto/multidataset/source/table_freddie_mondrian_group_audit.csv b/reports/crpto/multidataset/source/table_freddie_mondrian_group_audit.csv deleted file mode 100644 index 98d83e3..0000000 --- a/reports/crpto/multidataset/source/table_freddie_mondrian_group_audit.csv +++ /dev/null @@ -1,30 +0,0 @@ -risk_group,test_count,test_defaults,test_default_rate,calibration_count,fallback_90,fallback_alpha01,coverage_90,coverage_alpha01,eligible_min_cal_test -green|nan|ltv_gt90,2,1,0.5,1,True,True,0.5,0.5,False -green|nan|ltv_61_80,28,0,0.0,15,True,True,1.0,1.0,False -green|nan|ltv_81_90,2,0,0.0,0,True,True,1.0,1.0,False -green|nan|ltv_le60,11,0,0.0,4,True,True,1.0,1.0,False -red|lt620|nan,1928,232,0.1203319502074688,1581,False,False,0.8853734439834025,0.9813278008298756,True -red|lt620|ltv_gt90,1095,118,0.1077625570776255,972,False,False,0.8968036529680365,0.9744292237442922,True -red|lt620|ltv_81_90,1572,154,0.0979643765903307,1105,False,False,0.9083969465648856,0.9885496183206108,True -red|lt620|ltv_61_80,3895,324,0.083183568677792,1878,False,False,0.9170731707317074,0.9635430038510912,True -red|620_679|nan,5111,433,0.0847192330268049,4784,False,False,0.9176286441009588,0.9733907258853454,True -red|lt620|ltv_le60,2623,181,0.069004956157072,1108,False,False,0.9287075867327488,0.9565383149065956,True -red|620_679|ltv_gt90,13805,875,0.0633828323071351,3570,False,False,0.9366171676928648,0.9808040565012676,True -red|620_679|ltv_81_90,13473,784,0.0581904549840421,4332,False,False,0.9397313144808136,0.9785496919765456,True -red|680_739|nan,6604,333,0.0504239854633555,14179,False,False,0.9495760145366444,0.9736523319200484,True -red|620_679|ltv_61_80,60013,2594,0.0432239681402362,20977,False,False,0.9505273857330911,0.980870811324213,True -red|620_679|ltv_le60,23525,763,0.032433581296493,10608,False,False,0.9605100956429332,0.979893730074389,True -red|740_plus|nan,692,25,0.0361271676300578,2384,False,False,0.963872832369942,0.976878612716763,True -green|740_plus|nan,6261,197,0.0314646222648139,21699,False,False,0.967257626577224,0.9776393547356652,True -red|680_739|ltv_gt90,70954,1973,0.0278067480339374,17324,False,False,0.971164416382445,0.9841728443780478,True -red|680_739|ltv_61_80,206768,3294,0.0159308983982047,96000,False,False,0.9757699450591968,0.9867049059815832,True -green|740_plus|ltv_61_80,415576,1747,0.0042038038770285,299681,False,False,0.9759971701926964,0.9952427474156352,True -red|680_739|ltv_81_90,58094,1180,0.02031190828657,23873,False,False,0.9762281819120736,0.9860398664233828,True -green|740_plus|ltv_le60,174915,480,0.0027441900351599,193923,False,False,0.976274190321013,0.996661235457222,True -red|680_739|ltv_le60,59626,641,0.0107503438097474,40332,False,False,0.978901821353101,0.9892328849830612,True -red|740_plus|ltv_gt90,13118,210,0.016008537886873,3304,False,False,0.982314377191645,0.9852111602378412,True -green|740_plus|ltv_81_90,99169,617,0.0062217023464994,59010,False,False,0.9833214008409886,0.9937782976535006,True -red|740_plus|ltv_61_80,34448,298,0.008650719925685,19875,False,False,0.984382257315374,0.9913492800743148,True -green|740_plus|ltv_gt90,102486,930,0.0090744101633393,33728,False,False,0.9851882208301622,0.9918427882832778,True -red|740_plus|ltv_81_90,10709,110,0.0102717340554673,4957,False,False,0.9874871603324308,0.990942198151088,True -red|740_plus|ltv_le60,9550,58,0.0060732984293193,7447,False,False,0.9881675392670156,0.993821989528796,True diff --git a/reports/crpto/multidataset/source/table_freddie_segment_sensitivity.csv b/reports/crpto/multidataset/source/table_freddie_segment_sensitivity.csv deleted file mode 100644 index 74b7afd..0000000 --- a/reports/crpto/multidataset/source/table_freddie_segment_sensitivity.csv +++ /dev/null @@ -1,4 +0,0 @@ -segment,rows_total,rows_test,default_rate,auc_roc,pr_auc,coverage_90,min_group_coverage_90,alpha01_coverage,coverage90_pass,alpha01_pass,available_oot_candidates,robust_objective_all_candidates,nonrobust_objective_all_candidates,price_of_robustness_pct,all_lp_solved,max_funded_rank,funded_outside_top250k -both,3173355,1396053,0.014543598179214112,0.7838634726333499,0.05178220512695661,0.9745231735471361,0.5,0.9907116706887202,True,True,1396053,1291227.5801824057,1305409.0552784826,0.0108636254963404,True,551,0 -red,1158306,597603,0.0297209890995989,0.700087721623812,0.05952354487123149,0.9705289297409819,0.891597510373444,0.9850218288730144,True,False,597603,1236591.1430666302,1266595.1411772007,0.0236887045711261,True,155,0 -green,2015049,798450,0.005819213329303655,0.6997060916716304,0.013258970035509824,0.98100319368777,0.5,0.9950629344354687,True,True,798450,1287497.9769414272,1300554.7006127096,0.0100393498751964,True,745,0 diff --git a/reports/crpto/multidataset/source/table_freddie_window_sensitivity.csv b/reports/crpto/multidataset/source/table_freddie_window_sensitivity.csv deleted file mode 100644 index 8c477f8..0000000 --- a/reports/crpto/multidataset/source/table_freddie_window_sensitivity.csv +++ /dev/null @@ -1,8 +0,0 @@ -run_label,variant,window_months,rows_total,default_rate,auc_roc,pr_auc,coverage_90,min_group_coverage_90,coverage_alpha01,coverage90_pass,alpha01_pass,robust_objective,price_of_robustness_abs,recommendation -freddie_fm12_both,both,12.0,2849457,0.0059523621518064,0.7681863221355728,0.0222124268822717,0.7578453348849258,0.0,0.9774174647316434,False,False,1287040.6418207984,9570.992353857262,sensitivity only -freddie_fm24_both,both,24.0,2420190,0.0103363785487916,0.7800044357516288,0.0422297603093363,0.9369215610110188,0.8736444790605715,0.9889295157422392,True,False,1118619.780863266,8985.721653965535,sensitivity only -freddie_fm36_both,both,36.0,3581402,0.0114142450358825,0.7890898429830087,0.0496332913708696,0.9056827296391612,0.8571428571428571,0.9871002504662412,True,False,1042378.4068574588,11429.115930069587,sensitivity only -freddie_fm36_green,green,36.0,2383971,0.0047483799089837,0.7143282449429617,0.0122747793777575,0.8874942306290841,0.8787338525538769,0.9873205421994056,False,False,1035323.9699910176,6277.701643785229,sensitivity only -freddie_fm36_red,red,36.0,1197431,0.0246853472141609,0.7047998404460994,0.0575720961155313,0.9082773067081432,0.8401844532279314,0.9853776779890624,True,False,990317.6758330629,23641.45389096951,sensitivity only -freddie_fm48_both,both,48.0,3173355,0.0145435981792141,0.7838634726333499,0.0517822051269566,0.974523173547136,0.5,0.9907116706887202,True,True,1060065.9648987832,17947.475405741716,selected main external mortgage replication -freddie_fm60_both,both,60.0,3579151,0.0218331107013925,0.7270254323985861,0.0728705047312328,0.959101839256819,0.5,0.9822399292863524,True,False,1033755.5253137936,22943.455865225056,sensitivity only diff --git a/reports/crpto/multidataset/source/table_full_multidataset_gate_audit.csv b/reports/crpto/multidataset/source/table_full_multidataset_gate_audit.csv deleted file mode 100644 index 882701c..0000000 --- a/reports/crpto/multidataset/source/table_full_multidataset_gate_audit.csv +++ /dev/null @@ -1,10 +0,0 @@ -run_label,dataset_id,variant,window_months,role_recommendation,rows_total,default_rate,auc_roc,pr_auc,brier,coverage_90,coverage90_pass,min_group_coverage_90,width_90,coverage_alpha01,alpha01_pass,portfolio_status,robust_objective,price_of_robustness_abs,price_of_robustness_pct -home_credit_full,home_credit_default,canonical,,supplement,307511,0.0807288194568649,0.7520619425836951,0.2230199950608295,0.0684516100542153,0.9014264654873396,True,0.8994388182932336,0.2504784293937966,0.9922389871661464,True,,,, -prosper_full,prosper_final_status,canonical,,main,54807,0.3091758352035324,0.7074290432845566,0.4484817219126147,0.1781741704802637,0.9205159821682634,True,0.8850308641975309,0.858549572083799,0.9943090202029784,True,completed,143939.61221838504,25092.56128704615,0.1484484330211839 -freddie_fm36_red,freddie_fm36_v2,red,36.0,sensitivity,1197431,0.0246853472141609,0.7047998404460994,0.0575720961155313,0.0237280406622907,0.9082773067081432,True,0.8401844532279314,0.0415698882843216,0.9853776779890624,False,completed,990317.6758330629,23641.45389096951,0.0233159830588082 -freddie_fm36_green,freddie_fm36_v2,green,36.0,sensitivity,2383971,0.0047483799089837,0.7143282449429617,0.0122747793777575,0.0048940401658034,0.8874942306290841,False,0.8787338525538769,0.0075725761510611,0.9873205421994056,False,completed,1035323.9699910176,6277.701643785229,0.0060269696321937 -freddie_fm12_both,freddie_fm36_v2,both,12.0,sensitivity,2849457,0.0059523621518064,0.7681863221355728,0.0222124268822717,0.0065800774151727,0.7578453348849258,False,0.0,0.0188414273204437,0.9774174647316434,False,completed,1287040.6418207984,9570.992353857262,0.0073815413201575 -freddie_fm24_both,freddie_fm36_v2,both,24.0,sensitivity,2420190,0.0103363785487916,0.7800044357516288,0.0422297603093363,0.0116655888819794,0.9369215610110188,True,0.8736444790605715,0.0195696660988925,0.9889295157422392,False,completed,1118619.780863266,8985.721653965535,0.0079688522571999 -freddie_fm36_both,freddie_fm36_v2,both,36.0,sensitivity,3581402,0.0114142450358825,0.7890898429830087,0.0496332913708696,0.012285448467355,0.9056827296391612,True,0.8571428571428571,0.0207387004344619,0.9871002504662412,False,completed,1042378.4068574588,11429.115930069587,0.0108455440703605 -freddie_fm48_both,freddie_fm36_v2,both,48.0,main_best_freddie,3173355,0.0145435981792141,0.7838634726333499,0.0517822051269566,0.0128541034734516,0.974523173547136,True,0.5,0.0447935262941367,0.9907116706887202,True,completed,1060065.9648987832,17947.475405741716,0.0166486564403796 -freddie_fm60_both,freddie_fm36_v2,both,60.0,sensitivity,3579151,0.0218331107013925,0.7270254323985861,0.0728705047312328,0.0256232956032094,0.959101839256819,True,0.5,0.0682240583551882,0.9822399292863524,False,completed,1033755.5253137936,22943.455865225056,0.0217123857161532 diff --git a/reports/crpto/multidataset/source/table_full_portfolio_candidate_sensitivity.csv b/reports/crpto/multidataset/source/table_full_portfolio_candidate_sensitivity.csv deleted file mode 100644 index a30cb35..0000000 --- a/reports/crpto/multidataset/source/table_full_portfolio_candidate_sensitivity.csv +++ /dev/null @@ -1,32 +0,0 @@ -dataset_id,run_id,strategy,candidate_cap,seed,available_candidates,n_candidates,total_budget,max_portfolio_pd,nonrobust_objective,robust_objective,price_of_robustness_abs,price_of_robustness_pct,nonrobust_solver_status,robust_solver_status,robust_total_allocated,robust_n_funded,scenario_expected_loss,scenario_worst_case_loss,experiment -prosper_final_status,top_net_return_cap1000_seed42,top_net_return,1000,42,10531,1000,1000000.0,0.3245769893119755,220259.86083871548,199419.0887756709,20840.77206304454,0.0946190194785654,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,233,65372.52341896657,272418.3244733848,prosper_candidate_sensitivity -prosper_final_status,top_net_return_cap5000_seed42,top_net_return,5000,42,10531,5000,1000000.0,0.3245769893119755,220259.86083871548,199419.0887756709,20840.77206304454,0.0946190194785654,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,233,65372.52341896657,272418.3244733848,prosper_candidate_sensitivity -prosper_final_status,top_net_return_capall_seed42,top_net_return,all,42,10531,10531,1000000.0,0.3245769893119755,220259.8608387156,199419.08877567088,20840.77206304469,0.094619019478566,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,234,65372.523418966586,272418.3244733848,prosper_candidate_sensitivity -prosper_final_status,random_cap1000_seed42,random,1000,42,10531,1000,1000000.0,0.3245769893119755,166168.17660167778,140692.0500073207,25476.12659435708,0.1533153165387736,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,187,95562.73242437,348047.29625519505,prosper_candidate_sensitivity -prosper_final_status,random_cap1000_seed2026,random,1000,2026,10531,1000,1000000.0,0.3245769893119755,161084.18582561763,134957.02672474118,26127.15910087645,0.1621956802709393,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,179,95097.42178610816,349908.5388082424,prosper_candidate_sensitivity -prosper_final_status,random_cap1000_seed777,random,1000,777,10531,1000,1000000.0,0.3245769893119755,160550.8819707652,134780.4007618212,25770.481208943995,0.1605128598025176,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,195,95491.46051430142,348332.3838954694,prosper_candidate_sensitivity -prosper_final_status,random_cap5000_seed42,random,5000,42,10531,5000,1000000.0,0.3245769893119755,206976.7543587402,184805.77749464923,22170.97686409097,0.1071181975617579,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,225,75003.67748690525,293928.7276713456,prosper_candidate_sensitivity -prosper_final_status,random_cap5000_seed2026,random,5000,2026,10531,5000,1000000.0,0.3245769893119755,205186.3880237709,183337.67361416595,21848.71440960499,0.1064822799408789,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,229,73071.08057072849,288388.2577223504,prosper_candidate_sensitivity -prosper_final_status,random_cap5000_seed777,random,5000,777,10531,5000,1000000.0,0.3245769893119755,204500.22865076512,181977.3476818193,22522.88096894577,0.1101362141135263,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,239,77749.32673360837,300304.236579906,prosper_candidate_sensitivity -prosper_final_status,random_capall_seed42,random,all,42,10531,10531,1000000.0,0.3245769893119755,220259.8608387152,199419.0887756713,20840.772063043813,0.0946190194785623,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,234,65372.52341896658,272418.3244733848,prosper_candidate_sensitivity -prosper_final_status,random_capall_seed2026,random,all,2026,10531,10531,1000000.0,0.3245769893119755,220259.86083871528,199419.08877567103,20840.77206304425,0.0946190194785642,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,234,65372.52341896658,272418.3244733848,prosper_candidate_sensitivity -prosper_final_status,random_capall_seed777,random,all,777,10531,10531,1000000.0,0.3245769893119755,220259.8608387152,199419.0887756712,20840.772063043987,0.0946190194785631,TerminationCondition.optimal,TerminationCondition.optimal,1000000.0,235,65372.52341896658,272418.3244733848,prosper_candidate_sensitivity -freddie_fm36_v2,top_net_return_cap1000_seed42,top_net_return,1000,42,1396053,1000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,top_net_return_cap5000_seed42,top_net_return,5000,42,1396053,5000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,top_net_return_cap10000_seed42,top_net_return,10000,42,1396053,10000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,top_net_return_cap25000_seed42,top_net_return,25000,42,1396053,25000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap1000_seed42,random,1000,42,1396053,1000,25000000.0,0.02,1078013.440304525,1060065.9648987832,17947.475405741716,0.0166486564403796,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,116,36866.39859659643,214000.76364280016,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap1000_seed2026,random,1000,2026,1396053,1000,25000000.0,0.02,1076191.3042931615,1056120.6105589091,20070.69373425236,0.0186497453140403,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,112,34554.48622246302,232476.6184087452,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap1000_seed777,random,1000,777,1396053,1000,25000000.0,0.02,1072459.8392371244,1054043.8340842563,18416.005152868107,0.0171717433875827,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,111,33706.47893519176,212405.84874071687,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap5000_seed42,random,5000,42,1396053,5000,25000000.0,0.02,1140916.4640746063,1124577.3552450945,16339.108829511795,0.0143210386947692,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,124,31184.2996716773,189019.7505039571,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap5000_seed2026,random,5000,2026,1396053,5000,25000000.0,0.02,1153774.3281600133,1134306.807301479,19467.52085853438,0.0168729017307745,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,117,32039.38181828153,215564.99062068545,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap5000_seed777,random,5000,777,1396053,5000,25000000.0,0.02,1147609.8299150486,1130622.7865380284,16987.043377020163,0.0148021068957536,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,124,29688.836011586005,189956.110515451,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap10000_seed42,random,10000,42,1396053,10000,25000000.0,0.02,1179598.351821189,1163838.4351295484,15759.916691640625,0.01336040921667,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,107,25004.01650437762,178233.50016511057,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap10000_seed2026,random,10000,2026,1396053,10000,25000000.0,0.02,1177725.181556371,1160204.5486363675,17520.632920003496,0.0148766734331432,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,122,24876.006482667228,188880.45529232625,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap10000_seed777,random,10000,777,1396053,10000,25000000.0,0.02,1182163.9354725585,1165078.147595012,17085.787877546158,0.0144529767529376,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,115,27177.87215551701,192636.67465023417,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap25000_seed42,random,25000,42,1396053,25000,25000000.0,0.02,1205300.5442875423,1189915.324215843,15385.220071699005,0.0127646338040884,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,112,27642.436947106475,175618.8253176002,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap25000_seed2026,random,25000,2026,1396053,25000,25000000.0,0.02,1208080.69495844,1190718.9281311815,17361.766827258514,0.014371363518772,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,116,22148.48511536901,186671.8526498669,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,random_cap25000_seed777,random,25000,777,1396053,25000,25000000.0,0.02,1216314.4942769462,1198143.740233957,18170.75404298911,0.0149391905863877,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,120,25592.49845325713,191373.6115811216,freddie_fm48_candidate_sensitivity -freddie_fm36_v2,top_net_return_cap50000_seed42,top_net_return,50000,42,1396053,50000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_top_large_sensitivity -freddie_fm36_v2,top_net_return_cap100000_seed42,top_net_return,100000,42,1396053,100000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_top_large_sensitivity -freddie_fm36_v2,top_net_return_cap250000_seed42,top_net_return,250000,42,1396053,250000,25000000.0,0.02,1305409.055278483,1291227.5801824068,14181.47509607626,0.0108636254963398,TerminationCondition.optimal,TerminationCondition.optimal,25000000.0,146,20879.088677396183,160874.9000793736,freddie_fm48_top_large_sensitivity diff --git a/reports/crpto/multidataset/source/table_main_external_replications.csv b/reports/crpto/multidataset/source/table_main_external_replications.csv deleted file mode 100644 index 2ff0f78..0000000 --- a/reports/crpto/multidataset/source/table_main_external_replications.csv +++ /dev/null @@ -1,3 +0,0 @@ -dataset,paper_role,variant,rows_total,default_rate,auc_roc,pr_auc,brier,coverage_90,min_group_coverage_90,coverage_alpha01,coverage90_pass,alpha01_pass,available_oot_candidates,lp_candidate_cap,lp_n_candidates,robust_objective_best,nonrobust_objective,price_of_robustness_abs,price_of_robustness_pct,robust_n_funded,scenario_expected_loss,scenario_worst_case_loss -Prosper final-status loans,main external economic replication,Temporal OOT 2012-2013; full OOT economic universe,54807,0.3091758352035324,0.7074290432845566,0.4484817219126147,0.1781741704802637,0.9205159821682634,0.8850308641975309,0.9943090202029784,True,True,10531,all,10531,199419.08877567088,220259.8608387156,20840.77206304469,0.094619019478566,234,65372.523418966586,272418.3244733848 -Freddie/Mendeley SF mortgage FM48 both,main external economic replication,"48-month horizon, red+green, OOT split; all-candidate LP certified",3173355,0.0145435981792141,0.7838634726333499,0.0517822051269566,0.0128541034734516,0.974523173547136,0.5,0.9907116706887202,True,True,1396053,all,1396053,1291227.5801824057,1305409.0552784826,14181.4750960769,0.0108636254963404,143,20879.088677396183,160874.9000793736 diff --git a/reports/crpto/multidataset/source/table_portfolio_candidate_sensitivity_compact.csv b/reports/crpto/multidataset/source/table_portfolio_candidate_sensitivity_compact.csv deleted file mode 100644 index bebcc5d..0000000 --- a/reports/crpto/multidataset/source/table_portfolio_candidate_sensitivity_compact.csv +++ /dev/null @@ -1,18 +0,0 @@ -dataset,strategy,candidate_cap,available_candidates,runs,robust_objective_mean,robust_objective_min,robust_objective_max,nonrobust_objective_mean,robust_n_funded_mean -Prosper final-status loans,top_net_return,1000,10531,1,199419.0887756709,199419.0887756709,199419.0887756709,220259.86083871548,233.0 -Prosper final-status loans,top_net_return,5000,10531,1,199419.0887756709,199419.0887756709,199419.0887756709,220259.86083871548,233.0 -Prosper final-status loans,top_net_return,all,10531,1,199419.08877567088,199419.08877567088,199419.08877567088,220259.8608387156,234.0 -Prosper final-status loans,random,1000,10531,3,136809.82583129438,134780.4007618212,140692.0500073207,162601.0814660202,187.0 -Prosper final-status loans,random,5000,10531,3,183373.59959687816,181977.34768181932,184805.77749464923,205554.45701109208,231.0 -Prosper final-status loans,random,all,10531,3,199419.08877567117,199419.08877567103,199419.0887756713,220259.86083871522,234.33333333333334 -Freddie FM48 both,top_net_return,1000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,5000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,10000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,25000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,50000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,100000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,top_net_return,250000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48 both,random,1000,1396053,3,1056743.4698473162,1054043.8340842563,1060065.9648987832,1075554.8612782704,113.0 -Freddie FM48 both,random,5000,1396053,3,1129835.6496948672,1124577.3552450945,1134306.807301479,1147433.5407165561,121.66666666666667 -Freddie FM48 both,random,10000,1396053,3,1163040.3771203093,1160204.5486363675,1165078.147595012,1179829.1562833728,114.66666666666667 -Freddie FM48 both,random,25000,1396053,3,1192925.997526994,1189915.324215843,1198143.740233957,1209898.5778409762,116.0 diff --git a/reports/crpto/multidataset/source/table_prosper_default_definition_sensitivity.csv b/reports/crpto/multidataset/source/table_prosper_default_definition_sensitivity.csv deleted file mode 100644 index 18d8f84..0000000 --- a/reports/crpto/multidataset/source/table_prosper_default_definition_sensitivity.csv +++ /dev/null @@ -1,4 +0,0 @@ -variant,rows_total,rows_test,default_rate,auc_roc,pr_auc,coverage_90,min_group_coverage_90,alpha01_coverage,coverage90_pass,alpha01_pass,available_oot_candidates,robust_objective_all_candidates,nonrobust_objective_all_candidates,price_of_robustness_pct -main_chargedoff_or_defaulted,54807,10543,0.3091758352035324,0.7074290432845566,0.44848172191261476,0.9205159821682634,0.8850308641975309,0.9943090202029783,True,True,10531,199419.0887756553,220259.86083869895,0.0946190194785686 -defaulted_only,42862,8031,0.11665344594279314,0.6796193367985865,0.0855667183476656,0.9148300336197236,0.8671328671328671,0.9947702652222637,True,True,8019,272969.2286232216,282522.5257470741,0.0338142847144336 -chargedoff_only,49807,10162,0.2398257273074066,0.7131638982556929,0.42343662495095546,0.9035622908876205,0.8355855855855856,0.9921275339500099,True,True,10150,205828.04819793423,227235.77315193572,0.0942093080550645 diff --git a/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/evidence.json b/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/evidence.json new file mode 100644 index 0000000..ea1edb7 --- /dev/null +++ b/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/evidence.json @@ -0,0 +1,71 @@ +{ + "schema_version": "2026-07-14.1", + "status": "complete_frozen_score_label_lag_sensitivity", + "run_tag": "ijds-label-lag-sensitivity-2026-07-14-v1", + "protocol_tag": "protocol/ijds-label-lag-sensitivity-2026-07-14-v1", + "protocol_commit": "688f75dc4f285c75bc499c9e041dd30fb3acd70d", + "historical_archive_previously_inspected": true, + "outcome_based_selection": false, + "source_freeze": { + "path": "models/experiments/ijds_audit/ijds-binary-geometry-frontier-v4-2026-07-12-v1/protocol_freeze.json", + "bytes": 20362, + "sha256": "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + }, + "results": { + "rows": 40, + "lags": [ + 0, + 3, + 6, + 8, + 12 + ], + "windows": 8, + "minimum_monthly_retention_by_lag": { + "0": 0.9971164936562861, + "3": 0.9966694421315571, + "6": 0.9929564746252483, + "8": 0.9868159653241828, + "12": 0.9745349467220517 + }, + "retention_stop_pass_by_lag": { + "0": true, + "3": true, + "6": true, + "8": false, + "12": false + } + }, + "artifact": { + "path": "reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/label_lag_phase_sensitivity.csv", + "bytes": 6476, + "sha256": "ddafa5977bb3ae3e1698caf412c3824708633bb17310d88d43deb301e5f389ea" + }, + "implementation": { + "source_files": { + "configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml": { + "path": "configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml", + "bytes": 865, + "sha256": "89d1c31c28a3167f4b90d1fee812ce7d2027154159aaf5f8217b97e25f3db44d" + }, + "src/ijds_audit/lag_sensitivity.py": { + "path": "src/ijds_audit/lag_sensitivity.py", + "bytes": 4523, + "sha256": "1836826e4d2c1b2c71a16a1c55bb66a67a164a4d412fcd4b257440b3e26dbcc1" + }, + "scripts/experiments/run_ijds_label_lag_sensitivity.py": { + "path": "scripts/experiments/run_ijds_label_lag_sensitivity.py", + "bytes": 5860, + "sha256": "7c89c6f9a2cb7af5606b0382f4bcb7a782e3377d37416bca60413d0b408278c6" + }, + "docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md": { + "path": "docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md", + "bytes": 1491, + "sha256": "e7d3485344946359e219ad40a97665fd8dd9346aa3e93ba426cafee2597b49e4" + } + }, + "hash_algorithm": "sha256" + }, + "protected_stages_run": [], + "protected_artifacts_written": [] +} diff --git a/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/label_lag_phase_sensitivity.csv b/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/label_lag_phase_sensitivity.csv new file mode 100644 index 0000000..77252d6 --- /dev/null +++ b/reports/crpto/sensitivity/ijds-label-lag-sensitivity-2026-07-14-v1/label_lag_phase_sensitivity.csv @@ -0,0 +1,41 @@ +charged_off_lag_months,window_id,window_rows,retained_rows,window_retention,minimum_monthly_retention,passes_locked_retention,phase_stratum,phase_rows,phase_prevalence,phase_residual_quantile,phase_interval_width_mean,phase_set_both_share +0,w01_2012m01_m06,14967,14949,0.9987973541791942,0.9971164936562861,True,2,3249,0.11665127731609726,0.8919735883395368,0.9881387583482552,0.13727300707910126 +0,w02_2012m02_m07,16690,16675,0.9991012582384662,0.9971164936562861,True,2,3621,0.11654239160452913,0.8920116585417792,0.9883652306769672,0.14333057166528584 +0,w03_2012m03_m08,19068,19052,0.999160897839312,0.9971164936562861,True,2,4018,0.11398705823792932,0.8916085888119072,0.9880289633104609,0.12692882030861125 +0,w04_2012m04_m09,21791,21773,0.9991739709054197,0.9971164936562861,True,2,4511,0.10995344712923964,0.890281077239412,0.9867825422901555,0.07670139658612281 +0,w05_2012m05_m10,24313,24282,0.9987249619545099,0.9971164936562861,True,2,5001,0.10657868426314737,0.8894148014498154,0.9859853433399698,0.04979004199160168 +0,w06_2012m06_m11,26747,26709,0.9985792799192433,0.9971164936562861,True,2,5505,0.10572207084468666,0.8893536841967226,0.9859485251379693,0.047229791099000905 +0,w07_2012m07_m12,28503,28453,0.9982457986878575,0.9971164936562861,True,2,5939,0.10321602963461862,0.8886909233974287,0.9851535995888699,0.022057585452096313 +0,w08_2012m08_2013m01,30259,30198,0.9979840708549522,0.9971164936562861,True,2,6251,0.0990241561350184,0.11187000593928165,0.20825402445310648,0.0 +3,w01_2012m01_m06,14967,14949,0.9987973541791942,0.9966694421315571,True,2,3249,0.11665127731609726,0.8919735883395368,0.9881387583482552,0.13727300707910126 +3,w02_2012m02_m07,16690,16675,0.9991012582384662,0.9966694421315571,True,2,3621,0.11654239160452913,0.8920116585417792,0.9883652306769672,0.14333057166528584 +3,w03_2012m03_m08,19068,19052,0.999160897839312,0.9966694421315571,True,2,4018,0.11398705823792932,0.8916085888119072,0.9880289633104609,0.12692882030861125 +3,w04_2012m04_m09,21791,21773,0.9991739709054197,0.9966694421315571,True,2,4511,0.10995344712923964,0.890281077239412,0.9867825422901555,0.07670139658612281 +3,w05_2012m05_m10,24313,24282,0.9987249619545099,0.9966694421315571,True,2,5001,0.10657868426314737,0.8894148014498154,0.9859853433399698,0.04979004199160168 +3,w06_2012m06_m11,26747,26709,0.9985792799192433,0.9966694421315571,True,2,5505,0.10572207084468666,0.8893536841967226,0.9859485251379693,0.047229791099000905 +3,w07_2012m07_m12,28503,28449,0.998105462582886,0.9966694421315571,True,2,5937,0.10291392959407109,0.8886749896176616,0.9851387217798037,0.021222839818089943 +3,w08_2012m08_2013m01,30259,30191,0.9977527347235533,0.9966694421315571,True,2,6249,0.09873579772763642,0.11186160467827937,0.20824626847497563,0.0 +6,w01_2012m01_m06,14967,14948,0.9987305405224828,0.9929564746252483,True,2,3249,0.11665127731609726,0.8919735883395368,0.9881387583482552,0.13727300707910126 +6,w02_2012m02_m07,16690,16674,0.9990413421210306,0.9929564746252483,True,2,3621,0.11654239160452913,0.8920116585417792,0.9883652306769672,0.14333057166528584 +6,w03_2012m03_m08,19068,19049,0.999003566184183,0.9929564746252483,True,2,4018,0.11398705823792932,0.8916085888119072,0.9880289633104609,0.12692882030861125 +6,w04_2012m04_m09,21791,21767,0.9988986278738929,0.9929564746252483,True,2,4511,0.10995344712923964,0.890281077239412,0.9867825422901555,0.07670139658612281 +6,w05_2012m05_m10,24313,24270,0.9982313988401267,0.9929564746252483,True,2,5001,0.10657868426314737,0.8894148014498154,0.9859853433399698,0.04979004199160168 +6,w06_2012m06_m11,26747,26685,0.9976819830261338,0.9929564746252483,True,2,5502,0.10523446019629226,0.8892521057704302,0.9858494930099646,0.04471101417666303 +6,w07_2012m07_m12,28503,28411,0.9967722695856577,0.9929564746252483,True,2,5929,0.10170349131388093,0.8884345991499274,0.9848983200473447,0.012987012987012988 +6,w08_2012m08_2013m01,30259,30129,0.9957037575597343,0.9929564746252483,True,2,6238,0.0971465213209362,0.1118010883671265,0.20818172077889815,0.0 +8,w01_2012m01_m06,14967,14945,0.9985300995523485,0.9868159653241828,False,2,3249,0.11665127731609726,0.8919735883395368,0.9881387583482552,0.13727300707910126 +8,w02_2012m02_m07,16690,16669,0.9987417615338526,0.9868159653241828,False,2,3620,0.11629834254143646,0.8919735883395368,0.9883343129081893,0.1414364640883978 +8,w03_2012m03_m08,19068,19042,0.9986364589888819,0.9868159653241828,False,2,4016,0.11354581673306773,0.8914417663875044,0.9878822284936603,0.11902390438247012 +8,w04_2012m04_m09,21791,21758,0.9984856133266027,0.9868159653241828,False,2,4509,0.10955866045686405,0.8902427153564156,0.9867467962298961,0.07562652472832114 +8,w05_2012m05_m10,24313,24245,0.9972031423518283,0.9868159653241828,False,2,4996,0.10568454763811048,0.8893093428244802,0.9858815936643381,0.04643714971977582 +8,w06_2012m06_m11,26747,26640,0.9959995513515535,0.9868159653241828,False,2,5494,0.10393156170367673,0.88903966354385,0.9856492442766289,0.037131416090280304 +8,w07_2012m07_m12,28503,28349,0.9945970599586008,0.9868159653241828,False,2,5916,0.09972954699121028,0.11188325252114233,0.2083465264137485,0.0 +8,w08_2012m08_2013m01,30259,30035,0.9925972437952345,0.9868159653241828,False,2,6217,0.09409683126910086,0.11171235946047844,0.20808778095976202,0.0 +12,w01_2012m01_m06,14967,14918,0.9967261308211398,0.9745349467220517,False,2,3244,0.11528976572133169,0.8918727660654336,0.9880507163356359,0.13255240443896424 +12,w02_2012m02_m07,16690,16612,0.995326542840024,0.9745349467220517,False,2,3611,0.11409581833287179,0.8916414588574106,0.9880417238455119,0.1282193298255331 +12,w03_2012m03_m08,19068,18949,0.9937591776798825,0.9745349467220517,False,2,4004,0.1108891108891109,0.8907949705797163,0.9873001284400981,0.0959040959040959 +12,w04_2012m04_m09,21791,21621,0.9921986141067413,0.9745349467220517,False,2,4493,0.1063877142221233,0.8894829334991204,0.98602897814676,0.05052303583351881 +12,w05_2012m05_m10,24313,24058,0.9895117838193559,0.9745349467220517,False,2,4976,0.10209003215434084,0.8883803108986407,0.9849755772001694,0.011454983922829582 +12,w06_2012m06_m11,26747,26396,0.9868770329382734,0.9745349467220517,False,2,5462,0.09868180153789821,0.11186160467827937,0.20848073235193956,0.0 +12,w07_2012m07_m12,28503,28052,0.9841771041644739,0.9745349467220517,False,2,5873,0.09313808956240423,0.11169021047879088,0.20814243720284836,0.0 +12,w08_2012m08_2013m01,30259,29700,0.9815261575068575,0.9745349467220517,False,2,6170,0.08719611021069693,0.11150439345019356,0.2078719347850636,0.0 diff --git a/reports/crpto/tables/README.md b/reports/crpto/tables/README.md deleted file mode 100644 index a0a07fa..0000000 --- a/reports/crpto/tables/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# CRPTO paper-facing tables - -This directory holds the CSV/TeX exports consumed by the IJDS paper, the online -supplement, and the Quarto book. The paper-facing canon is **`crpto_table0`** -(key metrics) plus the journal appendix package **`A3`–`A40`**. Every one of -those is cited in the paper, the supplement, or a book chapter (see -`book/chapters/07-apendice-robustez.qmd` and `30-replicacion-multidataset.qmd`). - -## Active set - -| Range | Role | -| --- | --- | -| `crpto_table0_key_metrics` | Headline metrics block, except the retired historical `price_of_robustness*` rows. | -| `A3`–`A11` | Holdouts, sensitivity, shift, exact bound eval, funded set. | -| `A12`–`A21`, `A21b` | Tail risk, satisficing, dependence, regret-auditability, cluster/concentration bounds. | -| `A22`–`A24` | Tail-constrained re-opt, multi-distribution, online ACI diagnostics. | -| `A25`–`A34` | External multidataset replication (Prosper/Freddie) and cross-dataset price of robustness. | -| `A35` | Active exact-alpha replay, coverage, width, and endpoint-saturation audit. | -| `A36` | Active November selector, endpoint-cap stability, and December outcome-free replay. | -| `A37` | Active full-OOT and temporal fixed-policy evaluation. | -| `A38` | Active selected funded-set letter-grade composition. | -| `A39` | Active month-cluster bootstrap with funded-loan sensitivity. | -| `A40` | Active selected, conservative, and matched point-PD decision comparison. | - -## Legacy / superseded (retained for provenance, not paper-facing) - -These early exports are regenerated by upstream stages but are **not** surfaced in -the paper or book. The A3–A34 journal package is the paper-facing replacement. -Do not cite them in new prose; prefer the active set above. - -The frozen Lending Club `price_of_robustness=-10.56%` fields in table 0, table -1 and A2 are retained only for provenance. Their historical "nonrobust" -baseline inherited a conformal endpoint constraint and was not a point-PD -comparator. Protected historical pool93 exports retain their explicit -`*_pool93_*` filenames; they are provenance, not the active A35--A40 bundle. -See `docs/research/active_claims_2026-07-04.md` for the active contract and -`docs/research/pool93_certificate_semantics_v2_2026-07-09.md` for history. - -- `crpto_table1_robustness_summary` -- `crpto_table2_conformal_variant_benchmark` -- `crpto_tableA1_benchmark_by_group` -- `crpto_tableA2_robustness_frontier` - -## Drift guard - -`tests/test_supplement_table_sync.py` checks that the key external-replication -numbers shown as hand-authored Markdown in the paper/supplement/book match the -underlying CSVs (the supplement embeds several tables inline rather than reading -the CSV at render time). diff --git a/reports/crpto/tables/crpto_ijds_comparator_support_domain.csv b/reports/crpto/tables/crpto_ijds_comparator_support_domain.csv new file mode 100644 index 0000000..1579a40 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_comparator_support_domain.csv @@ -0,0 +1,9 @@ +source,cap_month_rows,decision_active,objective_boundary,objective_slack,primal_degenerate_bases,near_zero_bases +broad_support_lower,15,15,0,0,0,0 +broad_support_upper,15,0,0,15,0,0 +development_support_lower,1065,1065,0,0,14,0 +development_support_upper,1080,1074,0,6,0,0 +named_c0,45,0,0,45,0,0 +named_c1,1080,1079,0,1,7,0 +named_c2,1079,1075,4,0,4,0 +period_basis_breakpoint,2952,2922,15,15,2920,0 diff --git a/reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv b/reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv new file mode 100644 index 0000000..fa36f7d --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv @@ -0,0 +1,7 @@ +role,risk_tolerance,cells,gamma_100_decision_active,objective_difference_mean,objective_difference_min,objective_difference_max +policy_development,0.15,88,88,-2016.1828129153232,-4553.797428104561,-1094.981985942548 +policy_development,0.17,88,88,-1961.9325673834644,-2663.6701891340053,-723.4016405355651 +policy_development,0.19,88,88,-2043.841637572058,-2753.1448433307814,-579.9765933432209 +primary_oot,0.15,120,120,-2836.3998776694834,-8337.854696241615,-1940.0408873799897 +primary_oot,0.17,120,120,-2812.647021760713,-3743.411901880114,-1316.7470416328288 +primary_oot,0.19,120,120,-3035.929468323465,-4313.593440831493,-989.0226017109235 diff --git a/reports/crpto/tables/crpto_ijds_policy_family_domain.csv b/reports/crpto/tables/crpto_ijds_policy_family_domain.csv new file mode 100644 index 0000000..ca4f0fb --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_policy_family_domain.csv @@ -0,0 +1,36 @@ +role,gamma,risk_tolerance,cap_classification,cells,binding_cells,minimum_score,mean_score,maximum_score,objective_mean +policy_development,0.0,0.15,objective_slack,88,0,0.018962250683581334,0.0913791264637191,0.10070698552648051,136801.06324723674 +policy_development,0.0,0.17,objective_slack,88,0,0.018962250683581334,0.0913791264637191,0.10070698552648051,136801.06324723674 +policy_development,0.0,0.19,objective_slack,88,0,0.018962250683581334,0.0913791264637191,0.10070698552648051,136801.06324723674 +policy_development,0.25,0.15,decision_active,85,85,0.03107186777496268,0.15,0.26609104074254214,133093.62468299922 +policy_development,0.25,0.15,objective_slack,3,0,0.03182273459121894,0.14778246935725625,0.14972777022248734,136744.29927713264 +policy_development,0.25,0.17,decision_active,80,80,0.03107186777496268,0.17,0.26609104074254214,134321.08493640792 +policy_development,0.25,0.17,objective_slack,8,0,0.03143850491086809,0.15599721512171788,0.16918552068896126,137794.40558574925 +policy_development,0.25,0.19,decision_active,78,78,0.03107186777496268,0.19,0.26609104074254214,135574.23747489197 +policy_development,0.25,0.19,objective_slack,10,0,0.03143850491086809,0.1609300980062934,0.18556740191517496,136364.73957020076 +policy_development,0.5,0.15,decision_active,88,88,0.04318148486634403,0.15,0.4336355392892376,128464.64278529496 +policy_development,0.5,0.17,decision_active,88,88,0.04318148486634403,0.17,0.4336355392892376,130011.56190194725 +policy_development,0.5,0.19,decision_active,88,88,0.04318148486634403,0.18999999999999997,0.4336355392892376,131303.00563169338 +policy_development,0.75,0.15,decision_active,88,88,0.05529110195772537,0.15,0.6012096516641946,125675.2194051378 +policy_development,0.75,0.17,decision_active,88,88,0.05529110195772537,0.16999999999999998,0.6012096516641946,127075.63989162772 +policy_development,0.75,0.19,decision_active,88,88,0.05529110195772537,0.18999999999999997,0.6012096516641946,128312.88517942633 +policy_development,1.0,0.15,decision_active,88,88,0.06740071904910673,0.15,0.7687837640391516,123659.03659222247 +policy_development,1.0,0.17,decision_active,88,88,0.06740071904910673,0.17,0.7687837640391516,125113.70732424427 +policy_development,1.0,0.19,decision_active,88,88,0.06740071904910673,0.18999999999999997,0.7687837640391516,126269.04354185426 +primary_oot,0.0,0.15,objective_slack,120,0,0.01292347362151883,0.10718289699402918,0.1186807433291554,182511.67244672237 +primary_oot,0.0,0.17,objective_slack,120,0,0.01292347362151883,0.10718289699402918,0.1186807433291554,182511.67244672237 +primary_oot,0.0,0.19,objective_slack,120,0,0.01292347362151883,0.10718289699402918,0.1186807433291554,182511.67244672237 +primary_oot,0.25,0.15,decision_active,120,120,0.025033090712900184,0.15,0.3183243201559923,172234.83185381157 +primary_oot,0.25,0.17,decision_active,119,119,0.025033090712900184,0.17,0.3183243201559923,175132.58902240213 +primary_oot,0.25,0.17,objective_slack,1,0,0.025161588297824736,0.15774455171134974,0.1577445517113497,168221.95356983488 +primary_oot,0.25,0.19,decision_active,116,116,0.025033090712900184,0.19,0.3183243201559923,177479.49286239638 +primary_oot,0.25,0.19,objective_slack,4,0,0.025161588297824736,0.17743274213772214,0.18837815069639094,173614.838882224 +primary_oot,0.5,0.15,decision_active,120,120,0.03714270780428152,0.15,0.5179678969828292,164511.0062546627 +primary_oot,0.5,0.17,decision_active,120,120,0.03714270780428152,0.16999999999999998,0.5179678969828292,166875.69425747043 +primary_oot,0.5,0.19,decision_active,120,120,0.03714270780428152,0.19,0.5179678969828292,168999.96346780093 +primary_oot,0.75,0.15,decision_active,120,120,0.04925232489566286,0.15,0.7176114738096658,160438.69310691068 +primary_oot,0.75,0.17,decision_active,120,120,0.04925232489566286,0.17,0.7176114738096658,162500.43707177957 +primary_oot,0.75,0.19,decision_active,120,120,0.04925232489566286,0.19,0.7176114738096658,164385.76112934432 +primary_oot,1.0,0.15,decision_active,120,120,0.06136194198704421,0.15,0.9172550506365028,157602.29322924122 +primary_oot,1.0,0.17,decision_active,120,120,0.06136194198704421,0.16999999999999998,0.9172550506365028,159687.79005001887 +primary_oot,1.0,0.19,decision_active,120,120,0.06136194198704421,0.19,0.9172550506365028,161349.83166102084 diff --git a/reports/crpto/tables/crpto_ijds_v4_table1_coverage_windows.csv b/reports/crpto/tables/crpto_ijds_v4_table1_coverage_windows.csv new file mode 100644 index 0000000..46bce2a --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table1_coverage_windows.csv @@ -0,0 +1,17 @@ +learner,window_id,taxonomy_groups,role,conformal_group,candidate_rows,resolved_rows,unresolved_rows,coverage_resolved,coverage_lower,coverage_upper,score_min,score_max,fit_rows,fit_prevalence,fit_residual_quantile,fit_score_min,fit_score_max,scores_below_fit_range,scores_above_fit_range,rows,mean_width,lower_positive_share,upper_saturated_share,set_empty_count,set_empty_share,set_zero_only_count,set_zero_only_share,set_one_only_count,set_one_only_share,set_both_count,set_both_share,width_q00,width_q10,width_q25,width_q50,width_q75,width_q90,width_q100,fit_coverage +catboost_platt,w01_2012m01_m06,5,primary_oot,-1,376890,364814,12076,0.8770469335058413,0.8547135769057285,0.8805221682719095,0.005424291130823528,0.5544847575803251,14948,0.13674070109713674,0.8219519891493827,0.009701465766812493,0.5281745670852075,139,3,376890,0.6649918872956118,0.01250497492637109,0.18708376449361883,4713,0.01250497492637109,301667,0.8004112605800101,0,0.0,70510,0.18708376449361883,0.054537479077481146,0.0896353066409487,0.1426944072201049,0.9785780281145038,0.9953786894773093,1.0,1.0,0.9003880117741504 +catboost_platt,w02_2012m02_m07,5,primary_oot,-1,376890,364814,12076,0.879165821487114,0.8570245960359787,0.8825970442304121,0.005424291130823528,0.5544847575803251,16674,0.14081804006237256,0.82766418729986,0.009701465766812493,0.4831174891615209,139,32,376890,0.665340945235394,0.012327204224044151,0.19626946854519886,4646,0.012327204224044151,298272,0.791403327230757,0,0.0,73972,0.19626946854519886,0.0544282437770001,0.08952607134046764,0.14283617519766612,0.9793001695195559,0.9960733268504931,1.0,1.0,0.9004438047259206 +catboost_platt,w03_2012m03_m08,5,primary_oot,-1,376890,364814,12076,0.8777349553471084,0.8556528430045902,0.8811430390830216,0.005424291130823528,0.5544847575803251,19049,0.14116226573573415,0.8266480025829714,0.009701465766812493,0.4831174891615209,139,32,376890,0.6653200651843035,0.014086338188861472,0.19673644830056516,5309,0.014086338188861472,297433,0.7891772135105733,0,0.0,74148,0.19673644830056516,0.054239731200646374,0.08933755876411392,0.14275810252805138,0.9793603622579774,0.9961042627119951,1.0,1.0,0.9003622237387789 +catboost_platt,w04_2012m04_m09,5,primary_oot,-1,376890,364814,12076,0.8748732230670972,0.8525697153015469,0.8783411605508239,0.005424291130823528,0.5544847575803251,21767,0.14126889327881656,0.8255389571463173,0.009701465766812493,0.4831174891615209,139,32,376890,0.665031208384886,0.015378492398312505,0.18653453262225053,5796,0.015378492398312505,300791,0.7980869749794369,0,0.0,70303,0.18653453262225053,0.0539599419893023,0.08905776955276984,0.14283617519766612,0.9786919648415489,0.9953512704484286,1.0,1.0,0.9002618642899802 +catboost_platt,w05_2012m05_m10,5,primary_oot,-1,376890,364814,12076,0.87251311627295,0.8499323410013532,0.8760327947146382,0.005424291130823528,0.5544847575803251,24270,0.13976102183765965,0.8238022572208143,0.009371217051990668,0.4831174891615209,119,32,376890,0.6647076525088493,0.01602058956194115,0.17684470269840005,6038,0.01602058956194115,304201,0.8071347077396588,0,0.0,66651,0.17684470269840005,0.053862759496348916,0.08896058705981646,0.14283524655328017,0.9779902667559446,0.9946330525536466,1.0,1.0,0.9002472187886279 +catboost_platt,w06_2012m06_m11,5,primary_oot,-1,376890,364814,12076,0.8721129123334083,0.8494255618350182,0.8756692934277905,0.005424291130823528,0.5544847575803251,26685,0.1380925613640622,0.8228775147793896,0.009275781579087032,0.4831174891615209,113,32,376890,0.6645688455708323,0.015450131338056197,0.17227838361325587,5823,0.015450131338056197,306137,0.812271485048688,0,0.0,64930,0.17227838361325587,0.05389117253953069,0.08898900010299823,0.14289258993779208,0.9776415565716892,0.9942949912817858,1.0,1.0,0.9002435825370058 +catboost_platt,w07_2012m07_m12,5,primary_oot,-1,376890,364814,12076,0.8694814343747773,0.8465175515402372,0.8730929448910822,0.005424291130823528,0.5544847575803251,28411,0.13248389708211608,0.8164243127235671,0.009275781579087032,0.49345656902413654,113,18,376890,0.6641544351090525,0.01606569556103903,0.1612910928918252,6055,0.01606569556103903,310046,0.8226432115471358,0,0.0,60789,0.1612910928918252,0.05390479242464117,0.08900261998810871,0.142788277391194,0.9767883025628927,0.9934566726155654,1.0,1.0,0.9002147055717856 +catboost_platt,w08_2012m08_2013m01,5,primary_oot,-1,376890,364814,12076,0.8658549288130389,0.8424845445620738,0.8695640637851894,0.005424291130823528,0.5544847575803251,30129,0.12761790965514952,0.8091359890849514,0.009275781579087032,0.49345656902413654,113,18,376890,0.5005686881638921,0.016068348855103613,0.14419591923372868,6056,0.016068348855103613,316488,0.8397357319111677,0,0.0,54346,0.14419591923372868,0.05408270522336448,0.08918053278683202,0.14269205142933994,0.2108606159144652,0.9855777292913334,1.0,1.0,0.9002290152344917 +numeric_logistic_platt,w01_2012m01_m06,5,primary_oot,-1,376890,364814,12076,0.8930578322103867,0.8718193637400833,0.8962217092520364,1.9895529665330944e-46,1.0,14948,0.13674070109713674,0.8132704993326145,0.002386024381188275,0.4387428023626731,2268,1395,376890,0.7502982548753632,0.00845074159569105,0.23931120486083474,2900,0.007694552787285415,283796,0.7529942423518798,285,0.0007561888084056356,89909,0.2385550160524291,0.05584550688919372,0.10655796562651992,0.15853748633016512,0.9834217079741655,0.9992278304184096,1.0,1.0,0.9004549103559004 +numeric_logistic_platt,w02_2012m02_m07,5,primary_oot,-1,376890,364814,12076,0.8921011803275094,0.8708509114065112,0.8952771365650455,1.9895529665330944e-46,1.0,16674,0.14081804006237256,0.813209200317482,0.002386024381188275,0.4387428023626731,2268,1395,376890,0.7500855689952323,0.008891188410411525,0.2384409244076521,3062,0.008124386425747565,283962,0.7534346891666003,289,0.0007668019846639603,89577,0.23767412242298813,0.055803534960441346,0.10651599369776756,0.1585032799436104,0.983287132927271,0.9991742093226285,1.0,1.0,0.9003838311143098 +numeric_logistic_platt,w03_2012m03_m08,5,primary_oot,-1,376890,364814,12076,0.8918188446715313,0.8706413011754093,0.8949693544535541,1.9895529665330944e-46,1.0,19049,0.14116226573573415,0.8106670055920027,0.002386024381188275,0.4387428023626731,2268,1395,376890,0.750093920569841,0.00950675263339436,0.23981267743904056,3294,0.008739950648730399,283213,0.751447371912229,289,0.0007668019846639603,90094,0.2390458754543766,0.055762309681025385,0.1064747684183516,0.15843614395650368,0.9834163815599517,0.9992604741626012,1.0,1.0,0.9003622237387789 +numeric_logistic_platt,w04_2012m04_m09,5,primary_oot,-1,376890,364814,12076,0.8881567045124364,0.8666640133726021,0.8913741409960466,1.9895529665330944e-46,1.0,21767,0.14126889327881656,0.8077035644116417,0.002386024381188275,0.4387428023626731,2268,1395,376890,0.7496760765048108,0.011305686009180398,0.22668683170155748,3972,0.010538884024516438,287482,0.7627742842739261,289,0.0007668019846639603,85147,0.2259200297168935,0.05568070201998956,0.10639316075731577,0.15823378694722007,0.9825390355131967,0.9983719080437077,1.0,1.0,0.9003078053934855 +numeric_logistic_platt,w05_2012m05_m10,5,primary_oot,-1,376890,364814,12076,0.8856924350490936,0.8636631377855608,0.8890127092785693,1.9895529665330944e-46,1.0,24270,0.13976102183765965,0.8061316208044618,0.002386024381188275,0.4387428023626731,2268,1395,376890,0.7491068029128565,0.010719308020907958,0.20815887924858711,3751,0.009952506036243997,294686,0.7818886147151689,289,0.0007668019846639603,78164,0.20739207726392317,0.05578939611884056,0.10650185485616677,0.158237310021505,0.9813043415737832,0.9971105921513819,1.0,1.0,0.900288421920066 +numeric_logistic_platt,w06_2012m06_m11,5,primary_oot,-1,376890,364814,12076,0.8830445103532211,0.8605614370240654,0.886468200270636,1.9895529665330944e-46,1.0,26685,0.1380925613640622,0.8027175079686439,0.002386024381188275,0.43322525310526877,2268,1476,376890,0.7482864173070811,0.009997612035341876,0.18963092679561674,3476,0.009222850168484172,301944,0.8011462230358991,292,0.0007747618668577038,71178,0.18885616492875906,0.05580733474546511,0.10651979348279132,0.1583289847525134,0.979905548780851,0.9958109701446924,1.0,1.0,0.9002435825370058 +numeric_logistic_platt,w07_2012m07_m12,5,primary_oot,-1,376890,364814,12076,0.8774279495852682,0.8545172331449494,0.8809758815569529,1.9895529665330944e-46,1.0,28411,0.13248389708211608,0.7953701100393342,0.00026321481438389247,0.9937191448732018,929,87,376890,0.6012897474623677,0.011770012470482103,0.16899360556130436,4142,0.010989944015495237,309056,0.8200164504232004,294,0.0007800684549868662,63398,0.1682135371063175,0.05571640945671315,0.10642886819403935,0.15824444009055916,0.955000941141985,0.9894183165928746,1.0,1.0,0.9002147055717856 +numeric_logistic_platt,w08_2012m08_2013m01,5,primary_oot,-1,376890,364814,12076,0.8732340315886999,0.8500305128817427,0.8768632757568522,1.9895529665330944e-46,1.0,30129,0.12761790965514952,0.7860933691064516,1.0309719899645506e-06,0.9937191448732018,153,87,376890,0.6003912914346058,0.013468120671814058,0.1554432327734883,4779,0.012680092334633448,313526,0.8318766748918782,297,0.0007880283371806097,58288,0.15465520443630767,0.05573682003560672,0.10644927877293292,0.15820549584760407,0.9517732360026724,0.9874696670924321,1.0,1.0,0.9002622058481862 diff --git a/reports/crpto/tables/crpto_ijds_v4_table2_phase_transition.csv b/reports/crpto/tables/crpto_ijds_v4_table2_phase_transition.csv new file mode 100644 index 0000000..26468e0 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table2_phase_transition.csv @@ -0,0 +1,9 @@ +window_id,fit_rows,fit_prevalence,fit_residual_quantile,coverage_lower,coverage_upper,mean_width,set_empty_share,set_zero_only_share,set_both_share +w01_2012m01_m06,3249,0.11665127731609726,0.8919735883395368,0.8470909711943527,0.8755930016319405,0.9875801457582358,0.0,0.8844484926689185,0.11555150733108151 +w02_2012m02_m07,3621,0.11654239160452913,0.8920116585417792,0.8473313345224993,0.8758080635571243,0.9876137950975035,0.0,0.8831707718192974,0.11682922818070263 +w03_2012m03_m08,4018,0.11398705823792932,0.8916085888119072,0.845092160360292,0.8738851569319519,0.9872553690886569,0.0,0.8954419522562526,0.10455804774374738 +w04_2012m04_m09,4511,0.10995344712923964,0.890281077239412,0.837223424038863,0.8673573949675509,0.9860405848595958,0.0,0.9347223803559908,0.06527761964400926 +w05_2012m05_m10,5001,0.10657868426314737,0.8894148014498154,0.8320872392374157,0.8630941085683201,0.9852197606709866,0.0,0.9606310169898921,0.03936898301010791 +w06_2012m06_m11,5502,0.10523446019629226,0.8892521057704302,0.8311890394322365,0.8623603678823991,0.9850630399535071,0.0,0.9660202158209673,0.033979784179032725 +w07_2012m07_m12,5929,0.10170349131388093,0.8884345991499274,0.8264829784811568,0.858375396915759,0.9842633701640712,0.0,0.9904360696800638,0.00956393031993624 +w08_2012m08_2013m01,6238,0.0971465213209362,0.1118010883671265,0.8225359596189609,0.8547066934861538,0.2076312400549422,0.002618695206649209,0.9973813047933507,0.0 diff --git a/reports/crpto/tables/crpto_ijds_v4_table3_development_envelopes.csv b/reports/crpto/tables/crpto_ijds_v4_table3_development_envelopes.csv new file mode 100644 index 0000000..955b49e --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table3_development_envelopes.csv @@ -0,0 +1,217 @@ +window_id,paired_policy_id,scope,metric,lower,upper,direction,evaluated_comparators,nested_scope_independent_replications +w01_2012m01_m06,linear-001,development_admissible_exact_frontier,standardized_payoff,-109735.65398073764,185111.49462475546,crosses_zero,391,False +w01_2012m01_m06,linear-001,development_admissible_exact_frontier,terminal_default,-0.02252332801047616,0.014900846498859137,crosses_zero,391,False +w01_2012m01_m06,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.01418832801047616,0.020085411494833116,crosses_zero,391,False +w01_2012m01_m06,linear-002,development_admissible_exact_frontier,standardized_payoff,-216623.11267195913,40042.72249593843,crosses_zero,402,False +w01_2012m01_m06,linear-002,development_admissible_exact_frontier,terminal_default,-0.009379864001059855,0.027151625575540753,crosses_zero,402,False +w01_2012m01_m06,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0005670370523005721,0.030247428848029896,crosses_zero,402,False +w01_2012m01_m06,linear-003,development_admissible_exact_frontier,standardized_payoff,-277298.52745546674,15311.141252362097,crosses_zero,452,False +w01_2012m01_m06,linear-003,development_admissible_exact_frontier,terminal_default,-0.00714762078782274,0.035448057694965104,crosses_zero,452,False +w01_2012m01_m06,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.00784495441739567,0.044354367837538905,guardrail_higher,452,False +w01_2012m01_m06,linear-004,development_admissible_exact_frontier,standardized_payoff,-96961.82943944942,46319.76795530621,crosses_zero,355,False +w01_2012m01_m06,linear-004,development_admissible_exact_frontier,terminal_default,-0.008132744613745767,0.014859457227555804,crosses_zero,355,False +w01_2012m01_m06,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.002977148622079066,0.013625605698410577,crosses_zero,355,False +w01_2012m01_m06,linear-005,development_admissible_exact_frontier,standardized_payoff,-246085.88693346095,35774.4165533372,crosses_zero,456,False +w01_2012m01_m06,linear-005,development_admissible_exact_frontier,terminal_default,-0.009713414863560837,0.029073435170322355,crosses_zero,456,False +w01_2012m01_m06,linear-005,development_admissible_exact_frontier,funded_miscoverage,0.002687324166978515,0.03372480492103762,guardrail_higher,456,False +w01_2012m01_m06,linear-006,development_admissible_exact_frontier,standardized_payoff,-245830.95819276563,10387.815272239677,crosses_zero,459,False +w01_2012m01_m06,linear-006,development_admissible_exact_frontier,terminal_default,-0.004617304639850082,0.029153664734697028,crosses_zero,459,False +w01_2012m01_m06,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.0074683572999169326,0.03531073274882338,guardrail_higher,459,False +w01_2012m01_m06,linear-007,development_admissible_exact_frontier,standardized_payoff,-90334.22845018914,92802.24072025063,crosses_zero,345,False +w01_2012m01_m06,linear-007,development_admissible_exact_frontier,terminal_default,-0.011680558064484921,0.01274114154922367,crosses_zero,345,False +w01_2012m01_m06,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.008728528347244325,0.009459275786119303,crosses_zero,345,False +w01_2012m01_m06,linear-008,development_admissible_exact_frontier,standardized_payoff,-232696.8895801942,57131.91503576482,crosses_zero,475,False +w01_2012m01_m06,linear-008,development_admissible_exact_frontier,terminal_default,-0.011361218503168593,0.02846949493241623,crosses_zero,475,False +w01_2012m01_m06,linear-008,development_admissible_exact_frontier,funded_miscoverage,0.0007561678911738763,0.03018456999553835,guardrail_higher,475,False +w01_2012m01_m06,linear-009,development_admissible_exact_frontier,standardized_payoff,-264583.95264396485,-561.8605607499339,guardrail_lower,446,False +w01_2012m01_m06,linear-009,development_admissible_exact_frontier,terminal_default,-0.007087089137207651,0.03200860609233613,crosses_zero,446,False +w01_2012m01_m06,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.004284015930825994,0.0365459186585623,guardrail_higher,446,False +w02_2012m02_m07,linear-001,development_admissible_exact_frontier,standardized_payoff,-122580.67024488423,172365.96011138515,crosses_zero,380,False +w02_2012m02_m07,linear-001,development_admissible_exact_frontier,terminal_default,-0.021264502529096646,0.016169652592305492,crosses_zero,380,False +w02_2012m02_m07,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.013537835862429983,0.02074588425494614,crosses_zero,380,False +w02_2012m02_m07,linear-002,development_admissible_exact_frontier,standardized_payoff,-216469.19497998687,40108.39759542342,crosses_zero,402,False +w02_2012m02_m07,linear-002,development_admissible_exact_frontier,terminal_default,-0.009374144040644027,0.027123550205520475,crosses_zero,402,False +w02_2012m02_m07,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.000570500545918071,0.029612940536731823,crosses_zero,402,False +w02_2012m02_m07,linear-003,development_admissible_exact_frontier,standardized_payoff,-277397.11045469303,15430.250908385788,crosses_zero,452,False +w02_2012m02_m07,linear-003,development_admissible_exact_frontier,terminal_default,-0.007169447450603999,0.035457564735959554,crosses_zero,452,False +w02_2012m02_m07,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.007338758305035276,0.04433450542895422,guardrail_higher,452,False +w02_2012m02_m07,linear-004,development_admissible_exact_frontier,standardized_payoff,-97092.06393632239,46863.34683207117,crosses_zero,355,False +w02_2012m02_m07,linear-004,development_admissible_exact_frontier,terminal_default,-0.00810010992896509,0.01487343373742973,crosses_zero,355,False +w02_2012m02_m07,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.004041970431255641,0.01234742693783179,crosses_zero,355,False +w02_2012m02_m07,linear-005,development_admissible_exact_frontier,standardized_payoff,-245206.5257337064,36739.284184372504,crosses_zero,455,False +w02_2012m02_m07,linear-005,development_admissible_exact_frontier,terminal_default,-0.00978361447374726,0.02897391816689256,crosses_zero,455,False +w02_2012m02_m07,linear-005,development_admissible_exact_frontier,funded_miscoverage,0.0023701821885989023,0.03368773955700659,guardrail_higher,455,False +w02_2012m02_m07,linear-006,development_admissible_exact_frontier,standardized_payoff,-243268.95070967785,13021.155311261333,crosses_zero,459,False +w02_2012m02_m07,linear-006,development_admissible_exact_frontier,terminal_default,-0.00481157806848993,0.02891836008997413,crosses_zero,459,False +w02_2012m02_m07,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.006843586284946799,0.0344731398748996,guardrail_higher,459,False +w02_2012m02_m07,linear-007,development_admissible_exact_frontier,standardized_payoff,-89385.5781452073,93820.55390317207,crosses_zero,345,False +w02_2012m02_m07,linear-007,development_admissible_exact_frontier,terminal_default,-0.011801706590584406,0.012634638293406912,crosses_zero,345,False +w02_2012m02_m07,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.009458010206677138,0.008052285767084359,crosses_zero,345,False +w02_2012m02_m07,linear-008,development_admissible_exact_frontier,standardized_payoff,-234303.21436735784,55276.045922130055,crosses_zero,476,False +w02_2012m02_m07,linear-008,development_admissible_exact_frontier,terminal_default,-0.011189258076019937,0.02861213012495479,crosses_zero,476,False +w02_2012m02_m07,linear-008,development_admissible_exact_frontier,funded_miscoverage,0.0003207542956668386,0.03034812425683184,guardrail_higher,476,False +w02_2012m02_m07,linear-009,development_admissible_exact_frontier,standardized_payoff,-265040.049820799,-1215.9681407797034,guardrail_lower,445,False +w02_2012m02_m07,linear-009,development_admissible_exact_frontier,terminal_default,-0.007039404888820795,0.032032000132641195,crosses_zero,445,False +w02_2012m02_m07,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.004318211595644991,0.03596325306960306,guardrail_higher,445,False +w03_2012m03_m08,linear-001,development_admissible_exact_frontier,standardized_payoff,-124640.30565131974,172160.60192027324,crosses_zero,380,False +w03_2012m03_m08,linear-001,development_admissible_exact_frontier,terminal_default,-0.021238384008269308,0.016381803439726624,crosses_zero,380,False +w03_2012m03_m08,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.013236717341602639,0.021566368435700603,crosses_zero,380,False +w03_2012m03_m08,linear-002,development_admissible_exact_frontier,standardized_payoff,-213947.40293775004,42682.127731154615,crosses_zero,401,False +w03_2012m03_m08,linear-002,development_admissible_exact_frontier,terminal_default,-0.009633604246047317,0.02689464387720917,crosses_zero,401,False +w03_2012m03_m08,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0009729139756863353,0.030504643877209168,crosses_zero,401,False +w03_2012m03_m08,linear-003,development_admissible_exact_frontier,standardized_payoff,-281703.2598444567,15210.265877930913,crosses_zero,460,False +w03_2012m03_m08,linear-003,development_admissible_exact_frontier,terminal_default,-0.007134138422123399,0.036081032055396804,crosses_zero,460,False +w03_2012m03_m08,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.008907896120661356,0.04603680153553696,guardrail_higher,460,False +w03_2012m03_m08,linear-004,development_admissible_exact_frontier,standardized_payoff,-97166.40449153635,46456.55632461156,crosses_zero,355,False +w03_2012m03_m08,linear-004,development_admissible_exact_frontier,terminal_default,-0.008086541598505784,0.014879448488547126,crosses_zero,355,False +w03_2012m03_m08,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.003749377893110536,0.012960230665199534,crosses_zero,355,False +w03_2012m03_m08,linear-005,development_admissible_exact_frontier,standardized_payoff,-241438.64686187476,40417.79804689958,crosses_zero,454,False +w03_2012m03_m08,linear-005,development_admissible_exact_frontier,terminal_default,-0.010167945791142805,0.028607171664468353,crosses_zero,454,False +w03_2012m03_m08,linear-005,development_admissible_exact_frontier,funded_miscoverage,0.0016116257064932972,0.033315365200585825,guardrail_higher,454,False +w03_2012m03_m08,linear-006,development_admissible_exact_frontier,standardized_payoff,-243166.11273073417,13056.843136579671,crosses_zero,455,False +w03_2012m03_m08,linear-006,development_admissible_exact_frontier,terminal_default,-0.004860510289310175,0.028868108769244564,crosses_zero,455,False +w03_2012m03_m08,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.006819260286160139,0.03528687538435989,guardrail_higher,455,False +w03_2012m03_m08,linear-007,development_admissible_exact_frontier,standardized_payoff,-87024.77258902372,96099.60101237547,crosses_zero,345,False +w03_2012m03_m08,linear-007,development_admissible_exact_frontier,terminal_default,-0.012019973084688901,0.012401462479230334,crosses_zero,345,False +w03_2012m03_m08,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.008921276700781635,0.00810022662395258,crosses_zero,345,False +w03_2012m03_m08,linear-008,development_admissible_exact_frontier,standardized_payoff,-234178.08418057035,53843.00498808077,crosses_zero,470,False +w03_2012m03_m08,linear-008,development_admissible_exact_frontier,terminal_default,-0.010980834440844425,0.028607157454441708,crosses_zero,470,False +w03_2012m03_m08,linear-008,development_admissible_exact_frontier,funded_miscoverage,0.0010857881465610288,0.030322133052731345,guardrail_higher,470,False +w03_2012m03_m08,linear-009,development_admissible_exact_frontier,standardized_payoff,-265118.97580380255,-1120.2066905968968,guardrail_lower,444,False +w03_2012m03_m08,linear-009,development_admissible_exact_frontier,terminal_default,-0.007049168611562984,0.032036466020564265,crosses_zero,444,False +w03_2012m03_m08,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.003911279124279944,0.0368316445073644,guardrail_higher,444,False +w04_2012m04_m09,linear-001,development_admissible_exact_frontier,standardized_payoff,-124999.2710601422,171801.63651145075,crosses_zero,380,False +w04_2012m04_m09,linear-001,development_admissible_exact_frontier,terminal_default,-0.021174464479627023,0.01644572296836891,crosses_zero,380,False +w04_2012m04_m09,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.014277797812960357,0.020525287964342893,crosses_zero,380,False +w04_2012m04_m09,linear-002,development_admissible_exact_frontier,standardized_payoff,-214044.94551776227,42677.86966752786,crosses_zero,400,False +w04_2012m04_m09,linear-002,development_admissible_exact_frontier,terminal_default,-0.009637874922731212,0.026922094521345908,crosses_zero,400,False +w04_2012m04_m09,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0007712082560645444,0.029950427854679246,crosses_zero,400,False +w04_2012m04_m09,linear-003,development_admissible_exact_frontier,standardized_payoff,-286918.1123368604,9723.769119739969,crosses_zero,461,False +w04_2012m04_m09,linear-003,development_admissible_exact_frontier,terminal_default,-0.0066609538992372765,0.03651510318953319,crosses_zero,461,False +w04_2012m04_m09,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.008798781340958977,0.0466102400337515,guardrail_higher,461,False +w04_2012m04_m09,linear-004,development_admissible_exact_frontier,standardized_payoff,-97434.58154089143,44973.25935276075,crosses_zero,353,False +w04_2012m04_m09,linear-004,development_admissible_exact_frontier,terminal_default,-0.008058079377406109,0.014891225603169937,crosses_zero,353,False +w04_2012m04_m09,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.004108072394645407,0.01258019166224476,crosses_zero,353,False +w04_2012m04_m09,linear-005,development_admissible_exact_frontier,standardized_payoff,-237125.2980995345,44593.45173037635,crosses_zero,455,False +w04_2012m04_m09,linear-005,development_admissible_exact_frontier,terminal_default,-0.010591455867145574,0.02820402387291286,crosses_zero,455,False +w04_2012m04_m09,linear-005,development_admissible_exact_frontier,funded_miscoverage,-2.2577095345447003e-05,0.03258097561208666,crosses_zero,455,False +w04_2012m04_m09,linear-006,development_admissible_exact_frontier,standardized_payoff,-247573.34216693172,13017.68312408992,crosses_zero,471,False +w04_2012m04_m09,linear-006,development_admissible_exact_frontier,terminal_default,-0.004885902382961958,0.029817425502071157,crosses_zero,471,False +w04_2012m04_m09,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.007313503512815489,0.03548464063759782,guardrail_higher,471,False +w04_2012m04_m09,linear-007,development_admissible_exact_frontier,standardized_payoff,-105270.15825062637,77649.22619354632,crosses_zero,346,False +w04_2012m04_m09,linear-007,development_admissible_exact_frontier,terminal_default,-0.010340522677448317,0.014057102832713681,crosses_zero,346,False +w04_2012m04_m09,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.007650159626874385,0.009567631630620332,crosses_zero,346,False +w04_2012m04_m09,linear-008,development_admissible_exact_frontier,standardized_payoff,-232552.90687869646,55683.97415564628,crosses_zero,483,False +w04_2012m04_m09,linear-008,development_admissible_exact_frontier,terminal_default,-0.011197350749533076,0.02864453022061231,crosses_zero,483,False +w04_2012m04_m09,linear-008,development_admissible_exact_frontier,funded_miscoverage,-0.0002453133238711517,0.03084093979271676,crosses_zero,483,False +w04_2012m04_m09,linear-009,development_admissible_exact_frontier,standardized_payoff,-265198.2441925329,-951.9791559278965,guardrail_lower,446,False +w04_2012m04_m09,linear-009,development_admissible_exact_frontier,terminal_default,-0.007062538050317314,0.03203881081060214,crosses_zero,446,False +w04_2012m04_m09,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.003314498199108962,0.03589446984679026,guardrail_higher,446,False +w05_2012m05_m10,linear-001,development_admissible_exact_frontier,standardized_payoff,-125240.35857434799,171536.89165321834,crosses_zero,382,False +w05_2012m05_m10,linear-001,development_admissible_exact_frontier,terminal_default,-0.02112546477744199,0.016492510670177443,crosses_zero,382,False +w05_2012m05_m10,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.014228798110775324,0.020572075666151424,crosses_zero,382,False +w05_2012m05_m10,linear-002,development_admissible_exact_frontier,standardized_payoff,-214181.46773671597,42692.93145658399,crosses_zero,400,False +w05_2012m05_m10,linear-002,development_admissible_exact_frontier,terminal_default,-0.009644609493094699,0.02694805582965353,crosses_zero,400,False +w05_2012m05_m10,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0011779428264280314,0.029976389162986856,crosses_zero,400,False +w05_2012m05_m10,linear-003,development_admissible_exact_frontier,standardized_payoff,-287047.2200259118,9417.508885471878,crosses_zero,462,False +w05_2012m05_m10,linear-003,development_admissible_exact_frontier,terminal_default,-0.006641588865711542,0.036508960464413674,crosses_zero,462,False +w05_2012m05_m10,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.008097552528847862,0.045883503462995134,guardrail_higher,462,False +w05_2012m05_m10,linear-004,development_admissible_exact_frontier,standardized_payoff,-97665.62130924361,43986.52498015639,crosses_zero,354,False +w05_2012m05_m10,linear-004,development_admissible_exact_frontier,terminal_default,-0.008062441983340878,0.014909670635614153,crosses_zero,354,False +w05_2012m05_m10,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.003997186330088373,0.012595784038608017,crosses_zero,354,False +w05_2012m05_m10,linear-005,development_admissible_exact_frontier,standardized_payoff,-237024.49104346446,44578.07764602267,crosses_zero,454,False +w05_2012m05_m10,linear-005,development_admissible_exact_frontier,terminal_default,-0.010612610866399893,0.02820817361336099,crosses_zero,454,False +w05_2012m05_m10,linear-005,development_admissible_exact_frontier,funded_miscoverage,-0.0015879793953217152,0.032574957375920116,crosses_zero,454,False +w05_2012m05_m10,linear-006,development_admissible_exact_frontier,standardized_payoff,-245815.2082947563,14775.816996265334,crosses_zero,471,False +w05_2012m05_m10,linear-006,development_admissible_exact_frontier,terminal_default,-0.005085393722885425,0.029636275196381356,crosses_zero,471,False +w05_2012m05_m10,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.006881138907980647,0.03546165589978112,guardrail_higher,471,False +w05_2012m05_m10,linear-007,development_admissible_exact_frontier,standardized_payoff,-105412.85157570001,77347.11353651903,crosses_zero,346,False +w05_2012m05_m10,linear-007,development_admissible_exact_frontier,terminal_default,-0.010285483768995978,0.014094997112252363,crosses_zero,346,False +w05_2012m05_m10,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.007717138295617959,0.009517188409080064,crosses_zero,346,False +w05_2012m05_m10,linear-008,development_admissible_exact_frontier,standardized_payoff,-232296.14992810198,56175.62982933751,crosses_zero,481,False +w05_2012m05_m10,linear-008,development_admissible_exact_frontier,terminal_default,-0.011233223090865536,0.02861915558896682,crosses_zero,481,False +w05_2012m05_m10,linear-008,development_admissible_exact_frontier,funded_miscoverage,-0.0013477430921100169,0.030883199476368764,crosses_zero,481,False +w05_2012m05_m10,linear-009,development_admissible_exact_frontier,standardized_payoff,-265211.11833481013,-58.87683862279664,guardrail_lower,445,False +w05_2012m05_m10,linear-009,development_admissible_exact_frontier,terminal_default,-0.007156716485710656,0.0320377792548214,crosses_zero,445,False +w05_2012m05_m10,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.002130450642376571,0.0358866130495753,guardrail_higher,445,False +w06_2012m06_m11,linear-001,development_admissible_exact_frontier,standardized_payoff,-125226.98005226035,171477.41096926853,crosses_zero,382,False +w06_2012m06_m11,linear-001,development_admissible_exact_frontier,terminal_default,-0.021116271033098235,0.016494891959538734,crosses_zero,382,False +w06_2012m06_m11,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.01421960436643157,0.020241123622179384,crosses_zero,382,False +w06_2012m06_m11,linear-002,development_admissible_exact_frontier,standardized_payoff,-213260.61352245504,43613.821861474964,crosses_zero,400,False +w06_2012m06_m11,linear-002,development_admissible_exact_frontier,terminal_default,-0.009741222257514469,0.02684594356090761,crosses_zero,400,False +w06_2012m06_m11,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0019412222575144692,0.029207610227574274,crosses_zero,400,False +w06_2012m06_m11,linear-003,development_admissible_exact_frontier,standardized_payoff,-287019.21368684067,9495.311839585163,crosses_zero,461,False +w06_2012m06_m11,linear-003,development_admissible_exact_frontier,terminal_default,-0.006653218839196171,0.03650450058287602,crosses_zero,461,False +w06_2012m06_m11,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.007423807417284139,0.04521692844337839,guardrail_higher,461,False +w06_2012m06_m11,linear-004,development_admissible_exact_frontier,standardized_payoff,-97662.34423891819,43820.88600776362,crosses_zero,355,False +w06_2012m06_m11,linear-004,development_admissible_exact_frontier,terminal_default,-0.008069111954213466,0.014908034244885567,crosses_zero,355,False +w06_2012m06_m11,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.0046679717732441605,0.01259423028156912,crosses_zero,355,False +w06_2012m06_m11,linear-005,development_admissible_exact_frontier,standardized_payoff,-237013.24169197216,44594.29135251651,crosses_zero,452,False +w06_2012m06_m11,linear-005,development_admissible_exact_frontier,terminal_default,-0.010612111684765765,0.028207033691290198,crosses_zero,452,False +w06_2012m06_m11,linear-005,development_admissible_exact_frontier,funded_miscoverage,-0.002256772975604307,0.03190668862437222,crosses_zero,452,False +w06_2012m06_m11,linear-006,development_admissible_exact_frontier,standardized_payoff,-245826.513920838,14764.511370183653,crosses_zero,471,False +w06_2012m06_m11,linear-006,development_admissible_exact_frontier,terminal_default,-0.005068378142374051,0.029641066035218086,crosses_zero,471,False +w06_2012m06_m11,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.0062301605704651875,0.03480220238909644,guardrail_higher,471,False +w06_2012m06_m11,linear-007,development_admissible_exact_frontier,standardized_payoff,-105386.66369745717,77342.49616704136,crosses_zero,346,False +w06_2012m06_m11,linear-007,development_admissible_exact_frontier,terminal_default,-0.010277370822701828,0.014099797131975506,crosses_zero,346,False +w06_2012m06_m11,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.008381843013813442,0.00884917076431357,crosses_zero,346,False +w06_2012m06_m11,linear-008,development_admissible_exact_frontier,standardized_payoff,-232266.58906155173,56221.16248345987,crosses_zero,471,False +w06_2012m06_m11,linear-008,development_admissible_exact_frontier,terminal_default,-0.011236181820237565,0.028397842384791185,crosses_zero,471,False +w06_2012m06_m11,linear-008,development_admissible_exact_frontier,funded_miscoverage,-0.0019278484869042342,0.029624141895983017,crosses_zero,471,False +w06_2012m06_m11,linear-009,development_admissible_exact_frontier,standardized_payoff,-265190.25076861004,-273.2261989091203,guardrail_lower,445,False +w06_2012m06_m11,linear-009,development_admissible_exact_frontier,terminal_default,-0.007132449054708613,0.032036716012280184,crosses_zero,445,False +w06_2012m06_m11,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.001489767580759185,0.03522059931441465,guardrail_higher,445,False +w07_2012m07_m12,linear-001,development_admissible_exact_frontier,standardized_payoff,-125307.66884568967,170977.90126532305,crosses_zero,381,False +w07_2012m07_m12,linear-001,development_admissible_exact_frontier,terminal_default,-0.021043269766141244,0.016528732786054757,crosses_zero,381,False +w07_2012m07_m12,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.014146603099474579,0.02094163111536207,crosses_zero,381,False +w07_2012m07_m12,linear-002,development_admissible_exact_frontier,standardized_payoff,-213358.80001097452,43735.87892770971,crosses_zero,401,False +w07_2012m07_m12,linear-002,development_admissible_exact_frontier,terminal_default,-0.009759435807105747,0.026876384086161345,crosses_zero,401,False +w07_2012m07_m12,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.0031061024737724123,0.02990471741949468,crosses_zero,401,False +w07_2012m07_m12,linear-003,development_admissible_exact_frontier,standardized_payoff,-287005.61432112695,9302.345457300282,crosses_zero,462,False +w07_2012m07_m12,linear-003,development_admissible_exact_frontier,terminal_default,-0.006621520186674321,0.03650645634213108,crosses_zero,462,False +w07_2012m07_m12,linear-003,development_admissible_exact_frontier,funded_miscoverage,0.007754942399757395,0.04586958559522946,guardrail_higher,462,False +w07_2012m07_m12,linear-004,development_admissible_exact_frontier,standardized_payoff,-97881.64350588876,43230.224885251024,crosses_zero,356,False +w07_2012m07_m12,linear-004,development_admissible_exact_frontier,terminal_default,-0.008056320186599797,0.014925559753893721,crosses_zero,356,False +w07_2012m07_m12,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.004405345085675277,0.012607942779736966,crosses_zero,356,False +w07_2012m07_m12,linear-005,development_admissible_exact_frontier,standardized_payoff,-230705.95376461378,50752.48004215769,crosses_zero,454,False +w07_2012m07_m12,linear-005,development_admissible_exact_frontier,terminal_default,-0.011239884530184506,0.027621103804419657,crosses_zero,454,False +w07_2012m07_m12,linear-005,development_admissible_exact_frontier,funded_miscoverage,-0.0026615804648330507,0.03227160356486316,crosses_zero,454,False +w07_2012m07_m12,linear-006,development_admissible_exact_frontier,standardized_payoff,-245814.734201435,14776.291089586608,crosses_zero,472,False +w07_2012m07_m12,linear-006,development_admissible_exact_frontier,terminal_default,-0.005132350645017049,0.0296279120061189,crosses_zero,472,False +w07_2012m07_m12,linear-006,development_admissible_exact_frontier,funded_miscoverage,0.005025880829670295,0.035445377868473696,guardrail_higher,472,False +w07_2012m07_m12,linear-007,development_admissible_exact_frontier,standardized_payoff,-105902.03762146425,76648.74764170784,crosses_zero,347,False +w07_2012m07_m12,linear-007,development_admissible_exact_frontier,terminal_default,-0.010192709914966687,0.014165274880994031,crosses_zero,347,False +w07_2012m07_m12,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.007355083493904725,0.010641747125505674,crosses_zero,347,False +w07_2012m07_m12,linear-008,development_admissible_exact_frontier,standardized_payoff,-233082.40950094102,55736.07584258785,crosses_zero,471,False +w07_2012m07_m12,linear-008,development_admissible_exact_frontier,terminal_default,-0.011161716149828676,0.028509682150721694,crosses_zero,471,False +w07_2012m07_m12,linear-008,development_admissible_exact_frontier,funded_miscoverage,-0.0012667048502517262,0.030380659593021123,crosses_zero,471,False +w07_2012m07_m12,linear-009,development_admissible_exact_frontier,standardized_payoff,-265378.70401441393,911.7767068198445,crosses_zero,443,False +w07_2012m07_m12,linear-009,development_admissible_exact_frontier,terminal_default,-0.007257406851538429,0.03205944153368305,crosses_zero,443,False +w07_2012m07_m12,linear-009,development_admissible_exact_frontier,funded_miscoverage,0.00021112179113676535,0.035902970176358254,guardrail_higher,443,False +w08_2012m08_2013m01,linear-001,development_admissible_exact_frontier,standardized_payoff,-28778.67927795329,98688.05763732335,crosses_zero,386,False +w08_2012m08_2013m01,linear-001,development_admissible_exact_frontier,terminal_default,-0.006074486926858859,0.00706040235197988,crosses_zero,386,False +w08_2012m08_2013m01,linear-001,development_admissible_exact_frontier,funded_miscoverage,-0.005672820260192193,0.005298725378215204,crosses_zero,386,False +w08_2012m08_2013m01,linear-002,development_admissible_exact_frontier,standardized_payoff,-173223.78732016464,143636.13616856973,crosses_zero,778,False +w08_2012m08_2013m01,linear-002,development_admissible_exact_frontier,terminal_default,-0.018677720167983834,0.02987061137944106,crosses_zero,778,False +w08_2012m08_2013m01,linear-002,development_admissible_exact_frontier,funded_miscoverage,-0.01030949214023024,0.02907346760379815,crosses_zero,778,False +w08_2012m08_2013m01,linear-003,development_admissible_exact_frontier,standardized_payoff,-152265.6555837969,122959.11764731134,crosses_zero,765,False +w08_2012m08_2013m01,linear-003,development_admissible_exact_frontier,terminal_default,-0.01746258009744204,0.026723666096617575,crosses_zero,765,False +w08_2012m08_2013m01,linear-003,development_admissible_exact_frontier,funded_miscoverage,-0.004783796550868668,0.030370412835401473,crosses_zero,765,False +w08_2012m08_2013m01,linear-004,development_admissible_exact_frontier,standardized_payoff,-41417.020384164716,87490.21725019442,crosses_zero,509,False +w08_2012m08_2013m01,linear-004,development_admissible_exact_frontier,terminal_default,-0.0018245185973258925,0.01229897406686489,crosses_zero,509,False +w08_2012m08_2013m01,linear-004,development_admissible_exact_frontier,funded_miscoverage,-0.003892325922512743,0.008403917032016914,crosses_zero,509,False +w08_2012m08_2013m01,linear-005,development_admissible_exact_frontier,standardized_payoff,-71782.51735976638,139240.3906267634,crosses_zero,635,False +w08_2012m08_2013m01,linear-005,development_admissible_exact_frontier,terminal_default,-0.015250031640779071,0.016042028697511673,crosses_zero,635,False +w08_2012m08_2013m01,linear-005,development_admissible_exact_frontier,funded_miscoverage,-0.01050917431652687,0.01264181497160376,crosses_zero,635,False +w08_2012m08_2013m01,linear-006,development_admissible_exact_frontier,standardized_payoff,-202770.6057618514,161095.6692524375,crosses_zero,834,False +w08_2012m08_2013m01,linear-006,development_admissible_exact_frontier,terminal_default,-0.022390367788155105,0.0325846083245228,crosses_zero,834,False +w08_2012m08_2013m01,linear-006,development_admissible_exact_frontier,funded_miscoverage,-0.01093291795538864,0.03364505102335313,crosses_zero,834,False +w08_2012m08_2013m01,linear-007,development_admissible_exact_frontier,standardized_payoff,-25213.669566019344,129181.59228602203,crosses_zero,677,False +w08_2012m08_2013m01,linear-007,development_admissible_exact_frontier,terminal_default,-0.0009008075892627944,0.014439351741594657,crosses_zero,677,False +w08_2012m08_2013m01,linear-007,development_admissible_exact_frontier,funded_miscoverage,-0.009005016469837774,0.004507893151358549,crosses_zero,677,False +w08_2012m08_2013m01,linear-008,development_admissible_exact_frontier,standardized_payoff,-35377.61146143789,179845.0340250919,crosses_zero,514,False +w08_2012m08_2013m01,linear-008,development_admissible_exact_frontier,terminal_default,-0.018963746520195855,0.0130532329357471,crosses_zero,514,False +w08_2012m08_2013m01,linear-008,development_admissible_exact_frontier,funded_miscoverage,-0.017070795285041695,0.007373726378258207,crosses_zero,514,False +w08_2012m08_2013m01,linear-009,development_admissible_exact_frontier,standardized_payoff,-148658.5407787502,138490.80422970265,crosses_zero,769,False +w08_2012m08_2013m01,linear-009,development_admissible_exact_frontier,terminal_default,-0.016850394896097194,0.0282535641378876,crosses_zero,769,False +w08_2012m08_2013m01,linear-009,development_admissible_exact_frontier,funded_miscoverage,-0.010199912178950747,0.02592936484168276,crosses_zero,769,False diff --git a/reports/crpto/tables/crpto_ijds_v4_table4_direction_summary.csv b/reports/crpto/tables/crpto_ijds_v4_table4_direction_summary.csv new file mode 100644 index 0000000..a98b8e2 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table4_direction_summary.csv @@ -0,0 +1,6 @@ +metric,direction,cells +funded_miscoverage,crosses_zero,45 +funded_miscoverage,guardrail_higher,27 +standardized_payoff,crosses_zero,66 +standardized_payoff,guardrail_lower,6 +terminal_default,crosses_zero,72 diff --git a/reports/crpto/tables/crpto_ijds_v4_table5_two_ruler_tracks.csv b/reports/crpto/tables/crpto_ijds_v4_table5_two_ruler_tracks.csv new file mode 100644 index 0000000..b0e032f --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table5_two_ruler_tracks.csv @@ -0,0 +1,7 @@ +ruler,coordinate,ruler_semantics,window_cells,active_months_per_window_min,active_months_per_window_max,expected_objective_difference_usd_min,expected_objective_difference_usd_max,payoff_bound_usd_lower_min,payoff_bound_usd_upper_max,payoff_identification_width_usd_min,payoff_identification_width_usd_max,default_bound_pp_lower_min,default_bound_pp_upper_max,default_identification_width_pp_min,default_identification_width_pp_max,miscoverage_bound_pp_lower_min,miscoverage_bound_pp_upper_max,miscoverage_identification_width_pp_min,miscoverage_identification_width_pp_max,payoff_direction_pattern,default_direction_pattern,miscoverage_direction_pattern +objective_matched,0.25,common_plugin_objective_floor,8,4,4,0.0,0.0,-9134.339201705214,5603.660798333496,14738.0,14738.0,-0.006789927307658072,0.12654340602615935,0.13333333333333333,0.13333333333333333,-0.006789927307658072,0.12654340602615935,0.13333333333333333,0.13333333333333333,crosses_zero:8,crosses_zero:8,crosses_zero:8 +objective_matched,0.5,common_plugin_objective_floor,8,15,15,0.0,4.656612873077393e-10,-82616.17453934051,-27958.36861160415,53530.42261970967,54370.44762289172,0.457176922370002,1.0972986534287477,0.6281743811120005,0.6375359246695269,1.0153517160736112,1.9321319867620808,0.6281743811120005,0.6375359246695269,gamma_1_lower:8,gamma_1_higher:8,gamma_1_higher:8 +objective_matched,0.75,common_plugin_objective_floor,8,15,15,0.0,4.656612873077393e-10,-179484.66459580814,92558.17998965128,109550.45677067185,271561.8753013419,-0.4352498618487788,2.494842541954273,1.2006435014285781,2.9246052959625377,1.3252372459918247,4.184767620139838,1.1339768347619112,2.645938629295871,gamma_1_lower:1;crosses_zero:7,gamma_1_higher:1;crosses_zero:7,gamma_1_higher:8 +normalized_score,0.25,common_relative_score_relaxation,8,15,15,425196.4593238123,557294.0146438954,-626374.6147907167,-195967.628254245,340845.41518178507,373705.3081783684,8.482852571867097,13.424612654302775,3.7599027571533226,4.101884671392359,8.390955538535708,13.753589682046982,3.639902757153319,3.9432180047256917,gamma_1_lower:8,gamma_1_higher:8,gamma_1_higher:8 +normalized_score,0.5,common_relative_score_relaxation,8,15,15,152030.9877972738,226847.9705495569,-259658.18448523604,-54025.8191528258,146499.38968489924,198657.05281574934,3.2213520803827533,6.563699817357987,1.5345657876974395,2.0382675456422694,2.1069714896837946,5.21415906991543,1.5345657876974395,1.9182675456351144,gamma_1_lower:8,gamma_1_higher:8,gamma_1_higher:8 +normalized_score,0.75,common_relative_score_relaxation,8,15,15,28263.397028705105,51201.887608815916,-135781.22300933156,9812.591895865517,83832.63333388747,97897.24126478202,1.4391554644757423,2.3807174501084845,0.8041480359892984,0.9415619856327422,0.34472769373846956,1.6990552129348044,0.7256480359893016,0.9415619856327422,gamma_1_lower:7;crosses_zero:1,gamma_1_higher:8,gamma_1_higher:8 diff --git a/reports/crpto/tables/crpto_ijds_v4_table6_credit_controls.csv b/reports/crpto/tables/crpto_ijds_v4_table6_credit_controls.csv new file mode 100644 index 0000000..c836bdd --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_table6_credit_controls.csv @@ -0,0 +1,6 @@ +learner,learner_label,candidate_rows,resolved_rows,unresolved_rows,default_rate,roc_auc,gini,ks,average_precision,brier,log_loss,ece_10,mean_calibration_error,calibration_intercept,calibration_slope,coverage_lower_min,coverage_upper_max,windows_upper_below_0_90 +catboost_platt,CatBoost,376890,364814,12076,0.156167252353254,0.6406052990079556,0.2812105980159112,0.20588181451177345,0.2350998561447269,0.12987780627268664,0.42846570011698926,0.047215103762021676,-0.0471089772795769,0.020228532871166095,0.7954274631320676,0.8424845445620738,0.8825970442304121,8 +numeric_logistic_platt,Numeric logistic,376890,364814,12076,0.156167252353254,0.6420446945807546,0.2840893891615093,0.2096638411920359,0.24160749827700292,0.128845865115775,0.4327000703812235,0.031149374770624953,-0.028922756555184087,-0.5991391826510897,0.5432095920187805,0.8500305128817427,0.8962217092520364,8 +catboost_monotonic_platt,Monotonic CatBoost,376890,364814,12076,0.156167252353254,0.6519537792141734,0.3039075584283468,0.21969571391561254,0.24540282093264182,0.12861326735049058,0.42267695477753886,0.04156277982947754,-0.04126410836574358,0.052284509244371016,0.8351490384153631,0.8483960837379607,0.8864894266231527,8 +woe_scorecard_platform_platt,Platform-signal WOE scorecard,376890,364814,12076,0.156167252353254,0.6330661809401453,0.26613236188029066,0.18843820687722823,0.22388791917890766,0.12948509376427816,0.4239586771759354,0.035967619770033946,-0.03551023236967947,0.15367586807582181,0.9186554972959701,0.8489081694924249,0.8949083286900688,8 +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,376890,364814,12076,0.156167252353254,0.6129392722275023,0.22587854445500466,0.15809243291132968,0.2151918293521198,0.13019024412756694,0.42796595827034617,0.03176877520031669,-0.029751737101973152,-0.28557359284456246,0.7098359251738742,0.8520125235479848,0.8977261269866539,8 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv b/reports/crpto/tables/crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv new file mode 100644 index 0000000..5a0ee9b --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv @@ -0,0 +1,17 @@ +origin,window_id,candidate_rows,resolved_rows,unresolved_rows,coverage_resolved,coverage_lower,coverage_upper,mean_width +primary_2016,w01_2012m01_m06,376890,364814,12076,0.8770469335058413,0.8547135769057285,0.8805221682719095,0.6649918872956118 +primary_2016,w02_2012m02_m07,376890,364814,12076,0.879165821487114,0.8570245960359787,0.8825970442304121,0.665340945235394 +primary_2016,w03_2012m03_m08,376890,364814,12076,0.8777349553471084,0.8556528430045902,0.8811430390830216,0.6653200651843035 +primary_2016,w04_2012m04_m09,376890,364814,12076,0.8748732230670972,0.8525697153015469,0.8783411605508239,0.665031208384886 +primary_2016,w05_2012m05_m10,376890,364814,12076,0.87251311627295,0.8499323410013532,0.8760327947146382,0.6647076525088493 +primary_2016,w06_2012m06_m11,376890,364814,12076,0.8721129123334083,0.8494255618350182,0.8756692934277905,0.6645688455708323 +primary_2016,w07_2012m07_m12,376890,364814,12076,0.8694814343747773,0.8465175515402372,0.8730929448910822,0.6641544351090525 +primary_2016,w08_2012m08_2013m01,376890,364814,12076,0.8658549288130389,0.8424845445620738,0.8695640637851894,0.5005686881638921 +rolling_2017,w01_2013m01_m06,77105,66091,11014,0.8571061112708236,0.7535568380779456,0.8747681732702159,0.562496881282411 +rolling_2017,w02_2013m02_m07,77105,66091,11014,0.8565916690623534,0.7524285065819337,0.8743401854613838,0.5624542821226854 +rolling_2017,w03_2013m03_m08,77105,66091,11014,0.8546700761071855,0.7501588742623695,0.8725374489332728,0.5622224070109993 +rolling_2017,w04_2013m04_m09,77105,66091,11014,0.8552601715816072,0.7506257700538227,0.8731988846378316,0.5621829530516125 +rolling_2017,w05_2013m05_m10,77105,66091,11014,0.8554720007262714,0.7508203099669282,0.8734063938784774,0.5621735581049848 +rolling_2017,w06_2013m06_m11,77105,66091,11014,0.8561831414262153,0.7517411322222942,0.8739900136177939,0.5622719357684526 +rolling_2017,w07_2013m07_m12,77105,66091,11014,0.855956181628361,0.7509759418974127,0.873886258997471,0.5621830723291171 +rolling_2017,w08_2013m08_2014m01,77105,66091,11014,0.8553963474603199,0.7498994877115621,0.8735231178263407,0.5620254873690599 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS11_fit_label_completion.csv b/reports/crpto/tables/crpto_ijds_v4_tableS11_fit_label_completion.csv new file mode 100644 index 0000000..6e4ef7a --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS11_fit_label_completion.csv @@ -0,0 +1,5 @@ +fit_label_scenario,windows,coverage_lower_min,coverage_upper_max,windows_upper_below_nominal,all_windows_upper_below_nominal,mean_width_min,mean_width_max,w7_w8_stratum2_crossing +all_unavailable_default,8,0.8496617050067659,0.8846692668948499,8,True,0.6522207777018078,0.6537859842285183,False +all_unavailable_nondefault,8,0.8424659715036218,0.8832524078643637,8,True,0.49225574027300617,0.6485696557890056,True +hindsight_terminal,8,0.8489798084321685,0.8845790548966542,8,True,0.5003892753736984,0.6537743949931867,True +observed_only,8,0.8424845445620738,0.8825943909363475,8,True,0.5005686859367924,0.6653409371377675,True diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS12_allocation_granularity.csv b/reports/crpto/tables/crpto_ijds_v4_tableS12_allocation_granularity.csv new file mode 100644 index 0000000..8c04d3a --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS12_allocation_granularity.csv @@ -0,0 +1,2 @@ +tracks,cash_share_max,payoff_rate_perturbation_abs_max,default_rate_perturbation_abs_max,miscoverage_rate_perturbation_abs_max,portfolios,source_rows,rounded_positive_rows,changed_rows,cash_residual_min,cash_residual_mean,cash_residual_max,monthly_cash_share_max +96,3.333333333332966e-05,3.4483590284581524e-06,1.2842831024732755e-05,1.1901916329666367e-05,1440,143175,143167,2985,25.0,26.006944444444443,75.0,7.5e-05 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS1_named_comparators.csv b/reports/crpto/tables/crpto_ijds_v4_tableS1_named_comparators.csv new file mode 100644 index 0000000..69a9fcf --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS1_named_comparators.csv @@ -0,0 +1,10 @@ +comparator_rule,metric,guardrail_lower,crosses_zero,guardrail_higher +c0_same_numeric_cap,payoff,0,54,18 +c0_same_numeric_cap,default,72,0,0 +c0_same_numeric_cap,miscoverage,15,52,5 +c1_development_mean,payoff,28,40,4 +c1_development_mean,default,3,35,34 +c1_development_mean,miscoverage,3,24,45 +c2_contemporaneous,payoff,35,31,6 +c2_contemporaneous,default,7,58,7 +c2_contemporaneous,miscoverage,5,20,47 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS2_credit_prediction_metrics.csv b/reports/crpto/tables/crpto_ijds_v4_tableS2_credit_prediction_metrics.csv new file mode 100644 index 0000000..b4272a4 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS2_credit_prediction_metrics.csv @@ -0,0 +1,31 @@ +learner,learner_label,role,candidate_rows,resolved_rows,unresolved_rows,rows,default_rate,roc_auc,brier,log_loss,ece_10,gini,ks,average_precision,mean_calibration_error,calibration_intercept,calibration_slope,calibration_iterations,calibration_optimizer_success +catboost_platt,CatBoost,pd_development,17433,17392,41,17392,0.1343146274149034,0.8101500403061448,0.09948264323887722,0.330802559850395,0.04759062394314454,0.6203000806122896,0.46386959006012257,0.4567434124947417,-0.0007201320599743433,1.6909268326029592,2.05593147261856,13,True +catboost_platt,CatBoost,probability_calibration,14101,14077,24,14077,0.10478084819208638,0.6763269214733201,0.09020572858912285,0.3165431267613625,0.003272660824190425,0.35265384294664015,0.25828259705884726,0.19101817467894838,-2.9183134156632318e-05,0.0009053975249333084,1.000292827809217,14,True +catboost_platt,CatBoost,conformal_fit,49007,48953,54,48953,0.1336996711131085,0.6378353991182315,0.11291670496422315,0.38044140358977424,0.012469510090842538,0.275670798236463,0.2049914728457623,0.19926513892534806,-0.011282653117504954,-0.14802927635734306,0.8650592919239838,12,True +catboost_platt,CatBoost,policy_development,94885,94808,77,94808,0.12240528225466206,0.6429962166990311,0.10461910209260085,0.3579150652390907,0.003789132711363335,0.28599243339806213,0.207602774323704,0.18855734783762085,0.003010113936265338,-0.21043124170195918,0.9009170460185505,13,True +catboost_platt,CatBoost,primary_oot,376890,364814,12076,364814,0.156167252353254,0.6406052990079556,0.12987780627268664,0.42846570011698926,0.047215103762021676,0.2812105980159112,0.20588181451177345,0.2350998561447269,-0.0471089772795769,0.020228532871166095,0.7954274631320676,15,True +catboost_platt,CatBoost,censored_extension,88227,59291,28936,59291,0.19643790794555666,0.6527486506146079,0.15928357299240833,0.5081384508531989,0.08870185034779746,0.3054973012292157,0.22484506253006936,0.2985385882455722,-0.08870185034779744,0.4514207584220385,0.8581133191550745,15,True +numeric_logistic_platt,Numeric logistic,pd_development,17433,17392,41,17392,0.1343146274149034,0.6900394349679007,0.10992098244504024,0.36797560706921345,0.012073170617693566,0.3800788699358013,0.27027999577831796,0.25601651001828984,-0.004484055866539688,0.17103782488683247,1.0732185935357967,13,True +numeric_logistic_platt,Numeric logistic,probability_calibration,14101,14077,24,14077,0.10478084819208638,0.6568795913481584,0.090999768751691,0.32057387067778387,0.0008756912414165813,0.31375918269631686,0.2302584739037925,0.1776126126061553,-9.929537100919483e-05,0.002427870374444713,1.0006596079870937,14,True +numeric_logistic_platt,Numeric logistic,conformal_fit,49007,48953,54,48953,0.1336996711131085,0.6332270897760761,0.11309485185584098,0.3815450957740176,0.004623371738702063,0.2664541795521522,0.19567074707641968,0.19682246439485818,0.0014726597382865303,-0.28433730256642437,0.8464897182725436,9,True +numeric_logistic_platt,Numeric logistic,policy_development,94885,94808,77,94808,0.12240528225466206,0.6412747033991495,0.10534859439802753,0.36181137555800175,0.02147575714663183,0.282549406798299,0.20863736027481322,0.18608407049665526,0.02146943189872265,-0.393237105726378,0.8808976977048312,13,True +numeric_logistic_platt,Numeric logistic,primary_oot,376890,364814,12076,364814,0.156167252353254,0.6420446945807546,0.128845865115775,0.4327000703812235,0.031149374770624953,0.2840893891615093,0.2096638411920359,0.24160749827700292,-0.028922756555184087,-0.5991391826510897,0.5432095920187805,13,True +numeric_logistic_platt,Numeric logistic,censored_extension,88227,59291,28936,59291,0.19643790794555666,0.6586200783223694,0.15686474677417162,0.5051221784768172,0.07054214833729724,0.3172401566447387,0.23585259466050612,0.3005013938341795,-0.06494096361740603,-0.2871633643589278,0.5681696936805429,14,True +catboost_monotonic_platt,Monotonic CatBoost,pd_development,17433,17392,41,17392,0.1343146274149034,0.7770689198681088,0.10261955800473381,0.34130330977773565,0.03265059483156023,0.5541378397362176,0.4106709380868502,0.37799067702614,0.0009180171125425796,1.1177196268343803,1.6975911799339514,13,True +catboost_monotonic_platt,Monotonic CatBoost,probability_calibration,14101,14077,24,14077,0.10478084819208638,0.6780911827285957,0.09005445751312442,0.3157471244151023,0.004577827659153225,0.3561823654571914,0.25221345011149704,0.19439365496940159,-9.213914682165703e-05,0.0033058062368193676,1.0011511234584007,14,True +catboost_monotonic_platt,Monotonic CatBoost,conformal_fit,49007,48953,54,48953,0.1336996711131085,0.6380285679122192,0.11284217302179593,0.3801190713979001,0.010098955198479514,0.27605713582443836,0.20539978403256148,0.20095757145204168,-0.006849864323418164,-0.20673705652543603,0.8527775329213096,11,True +catboost_monotonic_platt,Monotonic CatBoost,policy_development,94885,94808,77,94808,0.12240528225466206,0.6447525798509144,0.10464382640540693,0.3578408001570101,0.008111845634861841,0.2895051597018288,0.2077148209994314,0.19067923627384495,0.008103602890827968,-0.2534068779736748,0.9007045783598007,14,True +catboost_monotonic_platt,Monotonic CatBoost,primary_oot,376890,364814,12076,364814,0.156167252353254,0.6519537792141734,0.12861326735049058,0.42267695477753886,0.04156277982947754,0.3039075584283468,0.21969571391561254,0.24540282093264182,-0.04126410836574358,0.052284509244371016,0.8351490384153631,15,True +catboost_monotonic_platt,Monotonic CatBoost,censored_extension,88227,59291,28936,59291,0.19643790794555666,0.6594296767595695,0.15777416870415434,0.5019042813732044,0.08396416343078279,0.3188593535191391,0.23120859555829543,0.30540717131377204,-0.08384206117076841,0.42087101244177494,0.8630583710003576,16,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,17433,17392,41,17392,0.1343146274149034,0.7123167685390068,0.1076960635588169,0.36052430692091275,0.010122429290401653,0.42463353707801366,0.3068707874164763,0.2848195241080906,0.007645312034570001,0.2436235541226129,1.1920150575524924,12,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,probability_calibration,14101,14077,24,14077,0.10478084819208638,0.6605331410940959,0.09077268402657197,0.31961691741460485,0.0015879435063299442,0.3210662821881918,0.2365472254874798,0.18308379831904797,-5.384620465985479e-05,0.0002915309769978997,0.9998499724461433,14,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,conformal_fit,49007,48953,54,48953,0.1336996711131085,0.6271779821153136,0.1132639532922482,0.3816347467265729,0.008962699269698067,0.25435596423062723,0.18509053670343995,0.19279333086603057,-0.0063652852847841945,-0.11991205451574843,0.903578076753919,12,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,policy_development,94885,94808,77,94808,0.12240528225466206,0.6196209430791464,0.10563451554252466,0.362033716632661,0.00960044699044564,0.23924188615829278,0.17034578763650804,0.17137608836842058,0.00960044699044564,-0.23879035076651506,0.9167982904589872,14,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,primary_oot,376890,364814,12076,364814,0.156167252353254,0.6330661809401453,0.12948509376427816,0.4239586771759354,0.035967619770033946,0.26613236188029066,0.18843820687722823,0.22388791917890766,-0.03551023236967947,0.15367586807582181,0.9186554972959701,14,True +woe_scorecard_platform_platt,Platform-signal WOE scorecard,censored_extension,88227,59291,28936,59291,0.19643790794555666,0.642765459981137,0.1589749376004857,0.5023943217659936,0.08054284782567817,0.285530919962274,0.20533968422406368,0.28344164657426113,-0.08051610538006741,0.6082123557025374,0.9807165471485974,16,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,17433,17392,41,17392,0.1343146274149034,0.7074458551089631,0.10791233473660404,0.36131848891852725,0.005969714516771254,0.41489171021792615,0.2997680804448779,0.2810031902325053,0.0020552543025720016,0.13007573120256186,1.0890778979546312,9,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,probability_calibration,14101,14077,24,14077,0.10478084819208638,0.6624006412756651,0.09076004261617288,0.31938291161277527,0.002062911380641378,0.32480128255133023,0.23837206362186253,0.18315040758461684,-9.852370378882203e-05,0.004075799155395636,1.0014817532071436,14,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,conformal_fit,49007,48953,54,48953,0.1336996711131085,0.6234618030470921,0.11356103980577402,0.38291990569245543,0.01288470692728091,0.24692360609418418,0.1780134958752756,0.19208230521437908,-0.006502402826390058,-0.2792594859848106,0.816508381165281,12,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,policy_development,94885,94808,77,94808,0.12240528225466206,0.605999391147712,0.10644919405009558,0.3654549463772465,0.013299656542569364,0.2119987822954239,0.1484333005653241,0.16740395975301567,0.010839815840192932,-0.5956855806746446,0.7243077870455613,14,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,primary_oot,376890,364814,12076,364814,0.156167252353254,0.6129392722275023,0.13019024412756694,0.42796595827034617,0.03176877520031669,0.22587854445500466,0.15809243291132968,0.2151918293521198,-0.029751737101973152,-0.28557359284456246,0.7098359251738742,12,True +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,censored_extension,88227,59291,28936,59291,0.19643790794555666,0.6276216510251899,0.15896082825220081,0.5042617227344806,0.07548427998908525,0.2552433020503797,0.18106334561105536,0.27420172721996416,-0.07518592179791153,0.17398111144311426,0.7800845809517352,15,True diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS3_woe_iv_psi.csv b/reports/crpto/tables/crpto_ijds_v4_tableS3_woe_iv_psi.csv new file mode 100644 index 0000000..ca80a3d --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS3_woe_iv_psi.csv @@ -0,0 +1,46 @@ +learner,feature,dtype,status,selected,n_bins,iv,js,gini,quality_score,primary_oot_psi +woe_scorecard_platform_platt,int_rate_bucket__grade,categorical,OPTIMAL,True,6,0.33756858301375126,0.04030108982523252,0.3038068292677757,0.8931322311039492,0.9628044275190203 +woe_scorecard_platform_platt,sub_grade,categorical,OPTIMAL,True,8,0.31932538983142866,0.038355779594492365,0.3008372540652854,0.7374060840552777,0.10427476362377315 +woe_scorecard_platform_platt,grade,categorical,OPTIMAL,True,5,0.29954351738889673,0.03619488925413361,0.2902113763776367,0.8524400315825463,0.09370322999517866 +woe_scorecard_platform_platt,int_rate,numerical,OPTIMAL,True,8,0.2784288707439742,0.0331030345896423,0.26219868247154365,0.4086440186665371,0.1800564225411688 +woe_scorecard_platform_platt,int_rate_bucket,categorical,OPTIMAL,True,4,0.24485815197270147,0.029096550329953043,0.23377858339237212,0.6630822192278296,0.14604333851969906 +woe_scorecard_platform_platt,fico_score,numerical,OPTIMAL,True,8,0.2135744222493438,0.025996163560484363,0.2458837062043855,0.27774633920049013,0.22729687344722427 +woe_scorecard_platform_platt,inq_last_6mths,numerical,OPTIMAL,True,4,0.17086393760483487,0.020856486793705394,0.2075187254016706,0.5105539832632361,0.39044275945118845 +woe_scorecard_platform_platt,purpose,categorical,OPTIMAL,True,7,0.08887832866960639,0.010926477290665853,0.14779348033365802,0.01768723653834336,0.18798514882172745 +woe_scorecard_platform_platt,rev_utilization,numerical,OPTIMAL,True,7,0.0739026629287923,0.009186114763468882,0.15097843927914056,0.019649087815197794,0.08694369969476334 +woe_scorecard_platform_platt,delinq_recency,numerical,OPTIMAL,True,4,0.04928782736183494,0.006080811618721303,0.09877612394585844,0.09134608884939553,0.1308635109765922 +woe_scorecard_platform_platt,installment_burden,numerical,OPTIMAL,True,7,0.04529649773543025,0.005627065898995917,0.11155248266308815,0.0005292328749668587,0.03511908413614586 +woe_scorecard_platform_platt,loan_to_income,numerical,OPTIMAL,True,8,0.040923308172628466,0.00508331899774769,0.10574445268725341,0.00034457932701394405,0.020767412564742786 +woe_scorecard_platform_platt,annual_inc,numerical,OPTIMAL,True,6,0.031788706777070526,0.00395203452222934,0.08954045844718961,0.0017127167326026935,0.05671526670323674 +woe_scorecard_platform_platt,revol_bal_to_income,numerical,OPTIMAL,True,8,0.031665831471954065,0.003936070094098116,0.0918789420590792,0.03651263795086532,0.13576298727737582 +woe_scorecard_platform_platt,pub_rec,numerical,OPTIMAL,True,2,0.028826877209471597,0.0035532475168816653,0.04436931603538807,0.018336356432410134,0.19628053307281587 +woe_scorecard_platform_platt,installment,numerical,OPTIMAL,True,7,0.027842133869223903,0.003457110620992919,0.08013790035783641,0.030241867435699874,0.14218212629612492 +woe_scorecard_platform_platt,loan_amnt,numerical,OPTIMAL,True,7,0.021515016074673814,0.0026803152211405446,0.07568700436234888,0.02581935064570112,0.11726402284277238 +woe_scorecard_platform_platt,total_acc,numerical,OPTIMAL,True,8,0.021292046332186305,0.0026531121642405834,0.07632569201688133,0.0055670026285511804,0.06274658172307467 +woe_scorecard_platform_platt,revol_bal,numerical,OPTIMAL,True,8,0.020440401565069988,0.002546117793200098,0.0695209436289913,0.00043669727612496366,0.12149533741863884 +woe_scorecard_platform_platt,dti,numerical,OPTIMAL,True,7,0.017206002175405456,0.002146397385909767,0.07211526444062544,0.0019053713853871563,0.5522627527176915 +woe_scorecard_platform_platt,credit_age_years,numerical,OPTIMAL,True,7,0.01657203613043824,0.0020608484885398207,0.05614851470893867,8.304031801534555e-05,0.23126184110280007 +woe_scorecard_platform_platt,open_acc,numerical,OPTIMAL,True,7,0.015087965269081374,0.0018804303529062565,0.06248394117007472,0.0037562084018691612,0.19045239287788585 +woe_scorecard_platform_platt,emp_length_num,numerical,OPTIMAL,True,6,0.011404805439449077,0.0014083528398145368,0.04489247285709963,0.0024661649530662495,0.10398963118282264 +woe_scorecard_platform_platt,verification_status,categorical,OPTIMAL,True,3,0.010125324332154644,0.0012600562698157894,0.02837748162746201,0.00232351305893576,0.6081334177376476 +woe_scorecard_platform_platt,delinq_severity,numerical,OPTIMAL,True,2,0.002230353442878537,0.00027862456254427494,0.01528001255647038,0.0023713438765294585,0.14721325645027647 +woe_scorecard_platform_platt,home_ownership,categorical,OPTIMAL,True,3,0.0012007260080548763,0.00015002666128414055,0.013938688988259895,0.000958045316845026,0.052311124583344644 +woe_scorecard_borrower_platt,fico_score,numerical,OPTIMAL,True,8,0.2135744222493438,0.025996163560484363,0.2458837062043855,0.27774633920049013,0.22729687344722427 +woe_scorecard_borrower_platt,inq_last_6mths,numerical,OPTIMAL,True,4,0.17086393760483487,0.020856486793705394,0.2075187254016706,0.5105539832632361,0.39044275945118845 +woe_scorecard_borrower_platt,purpose,categorical,OPTIMAL,True,7,0.08887832866960639,0.010926477290665853,0.14779348033365802,0.01768723653834336,0.18798514882172745 +woe_scorecard_borrower_platt,rev_utilization,numerical,OPTIMAL,True,7,0.0739026629287923,0.009186114763468882,0.15097843927914056,0.019649087815197794,0.08694369969476334 +woe_scorecard_borrower_platt,delinq_recency,numerical,OPTIMAL,True,4,0.04928782736183494,0.006080811618721303,0.09877612394585844,0.09134608884939553,0.1308635109765922 +woe_scorecard_borrower_platt,loan_to_income,numerical,OPTIMAL,True,7,0.04092180339356583,0.005083132080619217,0.10566871286374879,0.011306708816571379,0.020570876717486017 +woe_scorecard_borrower_platt,annual_inc,numerical,OPTIMAL,True,6,0.031788706777070526,0.00395203452222934,0.08954045844718961,0.0017127167326026935,0.05671526670323674 +woe_scorecard_borrower_platt,revol_bal_to_income,numerical,OPTIMAL,True,8,0.031665831471954065,0.003936070094098116,0.0918789420590792,0.03651263795086532,0.13576298727737582 +woe_scorecard_borrower_platt,pub_rec,numerical,OPTIMAL,True,2,0.028826877209471597,0.0035532475168816653,0.04436931603538807,0.018336356432410134,0.19628053307281587 +woe_scorecard_borrower_platt,loan_amnt,numerical,OPTIMAL,True,7,0.021515016074673814,0.0026803152211405446,0.07568700436234888,0.02581935064570112,0.11726402284277238 +woe_scorecard_borrower_platt,total_acc,numerical,OPTIMAL,True,8,0.021292046332186305,0.0026531121642405834,0.07632569201688133,0.0055670026285511804,0.06274658172307467 +woe_scorecard_borrower_platt,revol_bal,numerical,OPTIMAL,True,8,0.020440401565069988,0.002546117793200098,0.0695209436289913,0.00043669727612496366,0.12149533741863884 +woe_scorecard_borrower_platt,dti,numerical,OPTIMAL,True,7,0.017206002175405456,0.002146397385909767,0.07211526444062544,0.0019053713853871563,0.5522627527176915 +woe_scorecard_borrower_platt,credit_age_years,numerical,OPTIMAL,True,7,0.01657203613043824,0.0020608484885398207,0.05614851470893867,8.304031801534555e-05,0.23126184110280007 +woe_scorecard_borrower_platt,open_acc,numerical,OPTIMAL,True,7,0.015087965269081374,0.0018804303529062565,0.06248394117007472,0.0037562084018691612,0.19045239287788585 +woe_scorecard_borrower_platt,emp_length_num,numerical,OPTIMAL,True,6,0.011404805439449077,0.0014083528398145368,0.04489247285709963,0.0024661649530662495,0.10398963118282264 +woe_scorecard_borrower_platt,verification_status,categorical,OPTIMAL,True,3,0.010125324332154644,0.0012600562698157894,0.02837748162746201,0.00232351305893576,0.6081334177376476 +woe_scorecard_borrower_platt,delinq_severity,numerical,OPTIMAL,True,2,0.002230353442878537,0.00027862456254427494,0.01528001255647038,0.0023713438765294585,0.14721325645027647 +woe_scorecard_borrower_platt,home_ownership,categorical,OPTIMAL,True,3,0.0012007260080548763,0.00015002666128414055,0.013938688988259895,0.000958045316845026,0.052311124583344644 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS4_score_psi.csv b/reports/crpto/tables/crpto_ijds_v4_tableS4_score_psi.csv new file mode 100644 index 0000000..bb15d08 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS4_score_psi.csv @@ -0,0 +1,26 @@ +learner,learner_label,reference_role,comparison_role,reference_rows,comparison_rows,psi,bin_edges +catboost_platt,CatBoost,pd_development,probability_calibration,17433,14101,0.17407068828677333,"[-Infinity, 0.04261672704685651, 0.06355959079594822, 0.08339555766062527, 0.10286324786698922, 0.12166132268398075, 0.1414130325328107, 0.16341704213258806, 0.1927072483379503, 0.23853531204708556, Infinity]" +catboost_platt,CatBoost,pd_development,conformal_fit,17433,49007,0.09687542537370272,"[-Infinity, 0.04261672704685651, 0.06355959079594822, 0.08339555766062527, 0.10286324786698922, 0.12166132268398075, 0.1414130325328107, 0.16341704213258806, 0.1927072483379503, 0.23853531204708556, Infinity]" +catboost_platt,CatBoost,pd_development,policy_development,17433,94885,0.0804944200230353,"[-Infinity, 0.04261672704685651, 0.06355959079594822, 0.08339555766062527, 0.10286324786698922, 0.12166132268398075, 0.1414130325328107, 0.16341704213258806, 0.1927072483379503, 0.23853531204708556, Infinity]" +catboost_platt,CatBoost,pd_development,primary_oot,17433,376890,0.1397574884533394,"[-Infinity, 0.04261672704685651, 0.06355959079594822, 0.08339555766062527, 0.10286324786698922, 0.12166132268398075, 0.1414130325328107, 0.16341704213258806, 0.1927072483379503, 0.23853531204708556, Infinity]" +catboost_platt,CatBoost,pd_development,censored_extension,17433,88227,0.15508902744066488,"[-Infinity, 0.04261672704685651, 0.06355959079594822, 0.08339555766062527, 0.10286324786698922, 0.12166132268398075, 0.1414130325328107, 0.16341704213258806, 0.1927072483379503, 0.23853531204708556, Infinity]" +numeric_logistic_platt,Numeric logistic,pd_development,probability_calibration,17433,14101,0.13797815521803242,"[-Infinity, 0.05277788689082225, 0.06876201433363191, 0.08443091210141554, 0.09993699103367058, 0.11518839289701487, 0.1321813034220992, 0.1516382266905663, 0.1772320826894889, 0.22129847971211278, Infinity]" +numeric_logistic_platt,Numeric logistic,pd_development,conformal_fit,17433,49007,0.04100272203228261,"[-Infinity, 0.05277788689082225, 0.06876201433363191, 0.08443091210141554, 0.09993699103367058, 0.11518839289701487, 0.1321813034220992, 0.1516382266905663, 0.1772320826894889, 0.22129847971211278, Infinity]" +numeric_logistic_platt,Numeric logistic,pd_development,policy_development,17433,94885,0.1034365170682255,"[-Infinity, 0.05277788689082225, 0.06876201433363191, 0.08443091210141554, 0.09993699103367058, 0.11518839289701487, 0.1321813034220992, 0.1516382266905663, 0.1772320826894889, 0.22129847971211278, Infinity]" +numeric_logistic_platt,Numeric logistic,pd_development,primary_oot,17433,376890,0.005167903809465062,"[-Infinity, 0.05277788689082225, 0.06876201433363191, 0.08443091210141554, 0.09993699103367058, 0.11518839289701487, 0.1321813034220992, 0.1516382266905663, 0.1772320826894889, 0.22129847971211278, Infinity]" +numeric_logistic_platt,Numeric logistic,pd_development,censored_extension,17433,88227,0.005494130430828579,"[-Infinity, 0.05277788689082225, 0.06876201433363191, 0.08443091210141554, 0.09993699103367058, 0.11518839289701487, 0.1321813034220992, 0.1516382266905663, 0.1772320826894889, 0.22129847971211278, Infinity]" +catboost_monotonic_platt,Monotonic CatBoost,pd_development,probability_calibration,17433,14101,0.18254271016740511,"[-Infinity, 0.0437133457488556, 0.06477749039608523, 0.08479454311252721, 0.10452104540174735, 0.12371933386843262, 0.1428350095364703, 0.16474240744893798, 0.1936480061992157, 0.2416254435522038, Infinity]" +catboost_monotonic_platt,Monotonic CatBoost,pd_development,conformal_fit,17433,49007,0.07649837663049866,"[-Infinity, 0.0437133457488556, 0.06477749039608523, 0.08479454311252721, 0.10452104540174735, 0.12371933386843262, 0.1428350095364703, 0.16474240744893798, 0.1936480061992157, 0.2416254435522038, Infinity]" +catboost_monotonic_platt,Monotonic CatBoost,pd_development,policy_development,17433,94885,0.06879565388007686,"[-Infinity, 0.0437133457488556, 0.06477749039608523, 0.08479454311252721, 0.10452104540174735, 0.12371933386843262, 0.1428350095364703, 0.16474240744893798, 0.1936480061992157, 0.2416254435522038, Infinity]" +catboost_monotonic_platt,Monotonic CatBoost,pd_development,primary_oot,17433,376890,0.09370550856801749,"[-Infinity, 0.0437133457488556, 0.06477749039608523, 0.08479454311252721, 0.10452104540174735, 0.12371933386843262, 0.1428350095364703, 0.16474240744893798, 0.1936480061992157, 0.2416254435522038, Infinity]" +catboost_monotonic_platt,Monotonic CatBoost,pd_development,censored_extension,17433,88227,0.11971513945324881,"[-Infinity, 0.0437133457488556, 0.06477749039608523, 0.08479454311252721, 0.10452104540174735, 0.12371933386843262, 0.1428350095364703, 0.16474240744893798, 0.1936480061992157, 0.2416254435522038, Infinity]" +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,probability_calibration,17433,14101,0.28947005890143485,"[-Infinity, 0.051639511857074744, 0.07293806621304565, 0.0911892051364249, 0.1081442936583909, 0.1254537296840375, 0.14460263624638384, 0.16775053138646007, 0.19835288305302934, 0.24968579142679675, Infinity]" +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,conformal_fit,17433,49007,0.14720859274640988,"[-Infinity, 0.051639511857074744, 0.07293806621304565, 0.0911892051364249, 0.1081442936583909, 0.1254537296840375, 0.14460263624638384, 0.16775053138646007, 0.19835288305302934, 0.24968579142679675, Infinity]" +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,policy_development,17433,94885,0.18227568586918236,"[-Infinity, 0.051639511857074744, 0.07293806621304565, 0.0911892051364249, 0.1081442936583909, 0.1254537296840375, 0.14460263624638384, 0.16775053138646007, 0.19835288305302934, 0.24968579142679675, Infinity]" +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,primary_oot,17433,376890,0.14886689865662606,"[-Infinity, 0.051639511857074744, 0.07293806621304565, 0.0911892051364249, 0.1081442936583909, 0.1254537296840375, 0.14460263624638384, 0.16775053138646007, 0.19835288305302934, 0.24968579142679675, Infinity]" +woe_scorecard_platform_platt,Platform-signal WOE scorecard,pd_development,censored_extension,17433,88227,0.19282930306197582,"[-Infinity, 0.051639511857074744, 0.07293806621304565, 0.0911892051364249, 0.1081442936583909, 0.1254537296840375, 0.14460263624638384, 0.16775053138646007, 0.19835288305302934, 0.24968579142679675, Infinity]" +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,probability_calibration,17433,14101,0.20117450389076702,"[-Infinity, 0.04861776304603217, 0.06662708595896956, 0.08319285194171437, 0.09888242771470508, 0.11649125751644453, 0.13587450786502708, 0.15996645636981532, 0.19153805694933232, 0.24929481366337367, Infinity]" +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,conformal_fit,17433,49007,0.12021776634266133,"[-Infinity, 0.04861776304603217, 0.06662708595896956, 0.08319285194171437, 0.09888242771470508, 0.11649125751644453, 0.13587450786502708, 0.15996645636981532, 0.19153805694933232, 0.24929481366337367, Infinity]" +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,policy_development,17433,94885,0.1479539663610525,"[-Infinity, 0.04861776304603217, 0.06662708595896956, 0.08319285194171437, 0.09888242771470508, 0.11649125751644453, 0.13587450786502708, 0.15996645636981532, 0.19153805694933232, 0.24929481366337367, Infinity]" +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,primary_oot,17433,376890,0.07233216453444681,"[-Infinity, 0.04861776304603217, 0.06662708595896956, 0.08319285194171437, 0.09888242771470508, 0.11649125751644453, 0.13587450786502708, 0.15996645636981532, 0.19153805694933232, 0.24929481366337367, Infinity]" +woe_scorecard_borrower_platt,Pricing-excluded application WOE scorecard,pd_development,censored_extension,17433,88227,0.07618880966587974,"[-Infinity, 0.04861776304603217, 0.06662708595896956, 0.08319285194171437, 0.09888242771470508, 0.11649125751644453, 0.13587450786502708, 0.15996645636981532, 0.19153805694933232, 0.24929481366337367, Infinity]" diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS5_label_lag_sensitivity.csv b/reports/crpto/tables/crpto_ijds_v4_tableS5_label_lag_sensitivity.csv new file mode 100644 index 0000000..a5be8bf --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS5_label_lag_sensitivity.csv @@ -0,0 +1,41 @@ +charged_off_lag_months,window_id,window_rows,retained_rows,window_retention,minimum_monthly_retention,passes_locked_retention,phase_stratum,phase_rows,phase_prevalence,phase_residual_quantile,phase_interval_width_mean,phase_set_both_share +0,w01_2012m01_m06,14967,14949,0.9987973541791942,0.997116493656286,True,2,3249,0.1166512773160972,0.8919735883395368,0.9881387583482552,0.1372730070791012 +0,w02_2012m02_m07,16690,16675,0.9991012582384662,0.997116493656286,True,2,3621,0.1165423916045291,0.8920116585417792,0.9883652306769672,0.1433305716652858 +0,w03_2012m03_m08,19068,19052,0.999160897839312,0.997116493656286,True,2,4018,0.1139870582379293,0.8916085888119072,0.9880289633104608,0.1269288203086112 +0,w04_2012m04_m09,21791,21773,0.9991739709054196,0.997116493656286,True,2,4511,0.1099534471292396,0.890281077239412,0.9867825422901556,0.0767013965861228 +0,w05_2012m05_m10,24313,24282,0.99872496195451,0.997116493656286,True,2,5001,0.1065786842631473,0.8894148014498154,0.9859853433399698,0.0497900419916016 +0,w06_2012m06_m11,26747,26709,0.9985792799192432,0.997116493656286,True,2,5505,0.1057220708446866,0.8893536841967226,0.9859485251379692,0.0472297910990009 +0,w07_2012m07_m12,28503,28453,0.9982457986878576,0.997116493656286,True,2,5939,0.1032160296346186,0.8886909233974287,0.98515359958887,0.0220575854520963 +0,w08_2012m08_2013m01,30259,30198,0.9979840708549522,0.997116493656286,True,2,6251,0.0990241561350184,0.1118700059392816,0.2082540244531064,0.0 +3,w01_2012m01_m06,14967,14949,0.9987973541791942,0.9966694421315572,True,2,3249,0.1166512773160972,0.8919735883395368,0.9881387583482552,0.1372730070791012 +3,w02_2012m02_m07,16690,16675,0.9991012582384662,0.9966694421315572,True,2,3621,0.1165423916045291,0.8920116585417792,0.9883652306769672,0.1433305716652858 +3,w03_2012m03_m08,19068,19052,0.999160897839312,0.9966694421315572,True,2,4018,0.1139870582379293,0.8916085888119072,0.9880289633104608,0.1269288203086112 +3,w04_2012m04_m09,21791,21773,0.9991739709054196,0.9966694421315572,True,2,4511,0.1099534471292396,0.890281077239412,0.9867825422901556,0.0767013965861228 +3,w05_2012m05_m10,24313,24282,0.99872496195451,0.9966694421315572,True,2,5001,0.1065786842631473,0.8894148014498154,0.9859853433399698,0.0497900419916016 +3,w06_2012m06_m11,26747,26709,0.9985792799192432,0.9966694421315572,True,2,5505,0.1057220708446866,0.8893536841967226,0.9859485251379692,0.0472297910990009 +3,w07_2012m07_m12,28503,28449,0.998105462582886,0.9966694421315572,True,2,5937,0.102913929594071,0.8886749896176616,0.9851387217798036,0.0212228398180899 +3,w08_2012m08_2013m01,30259,30191,0.9977527347235532,0.9966694421315572,True,2,6249,0.0987357977276364,0.1118616046782793,0.2082462684749756,0.0 +6,w01_2012m01_m06,14967,14948,0.9987305405224828,0.9929564746252484,True,2,3249,0.1166512773160972,0.8919735883395368,0.9881387583482552,0.1372730070791012 +6,w02_2012m02_m07,16690,16674,0.9990413421210306,0.9929564746252484,True,2,3621,0.1165423916045291,0.8920116585417792,0.9883652306769672,0.1433305716652858 +6,w03_2012m03_m08,19068,19049,0.999003566184183,0.9929564746252484,True,2,4018,0.1139870582379293,0.8916085888119072,0.9880289633104608,0.1269288203086112 +6,w04_2012m04_m09,21791,21767,0.9988986278738928,0.9929564746252484,True,2,4511,0.1099534471292396,0.890281077239412,0.9867825422901556,0.0767013965861228 +6,w05_2012m05_m10,24313,24270,0.9982313988401268,0.9929564746252484,True,2,5001,0.1065786842631473,0.8894148014498154,0.9859853433399698,0.0497900419916016 +6,w06_2012m06_m11,26747,26685,0.9976819830261338,0.9929564746252484,True,2,5502,0.1052344601962922,0.8892521057704302,0.9858494930099646,0.044711014176663 +6,w07_2012m07_m12,28503,28411,0.9967722695856576,0.9929564746252484,True,2,5929,0.1017034913138809,0.8884345991499274,0.9848983200473448,0.0129870129870129 +6,w08_2012m08_2013m01,30259,30129,0.9957037575597344,0.9929564746252484,True,2,6238,0.0971465213209362,0.1118010883671265,0.2081817207788981,0.0 +8,w01_2012m01_m06,14967,14945,0.9985300995523484,0.9868159653241828,False,2,3249,0.1166512773160972,0.8919735883395368,0.9881387583482552,0.1372730070791012 +8,w02_2012m02_m07,16690,16669,0.9987417615338526,0.9868159653241828,False,2,3620,0.1162983425414364,0.8919735883395368,0.9883343129081892,0.1414364640883978 +8,w03_2012m03_m08,19068,19042,0.998636458988882,0.9868159653241828,False,2,4016,0.1135458167330677,0.8914417663875044,0.9878822284936604,0.1190239043824701 +8,w04_2012m04_m09,21791,21758,0.9984856133266028,0.9868159653241828,False,2,4509,0.109558660456864,0.8902427153564156,0.986746796229896,0.0756265247283211 +8,w05_2012m05_m10,24313,24245,0.9972031423518284,0.9868159653241828,False,2,4996,0.1056845476381104,0.8893093428244802,0.985881593664338,0.0464371497197758 +8,w06_2012m06_m11,26747,26640,0.9959995513515536,0.9868159653241828,False,2,5494,0.1039315617036767,0.88903966354385,0.9856492442766288,0.0371314160902803 +8,w07_2012m07_m12,28503,28349,0.9945970599586008,0.9868159653241828,False,2,5916,0.0997295469912102,0.1118832525211423,0.2083465264137485,0.0 +8,w08_2012m08_2013m01,30259,30035,0.9925972437952344,0.9868159653241828,False,2,6217,0.0940968312691008,0.1117123594604784,0.208087780959762,0.0 +12,w01_2012m01_m06,14967,14918,0.9967261308211398,0.9745349467220517,False,2,3244,0.1152897657213316,0.8918727660654336,0.988050716335636,0.1325524044389642 +12,w02_2012m02_m07,16690,16612,0.995326542840024,0.9745349467220517,False,2,3611,0.1140958183328717,0.8916414588574106,0.988041723845512,0.1282193298255331 +12,w03_2012m03_m08,19068,18949,0.9937591776798824,0.9745349467220517,False,2,4004,0.1108891108891109,0.8907949705797163,0.987300128440098,0.0959040959040959 +12,w04_2012m04_m09,21791,21621,0.9921986141067412,0.9745349467220517,False,2,4493,0.1063877142221233,0.8894829334991204,0.98602897814676,0.0505230358335188 +12,w05_2012m05_m10,24313,24058,0.989511783819356,0.9745349467220517,False,2,4976,0.1020900321543408,0.8883803108986407,0.9849755772001694,0.0114549839228295 +12,w06_2012m06_m11,26747,26396,0.9868770329382734,0.9745349467220517,False,2,5462,0.0986818015378982,0.1118616046782793,0.2084807323519395,0.0 +12,w07_2012m07_m12,28503,28052,0.984177104164474,0.9745349467220517,False,2,5873,0.0931380895624042,0.1116902104787908,0.2081424372028483,0.0 +12,w08_2012m08_2013m01,30259,29700,0.9815261575068576,0.9745349467220517,False,2,6170,0.0871961102106969,0.1115043934501935,0.2078719347850636,0.0 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv b/reports/crpto/tables/crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv new file mode 100644 index 0000000..2e89b5f --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv @@ -0,0 +1,6 @@ +charged_off_lag_months,primary_candidates,primary_resolved,primary_unresolved,primary_defaults,coverage_cells,coverage_upper_below_0_90_cells,coverage_upper_at_or_above_0_90_cells,coverage_upper_max,coverage_upper_max_learner,coverage_upper_max_window,two_ruler_payoff_gamma_1_lower_cells,two_ruler_payoff_gamma_1_higher_cells,two_ruler_payoff_crosses_zero_cells,two_ruler_payoff_exact_zero_cells,two_ruler_default_gamma_1_lower_cells,two_ruler_default_gamma_1_higher_cells,two_ruler_default_crosses_zero_cells,two_ruler_default_exact_zero_cells,two_ruler_miscoverage_gamma_1_lower_cells,two_ruler_miscoverage_gamma_1_higher_cells,two_ruler_miscoverage_crosses_zero_cells,two_ruler_miscoverage_exact_zero_cells,development_admissible_exact_frontier_standardized_payoff_guardrail_lower_cells,development_admissible_exact_frontier_standardized_payoff_guardrail_higher_cells,development_admissible_exact_frontier_standardized_payoff_crosses_zero_cells,broad_stress_exact_frontier_standardized_payoff_guardrail_lower_cells,broad_stress_exact_frontier_standardized_payoff_guardrail_higher_cells,broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells,development_admissible_exact_frontier_terminal_default_guardrail_lower_cells,development_admissible_exact_frontier_terminal_default_guardrail_higher_cells,development_admissible_exact_frontier_terminal_default_crosses_zero_cells,broad_stress_exact_frontier_terminal_default_guardrail_lower_cells,broad_stress_exact_frontier_terminal_default_guardrail_higher_cells,broad_stress_exact_frontier_terminal_default_crosses_zero_cells,development_admissible_exact_frontier_funded_miscoverage_guardrail_lower_cells,development_admissible_exact_frontier_funded_miscoverage_guardrail_higher_cells,development_admissible_exact_frontier_funded_miscoverage_crosses_zero_cells,broad_stress_exact_frontier_funded_miscoverage_guardrail_lower_cells,broad_stress_exact_frontier_funded_miscoverage_guardrail_higher_cells,broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells +0,376890,364861,12029,57019,40,40,0,0.8976412215765873,woe_scorecard_borrower_platt,w02_2012m02_m07,32,0,16,0,0,33,15,0,0,40,8,0,6,0,66,0,0,72,0,0,72,0,0,72,0,27,45,0,0,72 +3,376890,364861,12029,57019,40,40,0,0.8976412215765873,woe_scorecard_borrower_platt,w02_2012m02_m07,32,0,16,0,0,33,15,0,0,40,8,0,6,0,66,0,0,72,0,0,72,0,0,72,0,27,45,0,0,72 +6,376890,364814,12076,56972,40,40,0,0.8977261269866539,woe_scorecard_borrower_platt,w02_2012m02_m07,32,0,16,0,0,33,15,0,0,40,8,0,6,0,66,0,0,72,0,0,72,0,0,72,0,27,45,0,0,72 +8,376890,364570,12320,56728,40,40,0,0.8981506540369869,woe_scorecard_borrower_platt,w02_2012m02_m07,32,0,16,0,0,33,15,0,0,40,8,0,6,0,66,0,0,72,0,0,72,0,0,72,0,27,45,0,0,72 +12,376890,363288,13602,55446,40,39,1,0.9004112605800101,woe_scorecard_borrower_platt,w02_2012m02_m07,31,0,17,0,0,32,16,0,0,40,8,0,0,0,72,0,0,72,0,0,72,0,0,72,0,26,46,0,0,72 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv b/reports/crpto/tables/crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv new file mode 100644 index 0000000..a4d740f --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv @@ -0,0 +1,37 @@ +scenario_id,budget,purpose_cap,lgd,is_baseline,standardized_payoff_gamma_1_lower_cells,standardized_payoff_gamma_1_higher_cells,standardized_payoff_crosses_zero_cells,standardized_payoff_exact_zero_cells,funded_default_gamma_1_higher_cells,funded_default_gamma_1_lower_cells,funded_default_crosses_zero_cells,funded_default_exact_zero_cells,funded_binary_miscoverage_gamma_1_higher_cells,funded_binary_miscoverage_gamma_1_lower_cells,funded_binary_miscoverage_crosses_zero_cells,funded_binary_miscoverage_exact_zero_cells,activity_purpose_cap_binding_share,activity_frontier_constraint_binding_share,activity_portfolios,activity_maximum_loan_weight +b0500k_p020_l025,500000.0,0.2,0.25,False,16,1,31,0,39,1,8,0,39,0,9,0,1.0,1.0,1440,0.08000000000000448 +b0500k_p020_l045,500000.0,0.2,0.45,False,32,8,8,0,32,1,15,0,39,1,8,0,1.0,1.0,1440,0.08000000000001126 +b0500k_p020_l065,500000.0,0.2,0.65,False,31,8,1,8,32,8,0,8,32,3,5,8,1.0,1.0,1440,0.0800000000000165 +b0500k_p025_l025,500000.0,0.25,0.25,False,16,8,24,0,24,8,16,0,39,0,9,0,1.0,1.0,1440,0.08000000000000282 +b0500k_p025_l045,500000.0,0.25,0.45,False,30,2,8,8,31,1,8,8,23,1,16,8,1.0,1.0,1440,0.08000000000000006 +b0500k_p025_l065,500000.0,0.25,0.65,False,32,0,8,8,32,0,8,8,32,0,8,8,1.0,1.0,1440,0.08000000000000108 +b0500k_p030_l025,500000.0,0.3,0.25,False,17,8,23,0,40,1,7,0,40,6,2,0,1.0,1.0,1440,0.08000000000001112 +b0500k_p030_l045,500000.0,0.3,0.45,False,24,1,15,8,31,1,8,8,24,8,8,8,1.0,1.0,1440,0.08000000000001121 +b0500k_p030_l065,500000.0,0.3,0.65,False,31,0,9,8,31,0,9,8,32,0,8,8,1.0,1.0,1440,0.08000000000001108 +b0500k_p100_l025,500000.0,1.0,0.25,False,15,8,25,0,25,0,23,0,24,8,16,0,0.0,1.0,1440,0.08000000000001209 +b0500k_p100_l045,500000.0,1.0,0.45,False,30,1,9,8,31,1,8,8,30,0,10,8,0.0,1.0,1440,0.08000000000000625 +b0500k_p100_l065,500000.0,1.0,0.65,False,24,8,8,8,24,8,8,8,32,0,8,8,0.0,1.0,1440,0.08000000000000271 +b1000k_p020_l025,1000000.0,0.2,0.25,False,1,17,30,0,32,15,1,0,32,16,0,0,1.0,1.0,1440,0.04000000000000256 +b1000k_p020_l045,1000000.0,0.2,0.45,False,11,8,29,0,25,8,15,0,34,8,6,0,1.0,1.0,1440,0.0400000000000051 +b1000k_p020_l065,1000000.0,0.2,0.65,False,33,0,15,0,33,0,15,0,40,0,8,0,1.0,1.0,1440,0.040000000000000084 +b1000k_p025_l025,1000000.0,0.25,0.25,False,14,15,19,0,24,8,16,0,32,15,1,0,1.0,1.0,1440,0.04000000000000237 +b1000k_p025_l045,1000000.0,0.25,0.45,True,32,0,16,0,33,0,15,0,40,0,8,0,1.0,1.0,1440,0.04000000000000284 +b1000k_p025_l065,1000000.0,0.25,0.65,False,25,0,23,0,38,0,10,0,40,0,8,0,1.0,1.0,1440,0.04000000000000424 +b1000k_p030_l025,1000000.0,0.3,0.25,False,28,9,11,0,32,8,8,0,32,14,2,0,1.0,1.0,1440,0.0400000000000025 +b1000k_p030_l045,1000000.0,0.3,0.45,False,18,8,22,0,32,8,8,0,33,8,7,0,1.0,1.0,1440,0.040000000000002485 +b1000k_p030_l065,1000000.0,0.3,0.65,False,31,8,9,0,26,8,14,0,32,4,12,0,1.0,1.0,1440,0.04000000000000229 +b1000k_p100_l025,1000000.0,1.0,0.25,False,18,8,22,0,24,8,16,0,40,8,0,0,0.0,1.0,1440,0.04000000000000118 +b1000k_p100_l045,1000000.0,1.0,0.45,False,16,16,16,0,31,16,1,0,24,10,14,0,0.0,1.0,1440,0.04000000000000138 +b1000k_p100_l065,1000000.0,1.0,0.65,False,31,8,9,0,24,8,16,0,24,3,21,0,0.0,1.0,1440,0.0400000000000017 +b2000k_p020_l025,2000000.0,0.2,0.25,False,0,0,48,0,17,0,31,0,26,1,21,0,1.0,1.0,1440,0.020000000000003775 +b2000k_p020_l045,2000000.0,0.2,0.45,False,2,0,46,0,22,0,26,0,33,0,15,0,1.0,1.0,1440,0.02000000000000912 +b2000k_p020_l065,2000000.0,0.2,0.65,False,24,8,16,0,24,0,24,0,32,0,16,0,1.0,1.0,1440,0.02000000000005261 +b2000k_p025_l025,2000000.0,0.25,0.25,False,0,9,39,0,32,0,16,0,21,1,26,0,1.0,1.0,1440,0.020000000000063124 +b2000k_p025_l045,2000000.0,0.25,0.45,False,9,16,23,0,17,0,31,0,32,5,11,0,1.0,1.0,1440,0.02000000000009092 +b2000k_p025_l065,2000000.0,0.25,0.65,False,20,0,28,0,32,0,16,0,40,0,8,0,1.0,1.0,1440,0.02000000000003237 +b2000k_p030_l025,2000000.0,0.3,0.25,False,16,16,16,0,33,8,7,0,32,3,13,0,1.0,1.0,1440,0.020000000000002446 +b2000k_p030_l045,2000000.0,0.3,0.45,False,9,8,31,0,31,0,17,0,37,0,11,0,1.0,1.0,1440,0.020000000000004733 +b2000k_p030_l065,2000000.0,0.3,0.65,False,31,0,17,0,33,0,15,0,48,0,0,0,1.0,1.0,1440,0.02000000000000279 +b2000k_p100_l025,2000000.0,1.0,0.25,False,8,8,32,0,32,8,8,0,35,8,5,0,0.0,1.0,1440,0.020000000000001707 +b2000k_p100_l045,2000000.0,1.0,0.45,False,32,0,16,0,40,0,8,0,40,0,8,0,0.0,1.0,1440,0.02000000000000391 +b2000k_p100_l065,2000000.0,1.0,0.65,False,33,8,7,0,33,0,15,0,40,0,8,0,0.0,1.0,1440,0.020000000000002353 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS8_endpoint_resolution.csv b/reports/crpto/tables/crpto_ijds_v4_tableS8_endpoint_resolution.csv new file mode 100644 index 0000000..15eb671 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS8_endpoint_resolution.csv @@ -0,0 +1,6 @@ +role,snapshot_resolution,candidate_rows,resolved_rows,unresolved_rows +primary_oot,charged_off_by_reconstructed_cutoff,56972,56972,0 +primary_oot,fully_paid_by_reconstructed_cutoff,307842,307842,0 +primary_oot,nonterminal_or_unresolved_status,11551,0,11551 +primary_oot,terminal_after_reconstructed_cutoff,47,0,47 +primary_oot,terminal_availability_date_missing,478,0,478 diff --git a/reports/crpto/tables/crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv b/reports/crpto/tables/crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv new file mode 100644 index 0000000..b63bd42 --- /dev/null +++ b/reports/crpto/tables/crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv @@ -0,0 +1,4 @@ +learner,coverage_lower_min,coverage_upper_max,windows_with_upper_below_nominal,all_windows_upper_below_nominal,roc_auc,brier,log_loss,ece_10,calibration_slope +catboost_missing_indicators_platt,0.8455066465016318,0.8843322985486481,8,True,0.6436322403148622,0.12965175159650838,0.42810867198186225,0.04742800450704818,0.7913021921642017 +catboost_native_missing_platt,0.8452837698002069,0.8800366154580912,8,True,0.6442620078596174,0.12953346396196225,0.4267357005890355,0.0457674296671821,0.8184948803074936 +catboost_platt,0.8424845445620738,0.8825970442304121,8,True,0.6406052990079556,0.12987780627268664,0.42846570011698926,0.047215103762021676,0.7954274631320678 diff --git a/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv b/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv deleted file mode 100644 index c0362a9..0000000 --- a/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv +++ /dev/null @@ -1,4 +0,0 @@ -method,mean_regret,regret_delta_vs_two_stage_pct,coverage_90_range,exact_funded_set_bound,robust_region_pass,auditability_evidence_count,paper_role,source -Two-stage baseline,0.42589625543096554,0.0,not_applicable,False,False,0,Regret baseline for the decision-focused comparison.,models/spo_real_training_status.json -SPO+,0.21683697549265343,49.08690256662728,not_applicable,False,False,0,Regret-efficient comparator; temporal improvement range 41.73--57.56%.,models/spo_real_training_status.json -CRPTO robust,0.947429495839504,-122.45546509461502,0.9207--0.9858,True,True,3,"Auditable robust champion: coverage, exact bound and 45/45 region.",models/spo_real_training_status.json; data/processed/crpto_vs_spo_stability.json; models/final_project_promotion.json diff --git a/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.tex b/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.tex deleted file mode 100644 index 26e2ac8..0000000 --- a/reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.tex +++ /dev/null @@ -1,9 +0,0 @@ -\begin{tabular}{lrrlrrrll} -\toprule -method & mean\_regret & regret\_delta\_vs\_two\_stage\_pct & coverage\_90\_range & exact\_funded\_set\_bound & robust\_region\_pass & auditability\_evidence\_count & paper\_role & source \\ -\midrule -Two-stage baseline & 0.425896 & 0.000000 & not\_applicable & False & False & 0 & Regret baseline for the decision-focused comparison. & models/spo\_real\_training\_status.json \\ -SPO+ & 0.216837 & 49.086903 & not\_applicable & False & False & 0 & Regret-efficient comparator; temporal improvement range 41.73--57.56\%. & models/spo\_real\_training\_status.json \\ -CRPTO robust & 0.947429 & -122.455465 & 0.9207--0.9858 & True & True & 3 & Auditable robust champion: coverage, exact bound and 45/45 region. & models/spo\_real\_training\_status.json; data/processed/crpto\_vs\_spo\_stability.json; models/final\_project\_promotion.json \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv b/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv deleted file mode 100644 index bda9cbf..0000000 --- a/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv +++ /dev/null @@ -1,46 +0,0 @@ -tail_satisficing_rank,candidate_rank,paper_role,risk_tolerance,gamma,uncertainty_aversion,realized_total_return,return_delta_vs_champion_pct,cvar_95_loss_rate,cvar_delta_vs_champion_pct,entropic_oce_theta5,mean_loss_rate,alpha01_weighted_miscoverage_V,alpha01_gamma_cp,satisficing_pass,min_satisficing_margin,satisficing_shortfall,audit_n_funded,audit_solver_backend,audit_solver_status -1,31,robust_region_policy,0.17,0.55,0.1,167069.47757462133,-1.9916548600175177,0.1968157499999998,-22.579246171804865,-0.0835694670146279,-0.1670694775746475,0.026875,0.1523,True,-0.0,0.0,337,highs,TerminationCondition.optimal -2,20,robust_region_policy,0.16,0.55,0.05,164273.35080211036,-3.631953026706874,0.1968979607309555,-22.54690721131335,-0.0821047630877449,-0.1642733508018371,0.027861,0.137016,True,-0.0,0.0,342,highs,TerminationCondition.optimal -3,17,robust_region_policy,0.155,0.45,0.1,167067.29591823329,-1.992934690034108,0.1970557499999998,-22.48483817387397,-0.0835269647230217,-0.1670672959183653,0.026875,0.153284,True,-0.0,0.0,342,highs,TerminationCondition.optimal -4,32,robust_region_policy,0.17,0.55,0.05,166969.46159398026,-2.0503274607372104,0.1970557499999998,-22.48483817387397,-0.0834746390594157,-0.166969461593887,0.026875,0.152781,True,-0.0,0.0,343,highs,TerminationCondition.optimal -5,21,robust_region_policy,0.16,0.5,0.1,166244.65010111933,-2.4755252645790944,0.1970557499999998,-22.48483817387397,-0.0831040444217096,-0.1662446501011471,0.026875,0.148671,True,-0.0,0.0,341,highs,TerminationCondition.optimal -6,26,robust_region_policy,0.165,0.55,0.05,165547.8905382124,-2.8842669013658884,0.1970557499999998,-22.48483817387397,-0.0827443035522485,-0.1655478905381514,0.026875,0.144817,True,-0.0,0.0,344,highs,TerminationCondition.optimal -7,15,robust_region_policy,0.155,0.5,0.1,164689.02196237698,-3.388106914708639,0.1970557499999998,-22.48483817387397,-0.082301894850758,-0.1646890219624037,0.026875,0.140067,True,-0.0,0.0,346,highs,TerminationCondition.optimal -8,27,robust_region_policy,0.165,0.5,0.1,167692.389704731,-1.6262345095983184,0.1987236263626536,-21.828751223063964,-0.0835019553354872,-0.16769238970446,0.026875,0.157266,True,-0.0,0.0,344,highs,TerminationCondition.optimal -9,14,robust_region_policy,0.155,0.55,0.05,162753.66070085138,-4.523452264701919,0.1993517499999997,-21.5816683270018,-0.0807878479886671,-0.1627536607007726,0.028075,0.128851,True,-0.0,0.0,344,highs,TerminationCondition.optimal -10,4,robust_region_policy,0.16,0.55,0.0,163847.98926196393,-3.8814838281503983,0.2004537499999998,-21.148178269835768,-0.0812857202718335,-0.163895515783193,0.028075,0.138098,True,-0.0,0.0,350,highs,TerminationCondition.optimal -11,25,robust_region_policy,0.165,0.55,0.1,165504.23210778716,-2.90987834505689,0.2018572116529251,-20.5961032497297,-0.081677706029354,-0.1653666671079917,0.027293,0.144453,True,-0.0,0.0,343,highs,TerminationCondition.optimal -12,5,robust_region_policy,0.16,0.5,0.0,163748.62808782174,-3.9397722982540553,0.2023947499999998,-20.384653586569684,-0.0822938133365853,-0.165962628088214,0.028075,0.150322,True,-0.0,0.0,345,highs,TerminationCondition.optimal -13,2,robust_region_policy,0.155,0.5,0.0,162164.37781018534,-4.869144618397633,0.2023947499999998,-20.384653586569684,-0.0814654287780037,-0.1643783778104844,0.028075,0.141875,True,-0.0,0.0,351,highs,TerminationCondition.optimal -14,1,robust_region_policy,0.155,0.55,0.0,162326.28515545817,-4.774164546594208,0.2023947499999998,-20.384653586569684,-0.0804081256415588,-0.1623262851552953,0.028075,0.130655,True,-0.0,0.0,346,highs,TerminationCondition.optimal -15,3,robust_region_policy,0.155,0.45,0.0,166621.64409318674,-2.254368427156536,0.20481637928952,-19.432065365926302,-0.0821208372920303,-0.1666216440930363,0.028075,0.155245,True,-0.0,0.0,344,highs,TerminationCondition.optimal -16,22,robust_region_policy,0.16,0.5,0.05,165517.85855195278,-2.901884633325681,0.2108317499999997,-17.065819092640826,-0.07983597577446,-0.1655178585521693,0.028075,0.148889,True,-0.0,0.0,345,highs,TerminationCondition.optimal -17,16,robust_region_policy,0.155,0.5,0.05,163947.7110439952,-3.8229838138259447,0.2108317499999997,-17.065819092640826,-0.0790317867770713,-0.1639477110441848,0.028075,0.14036,True,-0.0,0.0,346,highs,TerminationCondition.optimal -18,18,robust_region_policy,0.155,0.45,0.05,164022.0917987445,-3.779349663596513,0.2116817499999997,-16.731457433302243,-0.0801349542528505,-0.1662360917987894,0.028075,0.154136,True,-0.0,0.0,347,highs,TerminationCondition.optimal -19,10,robust_region_policy,0.17,0.55,0.0,163897.7680059752,-3.852282011070959,0.2116817499999997,-16.731457433302243,-0.0800690319473513,-0.1661117680058082,0.028075,0.153506,True,-0.0,0.0,347,highs,TerminationCondition.optimal -20,7,robust_region_policy,0.165,0.55,0.0,163116.36710524172,-4.310676987122609,0.2116817499999997,-16.731457433302243,-0.0793081849097237,-0.1646450493798757,0.028075,0.145736,True,-0.0,0.0,350,highs,TerminationCondition.optimal -21,28,robust_region_policy,0.165,0.5,0.05,166777.3884430658,-2.163003767266868,0.2156381451728425,-15.175143486254218,-0.0794860311354021,-0.16677738844227,0.028075,0.157418,True,-0.0,0.0,346,highs,TerminationCondition.optimal -22,19,robust_region_policy,0.16,0.55,0.1,162351.17715355012,-4.759562097507543,0.2171264461078983,-14.589695521265488,-0.077342249947325,-0.1631164552371796,0.028623,0.136535,True,-0.0,0.0,342,highs,TerminationCondition.optimal -23,37,theorem_tight_comparator,0.175,0.55,0.1,166269.82231927008,-2.460758429366147,0.2312557499999998,-9.031698468721952,-0.0760187226893278,-0.1662698223192979,0.026875,0.160299,True,-0.0,0.0,342,highs,TerminationCondition.optimal -24,41,robust_region_policy,0.175,0.5,0.05,168801.31532762162,-0.975702965712392,0.2314957499999998,-8.937290470791053,-0.0772637389940441,-0.1688013153278322,0.026875,0.173665,True,-0.0,0.0,343,highs,TerminationCondition.optimal -25,42,robust_region_policy,0.175,0.5,0.0,168669.5041537059,-1.0530276525072486,0.2314957499999998,-8.937290470791053,-0.0771806432189668,-0.1686695041540769,0.026875,0.17428,True,-0.0,0.0,346,highs,TerminationCondition.optimal -26,29,robust_region_policy,0.165,0.45,0.1,168365.23091736945,-1.2315241486754518,0.2314957499999998,-8.937290470791053,-0.077047861589509,-0.1683652309175472,0.026875,0.171171,True,-0.0,0.0,345,highs,TerminationCondition.optimal -27,30,robust_region_policy,0.165,0.45,0.05,168244.12224854622,-1.3025704007245107,0.2314957499999998,-8.937290470791053,-0.0769851196624198,-0.1682441222486654,0.026875,0.17189,True,-0.0,0.0,347,highs,TerminationCondition.optimal -28,33,robust_region_policy,0.17,0.5,0.1,167226.8353415202,-1.8993437176558423,0.2314957499999998,-8.937290470791053,-0.0764780867612215,-0.1672268353415537,0.026875,0.165327,True,-0.0,0.0,345,highs,TerminationCondition.optimal -29,34,robust_region_policy,0.17,0.5,0.05,167124.6310620346,-1.959300045071384,0.2314957499999998,-8.937290470791053,-0.0764022189444275,-0.1671246310622178,0.026875,0.165864,True,-0.0,0.0,346,highs,TerminationCondition.optimal -30,23,robust_region_policy,0.16,0.45,0.1,166559.14303245154,-2.291033565643095,0.2314957499999998,-8.937290470791053,-0.0761216676147778,-0.1665627755668657,0.026875,0.162606,True,-0.0,0.0,347,highs,TerminationCondition.optimal -31,38,robust_region_policy,0.175,0.55,0.05,166186.4620978511,-2.5096602244180195,0.2314957499999998,-8.937290470791053,-0.0759338616340996,-0.16618646209781,0.026875,0.160699,True,-0.0,0.0,345,highs,TerminationCondition.optimal -32,13,robust_region_policy,0.155,0.55,0.1,160978.00154814168,-5.565111205737463,0.2314957499999998,-8.937290470791053,-0.0733474878115397,-0.1609780015481643,0.029875,0.12825,True,-0.0,0.0,346,highs,TerminationCondition.optimal -33,8,robust_region_policy,0.165,0.5,0.0,165401.62753483138,-2.97006949753535,0.2360412436011981,-7.149244843721016,-0.0749582567867925,-0.1654016275350454,0.028075,0.158457,True,-0.0,0.0,345,highs,TerminationCondition.optimal -34,35,balanced_comparator,0.17,0.45,0.1,169389.5810115051,-0.6306073383300145,0.2427357499999998,-4.51584923436096,-0.075239524977382,-0.1693895810116205,0.027875,0.179825,True,-0.0,0.0,343,highs,TerminationCondition.optimal -35,40,robust_region_policy,0.175,0.5,0.1,168128.1097153701,-1.3706270952036637,0.2427357499999998,-4.51584923436096,-0.0746075107597085,-0.1681281097154604,0.027875,0.17327,True,-0.0,0.0,345,highs,TerminationCondition.optimal -36,36,robust_region_policy,0.17,0.45,0.05,169337.1631898248,-0.6613573235057056,0.2429757499999998,-4.421441236430073,-0.0752112942780283,-0.1693371631899614,0.027875,0.180138,True,-0.0,0.0,344,highs,TerminationCondition.optimal -37,12,robust_region_policy,0.17,0.45,0.0,169262.37545367464,-0.7052302221539193,0.2429757499999998,-4.421441236430073,-0.0751358089764025,-0.1692623754538053,0.027875,0.180537,True,-0.0,0.0,343,highs,TerminationCondition.optimal -38,24,robust_region_policy,0.16,0.45,0.05,165829.33653326883,-2.719161601330249,0.2452717499999997,-3.518271389557902,-0.072963919558912,-0.1658293365333349,0.028075,0.162897,True,-0.0,0.0,348,highs,TerminationCondition.optimal -39,9,robust_region_policy,0.165,0.45,0.0,167494.8582103625,-1.742112856482107,0.2454126916184966,-3.4628296561102703,-0.0737902640323714,-0.1674948582106428,0.028075,0.172234,True,-0.0,0.0,348,highs,TerminationCondition.optimal -40,11,robust_region_policy,0.17,0.5,0.0,166313.57843638843,-2.435089679509638,0.2460147180624018,-3.22601252581636,-0.0731881486959823,-0.1663135784366802,0.028075,0.166451,True,-0.0,0.0,347,highs,TerminationCondition.optimal -41,39,robust_region_policy,0.175,0.55,0.0,165375.90860423315,-2.985157051906716,0.2461217499999997,-3.1839097302193324,-0.0726971387537753,-0.1653759086040766,0.028075,0.161223,True,-0.0,0.0,348,highs,TerminationCondition.optimal -42,6,robust_region_policy,0.16,0.45,0.0,165590.76458837948,-2.859115600438128,0.2480207499999997,-2.43690644659116,-0.0727457423939489,-0.1655907645885528,0.028075,0.164142,True,-0.0,0.0,348,highs,TerminationCondition.optimal -43,43,economic_champion,0.175,0.45,0.1,170464.5429284627,0.0,0.2542157499999998,0.0,-0.0734634367187448,-0.1704645429285968,0.028875,0.187987,True,-0.0,0.0,341,highs,TerminationCondition.optimal -44,44,robust_region_policy,0.175,0.45,0.05,166574.19464587132,-2.2822038036520054,0.3116975351392571,22.61141771871229,-0.0604519567180308,-0.1665741946456538,0.027875,0.189263,True,-0.0,0.0,342,highs,TerminationCondition.optimal -45,45,robust_region_policy,0.175,0.45,0.0,166210.47853140024,-2.495571409737525,0.3164477499999997,24.479993863480114,-0.0593654126511955,-0.1662104785315218,0.027875,0.189562,True,-0.0,0.0,343,highs,TerminationCondition.optimal diff --git a/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.tex b/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.tex deleted file mode 100644 index 1ee0ff1..0000000 --- a/reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.tex +++ /dev/null @@ -1,51 +0,0 @@ -\begin{tabular}{rrlrrrrrrrrrrrrrrrll} -\toprule -tail\_satisficing\_rank & candidate\_rank & paper\_role & risk\_tolerance & gamma & uncertainty\_aversion & realized\_total\_return & return\_delta\_vs\_champion\_pct & cvar\_95\_loss\_rate & cvar\_delta\_vs\_champion\_pct & entropic\_oce\_theta5 & mean\_loss\_rate & alpha01\_weighted\_miscoverage\_V & alpha01\_gamma\_cp & satisficing\_pass & min\_satisficing\_margin & satisficing\_shortfall & audit\_n\_funded & audit\_solver\_backend & audit\_solver\_status \\ -\midrule -1 & 31 & robust\_region\_policy & 0.170000 & 0.550000 & 0.100000 & 167069.477575 & -1.991655 & 0.196816 & -22.579246 & -0.083569 & -0.167069 & 0.026875 & 0.152300 & True & -0.000000 & 0.000000 & 337 & highs & TerminationCondition.optimal \\ -2 & 20 & robust\_region\_policy & 0.160000 & 0.550000 & 0.050000 & 164273.350802 & -3.631953 & 0.196898 & -22.546907 & -0.082105 & -0.164273 & 0.027861 & 0.137016 & True & -0.000000 & 0.000000 & 342 & highs & TerminationCondition.optimal \\ -3 & 17 & robust\_region\_policy & 0.155000 & 0.450000 & 0.100000 & 167067.295918 & -1.992935 & 0.197056 & -22.484838 & -0.083527 & -0.167067 & 0.026875 & 0.153284 & True & -0.000000 & 0.000000 & 342 & highs & TerminationCondition.optimal \\ -4 & 32 & robust\_region\_policy & 0.170000 & 0.550000 & 0.050000 & 166969.461594 & -2.050327 & 0.197056 & -22.484838 & -0.083475 & -0.166969 & 0.026875 & 0.152781 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -5 & 21 & robust\_region\_policy & 0.160000 & 0.500000 & 0.100000 & 166244.650101 & -2.475525 & 0.197056 & -22.484838 & -0.083104 & -0.166245 & 0.026875 & 0.148671 & True & -0.000000 & 0.000000 & 341 & highs & TerminationCondition.optimal \\ -6 & 26 & robust\_region\_policy & 0.165000 & 0.550000 & 0.050000 & 165547.890538 & -2.884267 & 0.197056 & -22.484838 & -0.082744 & -0.165548 & 0.026875 & 0.144817 & True & -0.000000 & 0.000000 & 344 & highs & TerminationCondition.optimal \\ -7 & 15 & robust\_region\_policy & 0.155000 & 0.500000 & 0.100000 & 164689.021962 & -3.388107 & 0.197056 & -22.484838 & -0.082302 & -0.164689 & 0.026875 & 0.140067 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -8 & 27 & robust\_region\_policy & 0.165000 & 0.500000 & 0.100000 & 167692.389705 & -1.626235 & 0.198724 & -21.828751 & -0.083502 & -0.167692 & 0.026875 & 0.157266 & True & -0.000000 & 0.000000 & 344 & highs & TerminationCondition.optimal \\ -9 & 14 & robust\_region\_policy & 0.155000 & 0.550000 & 0.050000 & 162753.660701 & -4.523452 & 0.199352 & -21.581668 & -0.080788 & -0.162754 & 0.028075 & 0.128851 & True & -0.000000 & 0.000000 & 344 & highs & TerminationCondition.optimal \\ -10 & 4 & robust\_region\_policy & 0.160000 & 0.550000 & 0.000000 & 163847.989262 & -3.881484 & 0.200454 & -21.148178 & -0.081286 & -0.163896 & 0.028075 & 0.138098 & True & -0.000000 & 0.000000 & 350 & highs & TerminationCondition.optimal \\ -11 & 25 & robust\_region\_policy & 0.165000 & 0.550000 & 0.100000 & 165504.232108 & -2.909878 & 0.201857 & -20.596103 & -0.081678 & -0.165367 & 0.027293 & 0.144453 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -12 & 5 & robust\_region\_policy & 0.160000 & 0.500000 & 0.000000 & 163748.628088 & -3.939772 & 0.202395 & -20.384654 & -0.082294 & -0.165963 & 0.028075 & 0.150322 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -13 & 2 & robust\_region\_policy & 0.155000 & 0.500000 & 0.000000 & 162164.377810 & -4.869145 & 0.202395 & -20.384654 & -0.081465 & -0.164378 & 0.028075 & 0.141875 & True & -0.000000 & 0.000000 & 351 & highs & TerminationCondition.optimal \\ -14 & 1 & robust\_region\_policy & 0.155000 & 0.550000 & 0.000000 & 162326.285155 & -4.774165 & 0.202395 & -20.384654 & -0.080408 & -0.162326 & 0.028075 & 0.130655 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -15 & 3 & robust\_region\_policy & 0.155000 & 0.450000 & 0.000000 & 166621.644093 & -2.254368 & 0.204816 & -19.432065 & -0.082121 & -0.166622 & 0.028075 & 0.155245 & True & -0.000000 & 0.000000 & 344 & highs & TerminationCondition.optimal \\ -16 & 22 & robust\_region\_policy & 0.160000 & 0.500000 & 0.050000 & 165517.858552 & -2.901885 & 0.210832 & -17.065819 & -0.079836 & -0.165518 & 0.028075 & 0.148889 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -17 & 16 & robust\_region\_policy & 0.155000 & 0.500000 & 0.050000 & 163947.711044 & -3.822984 & 0.210832 & -17.065819 & -0.079032 & -0.163948 & 0.028075 & 0.140360 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -18 & 18 & robust\_region\_policy & 0.155000 & 0.450000 & 0.050000 & 164022.091799 & -3.779350 & 0.211682 & -16.731457 & -0.080135 & -0.166236 & 0.028075 & 0.154136 & True & -0.000000 & 0.000000 & 347 & highs & TerminationCondition.optimal \\ -19 & 10 & robust\_region\_policy & 0.170000 & 0.550000 & 0.000000 & 163897.768006 & -3.852282 & 0.211682 & -16.731457 & -0.080069 & -0.166112 & 0.028075 & 0.153506 & True & -0.000000 & 0.000000 & 347 & highs & TerminationCondition.optimal \\ -20 & 7 & robust\_region\_policy & 0.165000 & 0.550000 & 0.000000 & 163116.367105 & -4.310677 & 0.211682 & -16.731457 & -0.079308 & -0.164645 & 0.028075 & 0.145736 & True & -0.000000 & 0.000000 & 350 & highs & TerminationCondition.optimal \\ -21 & 28 & robust\_region\_policy & 0.165000 & 0.500000 & 0.050000 & 166777.388443 & -2.163004 & 0.215638 & -15.175143 & -0.079486 & -0.166777 & 0.028075 & 0.157418 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -22 & 19 & robust\_region\_policy & 0.160000 & 0.550000 & 0.100000 & 162351.177154 & -4.759562 & 0.217126 & -14.589696 & -0.077342 & -0.163116 & 0.028623 & 0.136535 & True & -0.000000 & 0.000000 & 342 & highs & TerminationCondition.optimal \\ -23 & 37 & theorem\_tight\_comparator & 0.175000 & 0.550000 & 0.100000 & 166269.822319 & -2.460758 & 0.231256 & -9.031698 & -0.076019 & -0.166270 & 0.026875 & 0.160299 & True & -0.000000 & 0.000000 & 342 & highs & TerminationCondition.optimal \\ -24 & 41 & robust\_region\_policy & 0.175000 & 0.500000 & 0.050000 & 168801.315328 & -0.975703 & 0.231496 & -8.937290 & -0.077264 & -0.168801 & 0.026875 & 0.173665 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -25 & 42 & robust\_region\_policy & 0.175000 & 0.500000 & 0.000000 & 168669.504154 & -1.053028 & 0.231496 & -8.937290 & -0.077181 & -0.168670 & 0.026875 & 0.174280 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -26 & 29 & robust\_region\_policy & 0.165000 & 0.450000 & 0.100000 & 168365.230917 & -1.231524 & 0.231496 & -8.937290 & -0.077048 & -0.168365 & 0.026875 & 0.171171 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -27 & 30 & robust\_region\_policy & 0.165000 & 0.450000 & 0.050000 & 168244.122249 & -1.302570 & 0.231496 & -8.937290 & -0.076985 & -0.168244 & 0.026875 & 0.171890 & True & -0.000000 & 0.000000 & 347 & highs & TerminationCondition.optimal \\ -28 & 33 & robust\_region\_policy & 0.170000 & 0.500000 & 0.100000 & 167226.835342 & -1.899344 & 0.231496 & -8.937290 & -0.076478 & -0.167227 & 0.026875 & 0.165327 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -29 & 34 & robust\_region\_policy & 0.170000 & 0.500000 & 0.050000 & 167124.631062 & -1.959300 & 0.231496 & -8.937290 & -0.076402 & -0.167125 & 0.026875 & 0.165864 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -30 & 23 & robust\_region\_policy & 0.160000 & 0.450000 & 0.100000 & 166559.143032 & -2.291034 & 0.231496 & -8.937290 & -0.076122 & -0.166563 & 0.026875 & 0.162606 & True & -0.000000 & 0.000000 & 347 & highs & TerminationCondition.optimal \\ -31 & 38 & robust\_region\_policy & 0.175000 & 0.550000 & 0.050000 & 166186.462098 & -2.509660 & 0.231496 & -8.937290 & -0.075934 & -0.166186 & 0.026875 & 0.160699 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -32 & 13 & robust\_region\_policy & 0.155000 & 0.550000 & 0.100000 & 160978.001548 & -5.565111 & 0.231496 & -8.937290 & -0.073347 & -0.160978 & 0.029875 & 0.128250 & True & -0.000000 & 0.000000 & 346 & highs & TerminationCondition.optimal \\ -33 & 8 & robust\_region\_policy & 0.165000 & 0.500000 & 0.000000 & 165401.627535 & -2.970069 & 0.236041 & -7.149245 & -0.074958 & -0.165402 & 0.028075 & 0.158457 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -34 & 35 & balanced\_comparator & 0.170000 & 0.450000 & 0.100000 & 169389.581012 & -0.630607 & 0.242736 & -4.515849 & -0.075240 & -0.169390 & 0.027875 & 0.179825 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -35 & 40 & robust\_region\_policy & 0.175000 & 0.500000 & 0.100000 & 168128.109715 & -1.370627 & 0.242736 & -4.515849 & -0.074608 & -0.168128 & 0.027875 & 0.173270 & True & -0.000000 & 0.000000 & 345 & highs & TerminationCondition.optimal \\ -36 & 36 & robust\_region\_policy & 0.170000 & 0.450000 & 0.050000 & 169337.163190 & -0.661357 & 0.242976 & -4.421441 & -0.075211 & -0.169337 & 0.027875 & 0.180138 & True & -0.000000 & 0.000000 & 344 & highs & TerminationCondition.optimal \\ -37 & 12 & robust\_region\_policy & 0.170000 & 0.450000 & 0.000000 & 169262.375454 & -0.705230 & 0.242976 & -4.421441 & -0.075136 & -0.169262 & 0.027875 & 0.180537 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -38 & 24 & robust\_region\_policy & 0.160000 & 0.450000 & 0.050000 & 165829.336533 & -2.719162 & 0.245272 & -3.518271 & -0.072964 & -0.165829 & 0.028075 & 0.162897 & True & -0.000000 & 0.000000 & 348 & highs & TerminationCondition.optimal \\ -39 & 9 & robust\_region\_policy & 0.165000 & 0.450000 & 0.000000 & 167494.858210 & -1.742113 & 0.245413 & -3.462830 & -0.073790 & -0.167495 & 0.028075 & 0.172234 & True & -0.000000 & 0.000000 & 348 & highs & TerminationCondition.optimal \\ -40 & 11 & robust\_region\_policy & 0.170000 & 0.500000 & 0.000000 & 166313.578436 & -2.435090 & 0.246015 & -3.226013 & -0.073188 & -0.166314 & 0.028075 & 0.166451 & True & -0.000000 & 0.000000 & 347 & highs & TerminationCondition.optimal \\ -41 & 39 & robust\_region\_policy & 0.175000 & 0.550000 & 0.000000 & 165375.908604 & -2.985157 & 0.246122 & -3.183910 & -0.072697 & -0.165376 & 0.028075 & 0.161223 & True & -0.000000 & 0.000000 & 348 & highs & TerminationCondition.optimal \\ -42 & 6 & robust\_region\_policy & 0.160000 & 0.450000 & 0.000000 & 165590.764588 & -2.859116 & 0.248021 & -2.436906 & -0.072746 & -0.165591 & 0.028075 & 0.164142 & True & -0.000000 & 0.000000 & 348 & highs & TerminationCondition.optimal \\ -43 & 43 & economic\_champion & 0.175000 & 0.450000 & 0.100000 & 170464.542928 & 0.000000 & 0.254216 & 0.000000 & -0.073463 & -0.170465 & 0.028875 & 0.187987 & True & -0.000000 & 0.000000 & 341 & highs & TerminationCondition.optimal \\ -44 & 44 & robust\_region\_policy & 0.175000 & 0.450000 & 0.050000 & 166574.194646 & -2.282204 & 0.311698 & 22.611418 & -0.060452 & -0.166574 & 0.027875 & 0.189263 & True & -0.000000 & 0.000000 & 342 & highs & TerminationCondition.optimal \\ -45 & 45 & robust\_region\_policy & 0.175000 & 0.450000 & 0.000000 & 166210.478531 & -2.495571 & 0.316448 & 24.479994 & -0.059365 & -0.166210 & 0.027875 & 0.189562 & True & -0.000000 & 0.000000 & 343 & highs & TerminationCondition.optimal \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv b/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv deleted file mode 100644 index 00d720f..0000000 --- a/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv +++ /dev/null @@ -1,10 +0,0 @@ -slice_type,slice,n,coverage_90,default_rate,mean_width_90,robust_gap_vs_target,meets_target_90 -marginal,all,276869,0.9293384235866059,0.21977902907151034,0.764155715329014,0.0,True -grade,A,72388,0.9124026081670995,0.08658893739293806,0.24264355719096337,0.0,True -grade,B,75311,0.9426644182124789,0.17861932519817822,0.9309479159573925,0.0,True -grade,C,70568,0.9395476703321619,0.2760741412538261,0.9587326305088323,0.0,True -grade,D,45637,0.925192278195324,0.34745929837631745,0.9592537235601951,0.0,True -grade,E,10828,0.9003509420022164,0.4314739564093092,0.9578313800495935,0.0,True -grade,F,1731,0.9272097053726169,0.5210860774119006,0.9781572310912804,0.0,True -grade,G,406,0.9507389162561576,0.5344827586206896,0.9803783959660908,0.0,True -worst_grade_x_period_cell,E|2018Q4,2546,0.8762765121759623,0.42419481539670073,0.9420210022502649,0.02372348782403777,False diff --git a/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.tex b/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.tex deleted file mode 100644 index a632360..0000000 --- a/reports/crpto/tables/crpto_tableA23_multidistribution_robustness.tex +++ /dev/null @@ -1,15 +0,0 @@ -\begin{tabular}{llrrrrrr} -\toprule -slice\_type & slice & n & coverage\_90 & default\_rate & mean\_width\_90 & robust\_gap\_vs\_target & meets\_target\_90 \\ -\midrule -marginal & all & 276869 & 0.9293 & 0.2198 & 0.7642 & 0.0000 & True \\ -grade & A & 72388 & 0.9124 & 0.0866 & 0.2426 & 0.0000 & True \\ -grade & B & 75311 & 0.9427 & 0.1786 & 0.9309 & 0.0000 & True \\ -grade & C & 70568 & 0.9395 & 0.2761 & 0.9587 & 0.0000 & True \\ -grade & D & 45637 & 0.9252 & 0.3475 & 0.9593 & 0.0000 & True \\ -grade & E & 10828 & 0.9004 & 0.4315 & 0.9578 & 0.0000 & True \\ -grade & F & 1731 & 0.9272 & 0.5211 & 0.9782 & 0.0000 & True \\ -grade & G & 406 & 0.9507 & 0.5345 & 0.9804 & 0.0000 & True \\ -worst\_grade\_x\_period\_cell & E|2018Q4 & 2546 & 0.8763 & 0.4242 & 0.9420 & 0.0237 & False \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv b/reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv deleted file mode 100644 index 3e47468..0000000 --- a/reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv +++ /dev/null @@ -1,12 +0,0 @@ -period,n,default_rate,coverage_90,miscoverage_90,mean_width_90,cumulative_coverage_90,aci_alpha_target_before -2018Q1,53924,0.2329945849714413,0.9250241080038573,0.07497589199614274,0.7715862681573208,0.9250241080038573,0.09999999999999998 -2018Q2,56915,0.2556268119125011,0.923289115347448,0.07671088465255205,0.7755167800328171,0.9241332022122177,0.10125120540019283 -2018Q3,47168,0.23289094301221167,0.9275144165535957,0.07248558344640432,0.7622986411754243,0.9251425569753239,0.10241566116756523 -2018Q4,39171,0.23305506624798958,0.9225192106405249,0.07748078935947511,0.7638591783558729,0.9246214080678372,0.10379138199524501 -2019Q1,27508,0.21241093500072705,0.9273665842663952,0.07263341573360482,0.7597500310271217,0.924957496239196,0.10491734252727125 -2019Q2,22737,0.20191757927606985,0.9309055724150064,0.06909442758499362,0.7498006989607373,0.9255040962238757,0.106285671740591 -2019Q3,16603,0.1451544901523821,0.950129494669638,0.04987050533036197,0.750793621836384,0.9270526387552741,0.10783095036134133 -2019Q4,8557,0.08425850181138249,0.9682131588173425,0.03178684118265751,0.7624737976874291,0.9283447610452596,0.11033742509482322 -2020Q1,2977,0.0141081625797783,0.9919381928115553,0.008061807188444736,0.7472963733292841,0.9290317898098418,0.11374808303569034 -2020Q2,824,0.01820388349514563,0.991504854368932,0.008495145631068013,0.5992341301418823,0.9292180444598819,0.11834499267626811 -2020Q3,485,0.002061855670103093,0.9979381443298969,0.0020618556701030855,0.6029602945194906,0.9293384235866059,0.12292023539471471 diff --git a/reports/crpto/tables/crpto_tableA24_online_conformal_stability.tex b/reports/crpto/tables/crpto_tableA24_online_conformal_stability.tex deleted file mode 100644 index 1e6ae16..0000000 --- a/reports/crpto/tables/crpto_tableA24_online_conformal_stability.tex +++ /dev/null @@ -1,17 +0,0 @@ -\begin{tabular}{lrrrrrrr} -\toprule -period & n & default\_rate & coverage\_90 & miscoverage\_90 & mean\_width\_90 & cumulative\_coverage\_90 & aci\_alpha\_target\_before \\ -\midrule -2018Q1 & 53924 & 0.2330 & 0.9250 & 0.0750 & 0.7716 & 0.9250 & 0.1000 \\ -2018Q2 & 56915 & 0.2556 & 0.9233 & 0.0767 & 0.7755 & 0.9241 & 0.1013 \\ -2018Q3 & 47168 & 0.2329 & 0.9275 & 0.0725 & 0.7623 & 0.9251 & 0.1024 \\ -2018Q4 & 39171 & 0.2331 & 0.9225 & 0.0775 & 0.7639 & 0.9246 & 0.1038 \\ -2019Q1 & 27508 & 0.2124 & 0.9274 & 0.0726 & 0.7598 & 0.9250 & 0.1049 \\ -2019Q2 & 22737 & 0.2019 & 0.9309 & 0.0691 & 0.7498 & 0.9255 & 0.1063 \\ -2019Q3 & 16603 & 0.1452 & 0.9501 & 0.0499 & 0.7508 & 0.9271 & 0.1078 \\ -2019Q4 & 8557 & 0.0843 & 0.9682 & 0.0318 & 0.7625 & 0.9283 & 0.1103 \\ -2020Q1 & 2977 & 0.0141 & 0.9919 & 0.0081 & 0.7473 & 0.9290 & 0.1137 \\ -2020Q2 & 824 & 0.0182 & 0.9915 & 0.0085 & 0.5992 & 0.9292 & 0.1183 \\ -2020Q3 & 485 & 0.0021 & 0.9979 & 0.0021 & 0.6030 & 0.9293 & 0.1229 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA25_external_replication_gate.csv b/reports/crpto/tables/crpto_tableA25_external_replication_gate.csv deleted file mode 100644 index 015137b..0000000 --- a/reports/crpto/tables/crpto_tableA25_external_replication_gate.csv +++ /dev/null @@ -1,3 +0,0 @@ -dataset,credit_product,external_role,n_rows,default_rate,auc_roc,pr_auc,brier,coverage_90,min_group_coverage_90,alpha01_coverage,oot_candidates,lp_candidate_cap,robust_objective,price_of_robustness_pct,gate -Prosper,Marketplace personal loans,main external economic replication,54807,0.3091758352035324,0.7074290432845566,0.4484817219126147,0.1781741704802637,0.9205159821682634,0.8850308641975309,0.9943090202029784,10531,all,199419.08877567088,0.094619019478566,pass -Freddie FM48,Single-family mortgages,main external economic replication,3173355,0.0145435981792141,0.7838634726333499,0.0517822051269566,0.0128541034734516,0.974523173547136,0.5,0.9907116706887202,1396053,all,1291227.5801824057,0.0108636254963404,pass diff --git a/reports/crpto/tables/crpto_tableA25_external_replication_gate.tex b/reports/crpto/tables/crpto_tableA25_external_replication_gate.tex deleted file mode 100644 index d30d3a4..0000000 --- a/reports/crpto/tables/crpto_tableA25_external_replication_gate.tex +++ /dev/null @@ -1,8 +0,0 @@ -\begin{tabular}{lllrrrrrrrrrlrrl} -\toprule -dataset & credit\_product & external\_role & n\_rows & default\_rate & auc\_roc & pr\_auc & brier & coverage\_90 & min\_group\_coverage\_90 & alpha01\_coverage & oot\_candidates & lp\_candidate\_cap & robust\_objective & price\_of\_robustness\_pct & gate \\ -\midrule -Prosper & Marketplace personal loans & main external economic replication & 54807 & 0.3092 & 0.7074 & 0.4485 & 0.1782 & 0.9205 & 0.8850 & 0.9943 & 10531 & all & 199419.0888 & 0.0946 & pass \\ -Freddie FM48 & Single-family mortgages & main external economic replication & 3173355 & 0.0145 & 0.7839 & 0.0518 & 0.0129 & 0.9745 & 0.5000 & 0.9907 & 1396053 & all & 1291227.5802 & 0.0109 & pass \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.csv b/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.csv deleted file mode 100644 index 601ff0a..0000000 --- a/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.csv +++ /dev/null @@ -1,18 +0,0 @@ -dataset,screen,candidate_cap,available_candidates,runs,robust_objective_mean,robust_objective_min,robust_objective_max,nonrobust_objective_mean,robust_n_funded_mean -Prosper,top_net_return,1000,10531,1,199419.0887756709,199419.0887756709,199419.0887756709,220259.86083871548,233.0 -Prosper,top_net_return,5000,10531,1,199419.0887756709,199419.0887756709,199419.0887756709,220259.86083871548,233.0 -Prosper,top_net_return,all,10531,1,199419.08877567088,199419.08877567088,199419.08877567088,220259.8608387156,234.0 -Prosper,random,1000,10531,3,136809.82583129438,134780.4007618212,140692.0500073207,162601.0814660202,187.0 -Prosper,random,5000,10531,3,183373.59959687816,181977.3476818193,184805.77749464923,205554.45701109208,231.0 -Prosper,random,all,10531,3,199419.0887756712,199419.08877567103,199419.0887756713,220259.8608387152,234.33333333333331 -Freddie FM48,top_net_return,1000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,5000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,10000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,25000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,50000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,100000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,top_net_return,250000,1396053,1,1291227.5801824068,1291227.5801824068,1291227.5801824068,1305409.055278483,146.0 -Freddie FM48,random,1000,1396053,3,1056743.4698473162,1054043.8340842563,1060065.9648987832,1075554.8612782704,113.0 -Freddie FM48,random,5000,1396053,3,1129835.6496948672,1124577.3552450945,1134306.807301479,1147433.540716556,121.66666666666669 -Freddie FM48,random,10000,1396053,3,1163040.3771203093,1160204.5486363675,1165078.147595012,1179829.1562833728,114.66666666666669 -Freddie FM48,random,25000,1396053,3,1192925.997526994,1189915.324215843,1198143.740233957,1209898.5778409762,116.0 diff --git a/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.tex b/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.tex deleted file mode 100644 index 89204f7..0000000 --- a/reports/crpto/tables/crpto_tableA26_external_candidate_sensitivity.tex +++ /dev/null @@ -1,23 +0,0 @@ -\begin{tabular}{lllrrrrrrr} -\toprule -dataset & screen & candidate\_cap & available\_candidates & runs & robust\_objective\_mean & robust\_objective\_min & robust\_objective\_max & nonrobust\_objective\_mean & robust\_n\_funded\_mean \\ -\midrule -Prosper & top\_net\_return & 1000 & 10531 & 1 & 199419.0888 & 199419.0888 & 199419.0888 & 220259.8608 & 233.0000 \\ -Prosper & top\_net\_return & 5000 & 10531 & 1 & 199419.0888 & 199419.0888 & 199419.0888 & 220259.8608 & 233.0000 \\ -Prosper & top\_net\_return & all & 10531 & 1 & 199419.0888 & 199419.0888 & 199419.0888 & 220259.8608 & 234.0000 \\ -Prosper & random & 1000 & 10531 & 3 & 136809.8258 & 134780.4008 & 140692.0500 & 162601.0815 & 187.0000 \\ -Prosper & random & 5000 & 10531 & 3 & 183373.5996 & 181977.3477 & 184805.7775 & 205554.4570 & 231.0000 \\ -Prosper & random & all & 10531 & 3 & 199419.0888 & 199419.0888 & 199419.0888 & 220259.8608 & 234.3333 \\ -Freddie FM48 & top\_net\_return & 1000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 5000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 10000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 25000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 50000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 100000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & top\_net\_return & 250000 & 1396053 & 1 & 1291227.5802 & 1291227.5802 & 1291227.5802 & 1305409.0553 & 146.0000 \\ -Freddie FM48 & random & 1000 & 1396053 & 3 & 1056743.4698 & 1054043.8341 & 1060065.9649 & 1075554.8613 & 113.0000 \\ -Freddie FM48 & random & 5000 & 1396053 & 3 & 1129835.6497 & 1124577.3552 & 1134306.8073 & 1147433.5407 & 121.6667 \\ -Freddie FM48 & random & 10000 & 1396053 & 3 & 1163040.3771 & 1160204.5486 & 1165078.1476 & 1179829.1563 & 114.6667 \\ -Freddie FM48 & random & 25000 & 1396053 & 3 & 1192925.9975 & 1189915.3242 & 1198143.7402 & 1209898.5778 & 116.0000 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.csv b/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.csv deleted file mode 100644 index ae599e2..0000000 --- a/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.csv +++ /dev/null @@ -1,8 +0,0 @@ -variant,color_group,window_months,n_rows,default_rate,auc_roc,pr_auc,coverage_90,alpha01_coverage,coverage90_pass,alpha01_pass,screening_robust_objective,recommendation -freddie_fm12_both,both,12,2849457,0.0059523621518064,0.7681863221355728,0.0222124268822717,0.7578453348849258,0.9774174647316434,False,False,1287040.6418207984,sensitivity only -freddie_fm24_both,both,24,2420190,0.0103363785487916,0.7800044357516288,0.0422297603093363,0.9369215610110188,0.9889295157422392,True,False,1118619.780863266,sensitivity only -freddie_fm36_both,both,36,3581402,0.0114142450358825,0.7890898429830087,0.0496332913708696,0.9056827296391612,0.9871002504662412,True,False,1042378.4068574588,sensitivity only -freddie_fm36_green,green,36,2383971,0.0047483799089837,0.7143282449429617,0.0122747793777575,0.8874942306290841,0.9873205421994056,False,False,1035323.9699910176,sensitivity only -freddie_fm36_red,red,36,1197431,0.0246853472141609,0.7047998404460994,0.0575720961155313,0.9082773067081432,0.9853776779890624,True,False,990317.6758330629,sensitivity only -freddie_fm48_both,both,48,3173355,0.0145435981792141,0.7838634726333499,0.0517822051269566,0.974523173547136,0.9907116706887202,True,True,1060065.9648987832,selected main external mortgage replication -freddie_fm60_both,both,60,3579151,0.0218331107013925,0.7270254323985861,0.0728705047312328,0.959101839256819,0.9822399292863524,True,False,1033755.5253137936,sensitivity only diff --git a/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.tex b/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.tex deleted file mode 100644 index 93fb7eb..0000000 --- a/reports/crpto/tables/crpto_tableA27_freddie_horizon_sensitivity.tex +++ /dev/null @@ -1,13 +0,0 @@ -\begin{tabular}{llrrrrrrrrrrl} -\toprule -variant & color\_group & window\_months & n\_rows & default\_rate & auc\_roc & pr\_auc & coverage\_90 & alpha01\_coverage & coverage90\_pass & alpha01\_pass & screening\_robust\_objective & recommendation \\ -\midrule -freddie\_fm12\_both & both & 12 & 2849457 & 0.0060 & 0.7682 & 0.0222 & 0.7578 & 0.9774 & False & False & 1287040.6418 & sensitivity only \\ -freddie\_fm24\_both & both & 24 & 2420190 & 0.0103 & 0.7800 & 0.0422 & 0.9369 & 0.9889 & True & False & 1118619.7809 & sensitivity only \\ -freddie\_fm36\_both & both & 36 & 3581402 & 0.0114 & 0.7891 & 0.0496 & 0.9057 & 0.9871 & True & False & 1042378.4069 & sensitivity only \\ -freddie\_fm36\_green & green & 36 & 2383971 & 0.0047 & 0.7143 & 0.0123 & 0.8875 & 0.9873 & False & False & 1035323.9700 & sensitivity only \\ -freddie\_fm36\_red & red & 36 & 1197431 & 0.0247 & 0.7048 & 0.0576 & 0.9083 & 0.9854 & True & False & 990317.6758 & sensitivity only \\ -freddie\_fm48\_both & both & 48 & 3173355 & 0.0145 & 0.7839 & 0.0518 & 0.9745 & 0.9907 & True & True & 1060065.9649 & selected main external mortgage replication \\ -freddie\_fm60\_both & both & 60 & 3579151 & 0.0218 & 0.7270 & 0.0729 & 0.9591 & 0.9822 & True & False & 1033755.5253 & sensitivity only \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv b/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv deleted file mode 100644 index 4cf635b..0000000 --- a/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv +++ /dev/null @@ -1,5 +0,0 @@ -dataset,candidate_cap,available_candidates,n_candidates,robust_objective,nonrobust_objective,price_of_robustness_pct,robust_objective_boot_low,robust_objective_boot_high,robust_n_funded,robust_max_funded_rank,solver_success -Prosper,all,10531,10531,199419.0887756553,220259.86083869892,0.0946190194785686,186546.04708826492,213330.6986532356,234,508,True -Freddie FM48,500000,1396053,500000,1291227.5801824057,1305409.0552784826,0.0108636254963404,1062741.9024937386,1535012.3146356042,143,551,True -Freddie FM48,1000000,1396053,1000000,1291227.5801824057,1305409.0552784826,0.0108636254963404,1062741.9024937386,1535012.3146356042,143,551,True -Freddie FM48,all,1396053,1396053,1291227.5801824057,1305409.0552784826,0.0108636254963404,1062741.9024937386,1535012.3146356042,143,551,True diff --git a/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.tex b/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.tex deleted file mode 100644 index d37cf9a..0000000 --- a/reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.tex +++ /dev/null @@ -1,10 +0,0 @@ -\begin{tabular}{llrrrrrrrrrr} -\toprule -dataset & candidate\_cap & available\_candidates & n\_candidates & robust\_objective & nonrobust\_objective & price\_of\_robustness\_pct & robust\_objective\_boot\_low & robust\_objective\_boot\_high & robust\_n\_funded & robust\_max\_funded\_rank & solver\_success \\ -\midrule -Prosper & all & 10531 & 10531 & 199419.0888 & 220259.8608 & 0.0946 & 186546.0471 & 213330.6987 & 234 & 508 & True \\ -Freddie FM48 & 500000 & 1396053 & 500000 & 1291227.5802 & 1305409.0553 & 0.0109 & 1062741.9025 & 1535012.3146 & 143 & 551 & True \\ -Freddie FM48 & 1000000 & 1396053 & 1000000 & 1291227.5802 & 1305409.0553 & 0.0109 & 1062741.9025 & 1535012.3146 & 143 & 551 & True \\ -Freddie FM48 & all & 1396053 & 1396053 & 1291227.5802 & 1305409.0553 & 0.0109 & 1062741.9025 & 1535012.3146 & 143 & 551 & True \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.csv b/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.csv deleted file mode 100644 index 59a9c6a..0000000 --- a/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.csv +++ /dev/null @@ -1,4 +0,0 @@ -scope,n_groups,min_coverage_90,min_coverage_alpha01,test_rows -all_groups,29,0.5,0.5,1396053 -eligible_cal_and_test_ge_500,25,0.8853734439834025,0.9565383149065956,1396010 -sparse_or_fallback_groups,4,0.5,0.5,43 diff --git a/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.tex b/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.tex deleted file mode 100644 index 3bdc00a..0000000 --- a/reports/crpto/tables/crpto_tableA29_freddie_mondrian_sparse_group_audit.tex +++ /dev/null @@ -1,9 +0,0 @@ -\begin{tabular}{lrrrr} -\toprule -scope & n\_groups & min\_coverage\_90 & min\_coverage\_alpha01 & test\_rows \\ -\midrule -all\_groups & 29 & 0.5000 & 0.5000 & 1396053 \\ -eligible\_cal\_and\_test\_ge\_500 & 25 & 0.8854 & 0.9565 & 1396010 \\ -sparse\_or\_fallback\_groups & 4 & 0.5000 & 0.5000 & 43 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA30_external_metric_intervals.csv b/reports/crpto/tables/crpto_tableA30_external_metric_intervals.csv deleted file mode 100644 index 955c7c1..0000000 --- a/reports/crpto/tables/crpto_tableA30_external_metric_intervals.csv +++ /dev/null @@ -1,9 +0,0 @@ -dataset,metric,estimate,ci_low,ci_high,method,n -Prosper,auc_roc,0.7072982821805249,0.6955733258816219,0.719023238479428,Hanley-McNeil large-sample approximation,10531 -Prosper,coverage_90,0.920425410692242,0.9151019142826232,0.9254422965261512,Wilson binomial interval,10531 -Prosper,coverage_alpha01,0.9943025353717596,0.9926737771462196,0.995570805432002,Wilson binomial interval,10531 -Prosper,robust_objective,199419.0887756553,186546.04708826492,213330.6986532356,Funded-loan contribution bootstrap; solver input uncertainty is not resampled,234 -Freddie FM48,auc_roc,0.7838634726333499,0.7799265563524038,0.7878003889142959,Hanley-McNeil large-sample approximation,1396053 -Freddie FM48,coverage_90,0.974523173547136,0.9742604888155064,0.974783246836056,Wilson binomial interval,1396053 -Freddie FM48,coverage_alpha01,0.9907116706887202,0.9905511894862964,0.9908694513583008,Wilson binomial interval,1396053 -Freddie FM48,robust_objective,1291227.5801824057,1062741.9024937386,1535012.3146356042,Funded-loan contribution bootstrap; solver input uncertainty is not resampled,143 diff --git a/reports/crpto/tables/crpto_tableA30_external_metric_intervals.tex b/reports/crpto/tables/crpto_tableA30_external_metric_intervals.tex deleted file mode 100644 index 543642f..0000000 --- a/reports/crpto/tables/crpto_tableA30_external_metric_intervals.tex +++ /dev/null @@ -1,14 +0,0 @@ -\begin{tabular}{llrrrlr} -\toprule -dataset & metric & estimate & ci\_low & ci\_high & method & n \\ -\midrule -Prosper & auc\_roc & 0.7073 & 0.6956 & 0.7190 & Hanley-McNeil large-sample approximation & 10531 \\ -Prosper & coverage\_90 & 0.9204 & 0.9151 & 0.9254 & Wilson binomial interval & 10531 \\ -Prosper & coverage\_alpha01 & 0.9943 & 0.9927 & 0.9956 & Wilson binomial interval & 10531 \\ -Prosper & robust\_objective & 199419.0888 & 186546.0471 & 213330.6987 & Funded-loan contribution bootstrap; solver input uncertainty is not resampled & 234 \\ -Freddie FM48 & auc\_roc & 0.7839 & 0.7799 & 0.7878 & Hanley-McNeil large-sample approximation & 1396053 \\ -Freddie FM48 & coverage\_90 & 0.9745 & 0.9743 & 0.9748 & Wilson binomial interval & 1396053 \\ -Freddie FM48 & coverage\_alpha01 & 0.9907 & 0.9906 & 0.9909 & Wilson binomial interval & 1396053 \\ -Freddie FM48 & robust\_objective & 1291227.5802 & 1062741.9025 & 1535012.3146 & Funded-loan contribution bootstrap; solver input uncertainty is not resampled & 143 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.csv b/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.csv deleted file mode 100644 index 715ce3f..0000000 --- a/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.csv +++ /dev/null @@ -1,7 +0,0 @@ -dataset,period,n,defaults,default_rate,auc_roc,pr_auc,coverage_90,coverage_alpha01,economic_candidates -Prosper,2012,8010,2577,0.3217228464419475,0.6988646610585748,0.4888868352024403,0.9116104868913858,0.9933832709113608,8010 -Prosper,2013,2521,316,0.1253470844902816,0.687683343379546,0.221384282793829,0.9484331614438716,0.9972233240777468,2521 -Freddie FM48,2015Q1,346682,3906,0.0112668093526632,0.7962807462700461,0.0507110650457581,0.9785480642202364,0.991992661863033,346682 -Freddie FM48,2015Q2,389075,4804,0.0123472338238128,0.7823109517507605,0.0491288479707842,0.9774413673456276,0.9910043050825676,389075 -Freddie FM48,2015Q3,345167,4975,0.0144133129760376,0.7736149116384292,0.0506352703987166,0.9707793618741072,0.990080164094482,345167 -Freddie FM48,2015Q4,315129,4867,0.0154444687731056,0.7791377151096768,0.0573174166539526,0.9705929952495644,0.9896328170368326,315129 diff --git a/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.tex b/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.tex deleted file mode 100644 index d9cf0dd..0000000 --- a/reports/crpto/tables/crpto_tableA31_external_subperiod_metrics.tex +++ /dev/null @@ -1,12 +0,0 @@ -\begin{tabular}{llrrrrrrrr} -\toprule -dataset & period & n & defaults & default\_rate & auc\_roc & pr\_auc & coverage\_90 & coverage\_alpha01 & economic\_candidates \\ -\midrule -Prosper & 2012 & 8010 & 2577 & 0.3217 & 0.6989 & 0.4889 & 0.9116 & 0.9934 & 8010 \\ -Prosper & 2013 & 2521 & 316 & 0.1253 & 0.6877 & 0.2214 & 0.9484 & 0.9972 & 2521 \\ -Freddie FM48 & 2015Q1 & 346682 & 3906 & 0.0113 & 0.7963 & 0.0507 & 0.9785 & 0.9920 & 346682 \\ -Freddie FM48 & 2015Q2 & 389075 & 4804 & 0.0123 & 0.7823 & 0.0491 & 0.9774 & 0.9910 & 389075 \\ -Freddie FM48 & 2015Q3 & 345167 & 4975 & 0.0144 & 0.7736 & 0.0506 & 0.9708 & 0.9901 & 345167 \\ -Freddie FM48 & 2015Q4 & 315129 & 4867 & 0.0154 & 0.7791 & 0.0573 & 0.9706 & 0.9896 & 315129 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.csv b/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.csv deleted file mode 100644 index 11dcfee..0000000 --- a/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.csv +++ /dev/null @@ -1,4 +0,0 @@ -variant,rows_total,default_rate,auc_roc,coverage_90,min_group_coverage_90,alpha01_coverage,coverage90_pass,alpha01_pass,available_oot_candidates,robust_objective_all_candidates,price_of_robustness_pct -main_chargedoff_or_defaulted,54807,0.3091758352035324,0.7074290432845566,0.9205159821682634,0.8850308641975309,0.9943090202029784,True,True,10531,199419.0887756553,0.0946190194785686 -defaulted_only,42862,0.1166534459427931,0.6796193367985865,0.9148300336197236,0.8671328671328671,0.9947702652222636,True,True,8019,272969.2286232216,0.0338142847144336 -chargedoff_only,49807,0.2398257273074066,0.7131638982556929,0.9035622908876204,0.8355855855855856,0.99212753395001,True,True,10150,205828.04819793423,0.0942093080550645 diff --git a/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.tex b/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.tex deleted file mode 100644 index 3fe28af..0000000 --- a/reports/crpto/tables/crpto_tableA32_prosper_default_definition_sensitivity.tex +++ /dev/null @@ -1,9 +0,0 @@ -\begin{tabular}{lrrrrrrrrrrr} -\toprule -variant & rows\_total & default\_rate & auc\_roc & coverage\_90 & min\_group\_coverage\_90 & alpha01\_coverage & coverage90\_pass & alpha01\_pass & available\_oot\_candidates & robust\_objective\_all\_candidates & price\_of\_robustness\_pct \\ -\midrule -main\_chargedoff\_or\_defaulted & 54807 & 0.3092 & 0.7074 & 0.9205 & 0.8850 & 0.9943 & True & True & 10531 & 199419.0888 & 0.0946 \\ -defaulted\_only & 42862 & 0.1167 & 0.6796 & 0.9148 & 0.8671 & 0.9948 & True & True & 8019 & 272969.2286 & 0.0338 \\ -chargedoff\_only & 49807 & 0.2398 & 0.7132 & 0.9036 & 0.8356 & 0.9921 & True & True & 10150 & 205828.0482 & 0.0942 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv b/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv deleted file mode 100644 index 596121e..0000000 --- a/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv +++ /dev/null @@ -1,4 +0,0 @@ -segment,rows_total,default_rate,auc_roc,coverage_90,min_group_coverage_90,alpha01_coverage,coverage90_pass,alpha01_pass,available_oot_candidates,robust_objective_all_candidates,price_of_robustness_pct,all_lp_solved,max_funded_rank,funded_outside_top250k -both,3173355,0.0145435981792141,0.7838634726333499,0.974523173547136,0.5,0.9907116706887202,True,True,1396053,1291227.5801824057,0.0108636254963404,True,551,0 -red,1158306,0.0297209890995989,0.700087721623812,0.970528929740982,0.891597510373444,0.9850218288730144,True,False,597603,1236591.1430666302,0.0236887045711261,True,155,0 -green,2015049,0.0058192133293036,0.6997060916716304,0.98100319368777,0.5,0.9950629344354688,True,True,798450,1287497.9769414272,0.0100393498751964,True,745,0 diff --git a/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.tex b/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.tex deleted file mode 100644 index 85424d9..0000000 --- a/reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.tex +++ /dev/null @@ -1,9 +0,0 @@ -\begin{tabular}{lrrrrrrrrrrrrrr} -\toprule -segment & rows\_total & default\_rate & auc\_roc & coverage\_90 & min\_group\_coverage\_90 & alpha01\_coverage & coverage90\_pass & alpha01\_pass & available\_oot\_candidates & robust\_objective\_all\_candidates & price\_of\_robustness\_pct & all\_lp\_solved & max\_funded\_rank & funded\_outside\_top250k \\ -\midrule -both & 3173355 & 0.0145 & 0.7839 & 0.9745 & 0.5000 & 0.9907 & True & True & 1396053 & 1291227.5802 & 0.0109 & True & 551 & 0 \\ -red & 1158306 & 0.0297 & 0.7001 & 0.9705 & 0.8916 & 0.9850 & True & False & 597603 & 1236591.1431 & 0.0237 & True & 155 & 0 \\ -green & 2015049 & 0.0058 & 0.6997 & 0.9810 & 0.5000 & 0.9951 & True & True & 798450 & 1287497.9769 & 0.0100 & True & 745 & 0 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.csv b/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.csv deleted file mode 100644 index 86eacf8..0000000 --- a/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.csv +++ /dev/null @@ -1,5 +0,0 @@ -application,panel_default_rate,auc_roc,price_of_robustness_pct,protocol -Freddie FM48 (green),0.005819,0.6997,0.010039,frozen_application -Freddie FM48 (combined),0.014544,0.7839,0.010864,frozen_application -Freddie FM48 (red),0.029721,0.7001,0.023689,frozen_application -Prosper final-status,0.309176,0.7074,0.094619,frozen_application diff --git a/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.tex b/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.tex deleted file mode 100644 index dbcf1f7..0000000 --- a/reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.tex +++ /dev/null @@ -1,10 +0,0 @@ -\begin{tabular}{lrrrl} -\toprule -application & panel\_default\_rate & auc\_roc & price\_of\_robustness\_pct & protocol \\ -\midrule -Freddie FM48 (green) & 0.0058 & 0.6997 & 0.0100 & frozen\_application \\ -Freddie FM48 (combined) & 0.0145 & 0.7839 & 0.0109 & frozen\_application \\ -Freddie FM48 (red) & 0.0297 & 0.7001 & 0.0237 & frozen\_application \\ -Prosper final-status & 0.3092 & 0.7074 & 0.0946 & frozen\_application \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv deleted file mode 100644 index 004b318..0000000 --- a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv +++ /dev/null @@ -1,9 +0,0 @@ -selected_for_policy,target_alpha,used_alpha,target_coverage,empirical_coverage,coverage_gap,avg_width,min_partition_coverage,min_grade_coverage,high_endpoint_at_one_rate -False,0.01,0.0095,0.99,0.9967204706919157,0.006720470691915725,0.9882149910656284,0.989628743645835,0.9722703639514731,0.9354243342519386 -False,0.03,0.028499999999999998,0.97,0.9884783056246818,0.01847830562468178,0.9697789837688854,0.9697990273996318,0.9664933564413634,0.8323141991338864 -False,0.05,0.0475,0.95,0.9728246932664907,0.022824693266490725,0.9535799768723978,0.9505830880848586,0.9525750124329999,0.7181555175913519 -False,0.07,0.0665,0.9299999999999999,0.9552712654721186,0.025271265472118665,0.795666199875876,0.9347822665680426,0.938442835829143,0.6135500904760013 -True,0.1,0.095,0.9,0.9348356081757077,0.03483560817570763,0.7888790790793794,0.9263099219620958,0.9267968822615554,0.5178730735474177 -False,0.12,0.11399999999999999,0.88,0.9182645944471934,0.038264594447193434,0.7831737841625059,0.906874767744333,0.9057375416605808,0.4545109781160043 -False,0.15,0.1425,0.85,0.8860977574231856,0.03609775742318566,0.6461762449744832,0.8635623982976592,0.8631010078209027,0.33485872380078663 -False,0.2,0.19,0.8,0.8493800317117481,0.049380031711748096,0.6365853175970958,0.8293756967670011,0.828006532910199,0.24466805601204902 diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex deleted file mode 100644 index 8af704c..0000000 --- a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex +++ /dev/null @@ -1,14 +0,0 @@ -\begin{tabular}{rrrrrrrrrr} -\toprule -selected\_for\_policy & target\_alpha & used\_alpha & target\_coverage & empirical\_coverage & coverage\_gap & avg\_width & min\_partition\_coverage & min\_grade\_coverage & high\_endpoint\_at\_one\_rate \\ -\midrule -False & 0.010000 & 0.009500 & 0.990000 & 0.996720 & 0.006720 & 0.988215 & 0.989629 & 0.972270 & 0.935424 \\ -False & 0.030000 & 0.028500 & 0.970000 & 0.988478 & 0.018478 & 0.969779 & 0.969799 & 0.966493 & 0.832314 \\ -False & 0.050000 & 0.047500 & 0.950000 & 0.972825 & 0.022825 & 0.953580 & 0.950583 & 0.952575 & 0.718156 \\ -False & 0.070000 & 0.066500 & 0.930000 & 0.955271 & 0.025271 & 0.795666 & 0.934782 & 0.938443 & 0.613550 \\ -True & 0.100000 & 0.095000 & 0.900000 & 0.934836 & 0.034836 & 0.788879 & 0.926310 & 0.926797 & 0.517873 \\ -False & 0.120000 & 0.114000 & 0.880000 & 0.918265 & 0.038265 & 0.783174 & 0.906875 & 0.905738 & 0.454511 \\ -False & 0.150000 & 0.142500 & 0.850000 & 0.886098 & 0.036098 & 0.646176 & 0.863562 & 0.863101 & 0.334859 \\ -False & 0.200000 & 0.190000 & 0.800000 & 0.849380 & 0.049380 & 0.636585 & 0.829376 & 0.828007 & 0.244668 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv deleted file mode 100644 index 1e270cf..0000000 --- a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv +++ /dev/null @@ -1,10 +0,0 @@ -selected,eligible,audit_selected,audit_eligible,candidate_id,risk_tolerance,gamma,expected_objective,n_funded,weighted_pd_point,weighted_pd_effective,endpoint_budget,audit_expected_objective,audit_n_funded,audit_endpoint_budget -True,True,True,True,linear-005,0.17,0.5,99387.12330098984,162,0.08096395060564901,0.17,0.25903604939435104,103567.0099590662,193,0.2620820323280323 -False,True,False,True,linear-009,0.19,0.75,96446.15634571345,165,0.07983135268116329,0.19,0.22672288243961225,100439.75691860713,192,0.22857742256378005 -False,True,False,True,linear-002,0.15,0.5,95995.21888305439,166,0.07391185908208518,0.15000000000000002,0.2260881409179148,100253.3313725306,190,0.22769467893384143 -False,True,False,True,linear-006,0.17,0.75,93760.12764708955,168,0.0732219020269643,0.17,0.20225936599101194,97897.74942687502,191,0.2035038230017554 -False,True,False,True,linear-003,0.15,0.75,90893.16015692815,167,0.06715742012587558,0.15,0.17761419329137482,95103.61442940257,186,0.17820330775871246 -False,False,False,False,linear-007,0.19,0.25,113986.45107415256,170,0.11259889470623649,0.19,0.42220331588129056,118537.51039714873,194,0.4463778904768255 -False,False,False,False,linear-004,0.17,0.25,109885.56215750676,166,0.10163175926165047,0.16999999999999998,0.3751047222150486,114373.21276113224,196,0.396751177625259 -False,False,False,False,linear-001,0.15,0.25,105398.22903742193,166,0.09052889811149456,0.15,0.3284133056655163,109967.55921988936,194,0.34350244765822646 -False,False,False,False,linear-008,0.19,0.5,102671.8741997059,163,0.08950921111283665,0.19,0.29049078888716334,106734.80147640246,191,0.29486067740758526 diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex deleted file mode 100644 index 1146224..0000000 --- a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex +++ /dev/null @@ -1,15 +0,0 @@ -\begin{tabular}{rrrrlrrrrrrrrrr} -\toprule -selected & eligible & audit\_selected & audit\_eligible & candidate\_id & risk\_tolerance & gamma & expected\_objective & n\_funded & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & audit\_expected\_objective & audit\_n\_funded & audit\_endpoint\_budget \\ -\midrule -True & True & True & True & linear-005 & 0.170000 & 0.500000 & 99387.123301 & 162 & 0.080964 & 0.170000 & 0.259036 & 103567.009959 & 193 & 0.262082 \\ -False & True & False & True & linear-009 & 0.190000 & 0.750000 & 96446.156346 & 165 & 0.079831 & 0.190000 & 0.226723 & 100439.756919 & 192 & 0.228577 \\ -False & True & False & True & linear-002 & 0.150000 & 0.500000 & 95995.218883 & 166 & 0.073912 & 0.150000 & 0.226088 & 100253.331373 & 190 & 0.227695 \\ -False & True & False & True & linear-006 & 0.170000 & 0.750000 & 93760.127647 & 168 & 0.073222 & 0.170000 & 0.202259 & 97897.749427 & 191 & 0.203504 \\ -False & True & False & True & linear-003 & 0.150000 & 0.750000 & 90893.160157 & 167 & 0.067157 & 0.150000 & 0.177614 & 95103.614429 & 186 & 0.178203 \\ -False & False & False & False & linear-007 & 0.190000 & 0.250000 & 113986.451074 & 170 & 0.112599 & 0.190000 & 0.422203 & 118537.510397 & 194 & 0.446378 \\ -False & False & False & False & linear-004 & 0.170000 & 0.250000 & 109885.562158 & 166 & 0.101632 & 0.170000 & 0.375105 & 114373.212761 & 196 & 0.396751 \\ -False & False & False & False & linear-001 & 0.150000 & 0.250000 & 105398.229037 & 166 & 0.090529 & 0.150000 & 0.328413 & 109967.559220 & 194 & 0.343502 \\ -False & False & False & False & linear-008 & 0.190000 & 0.500000 & 102671.874200 & 163 & 0.089509 & 0.190000 & 0.290491 & 106734.801476 & 191 & 0.294861 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv deleted file mode 100644 index 9ed9783..0000000 --- a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv +++ /dev/null @@ -1,19 +0,0 @@ -period,policy,risk_tolerance,gamma,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,endpoint_budget,markov_loss_threshold -full_oot,Calibration-selected 50/50 CRPTO,0.17,0.5,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.0819489105765324,0.17,0.2580510894234676,0.5742788554403055 -full_oot,More-conservative 75% blend,0.17,0.75,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.0788113673633701,0.1699999999999999,0.2003962108788766,0.5166239768957146 -full_oot,Point-PD matched-tau,0.17,0.0,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.1538437011518627,0.1538437011518627,0.9213174385285344,1.2375452045453723 -2018H1,Calibration-selected 50/50 CRPTO,0.17,0.5,260,123713.651457411,92530.7342546382,0.106703492661675,0.0974284926616751,0.0808296789072766,0.17,0.2591703210927234,0.5753980871095614 -2018H1,More-conservative 75% blend,0.17,0.75,263,116631.43710632224,75935.74005281724,0.12065,0.11815,0.0758885509922111,0.17,0.2013704830025962,0.5175982490194342 -2018H1,Point-PD matched-tau,0.17,0.0,211,176503.44846593234,118101.99096593227,0.190825,0.076425,0.17,0.17,0.9389119545998964,1.2551397206167343 -2018H2,Calibration-selected 50/50 CRPTO,0.17,0.5,289,138315.09206362677,156185.50718903824,0.026725,0.026725,0.0775592058342477,0.1699999999999998,0.2624407941657518,0.5786685601825898 -2018H2,More-conservative 75% blend,0.17,0.75,286,131160.69145112435,145764.2646632018,0.031025,0.031025,0.0752711349157277,0.1699999999999999,0.2015762883614241,0.517804054378262 -2018H2,Point-PD matched-tau,0.17,0.0,256,192790.6360653248,95603.57631688572,0.2367281267296992,0.113175,0.17,0.17,0.9155458351509776,1.2317736011678155 -2019H1,Calibration-selected 50/50 CRPTO,0.17,0.5,244,137182.83555860247,123590.68683599537,0.077325,0.0617,0.0781281806164286,0.17,0.2618718193835714,0.5780995854004094 -2019H1,More-conservative 75% blend,0.17,0.75,241,129155.4740135068,117840.14770205291,0.0714,0.0608,0.0725032304189913,0.1699999999999999,0.2024989231936695,0.5187266892105076 -2019H1,Point-PD matched-tau,0.17,0.0,203,191975.2452370317,144281.46023703172,0.170275,0.038275,0.17,0.17,0.8848375856531457,1.2010653516699836 -2019H2,Calibration-selected 50/50 CRPTO,0.17,0.5,219,136249.87920464447,110251.95086330731,0.10325,0.10325,0.0784966036859173,0.1699999999999999,0.2615033963140826,0.5777311623309205 -2019H2,More-conservative 75% blend,0.17,0.75,220,127777.26882054852,99327.40175400236,0.10325,0.10325,0.0730478509632306,0.17,0.2023173830122564,0.5185451490290944 -2019H2,Point-PD matched-tau,0.17,0.0,129,197742.1451160193,256966.1951160193,0.023775,0.0079,0.1699999999999999,0.1699999999999999,0.8988048868151989,1.2150326528320368 -2020+,Calibration-selected 50/50 CRPTO,0.17,0.5,166,113753.93195239925,99689.53961659266,0.0837749999999999,0.0837749999999999,0.075537928142652,0.17,0.264462071857348,0.580689837874186 -2020+,More-conservative 75% blend,0.17,0.75,169,106744.29312891632,83162.7196944619,0.093775,0.093775,0.0673508590345457,0.17,0.2042163803218181,0.520444146338656 -2020+,Point-PD matched-tau,0.17,0.0,159,154068.98539948836,218629.1353994884,0.0169,0.0,0.17,0.17,0.861424853142836,1.177652619159674 diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex deleted file mode 100644 index 29b5993..0000000 --- a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex +++ /dev/null @@ -1,24 +0,0 @@ -\begin{tabular}{llrrrrrrrrrrr} -\toprule -period & policy & risk\_tolerance & gamma & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & markov\_loss\_threshold \\ -\midrule -full\_oot & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.081949 & 0.170000 & 0.258051 & 0.574279 \\ -full\_oot & More-conservative 75\% blend & 0.170000 & 0.750000 & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.078811 & 0.170000 & 0.200396 & 0.516624 \\ -full\_oot & Point-PD matched-tau & 0.170000 & 0.000000 & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.153844 & 0.153844 & 0.921317 & 1.237545 \\ -2018H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 260 & 123713.651457 & 92530.734255 & 0.106703 & 0.097428 & 0.080830 & 0.170000 & 0.259170 & 0.575398 \\ -2018H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 263 & 116631.437106 & 75935.740053 & 0.120650 & 0.118150 & 0.075889 & 0.170000 & 0.201370 & 0.517598 \\ -2018H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 211 & 176503.448466 & 118101.990966 & 0.190825 & 0.076425 & 0.170000 & 0.170000 & 0.938912 & 1.255140 \\ -2018H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 289 & 138315.092064 & 156185.507189 & 0.026725 & 0.026725 & 0.077559 & 0.170000 & 0.262441 & 0.578669 \\ -2018H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 286 & 131160.691451 & 145764.264663 & 0.031025 & 0.031025 & 0.075271 & 0.170000 & 0.201576 & 0.517804 \\ -2018H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 256 & 192790.636065 & 95603.576317 & 0.236728 & 0.113175 & 0.170000 & 0.170000 & 0.915546 & 1.231774 \\ -2019H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 244 & 137182.835559 & 123590.686836 & 0.077325 & 0.061700 & 0.078128 & 0.170000 & 0.261872 & 0.578100 \\ -2019H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 241 & 129155.474014 & 117840.147702 & 0.071400 & 0.060800 & 0.072503 & 0.170000 & 0.202499 & 0.518727 \\ -2019H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 203 & 191975.245237 & 144281.460237 & 0.170275 & 0.038275 & 0.170000 & 0.170000 & 0.884838 & 1.201065 \\ -2019H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 219 & 136249.879205 & 110251.950863 & 0.103250 & 0.103250 & 0.078497 & 0.170000 & 0.261503 & 0.577731 \\ -2019H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 220 & 127777.268821 & 99327.401754 & 0.103250 & 0.103250 & 0.073048 & 0.170000 & 0.202317 & 0.518545 \\ -2019H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 129 & 197742.145116 & 256966.195116 & 0.023775 & 0.007900 & 0.170000 & 0.170000 & 0.898805 & 1.215033 \\ -2020+ & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 166 & 113753.931952 & 99689.539617 & 0.083775 & 0.083775 & 0.075538 & 0.170000 & 0.264462 & 0.580690 \\ -2020+ & More-conservative 75\% blend & 0.170000 & 0.750000 & 169 & 106744.293129 & 83162.719694 & 0.093775 & 0.093775 & 0.067351 & 0.170000 & 0.204216 & 0.520444 \\ -2020+ & Point-PD matched-tau & 0.170000 & 0.000000 & 159 & 154068.985399 & 218629.135399 & 0.016900 & 0.000000 & 0.170000 & 0.170000 & 0.861425 & 1.177653 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv deleted file mode 100644 index b3a4aa2..0000000 --- a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv +++ /dev/null @@ -1,7 +0,0 @@ -grade,n_funded,exposure,exposure_share,weighted_default_rate,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,weighted_pd_high,realized_return -B,1,4000.0,0.004,0.0,0.0,0.06207486505052294,0.10147180683774551,0.14086874862496807,643.1999999999999 -C,91,313565.9904450649,0.31356599044506495,0.028622364266166715,0.028622364266166715,0.06780118581238306,0.10859786707930257,0.14939454834622204,47303.7528804385 -D,170,586834.0095549352,0.5868340095549351,0.045839197391441946,0.045839197391441946,0.08832773498776167,0.19492559567178316,0.30152345635580474,109618.0772518213 -E,33,80600.0,0.0806,0.0,0.0,0.08524228589130531,0.17955099586752693,0.27385970584374847,19879.705 -F,11,13000.0,0.013000000000000001,0.2692307692307692,0.07692307692307691,0.11097039235072648,0.4256872635040547,0.7404041346573831,1266.55 -G,2,2000.0,0.002,0.0,0.0,0.14680145627400495,0.5734007281370025,1.0,616.3000000000001 diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex deleted file mode 100644 index f08cf37..0000000 --- a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex +++ /dev/null @@ -1,12 +0,0 @@ -\begin{tabular}{lrrrrrrrrr} -\toprule -grade & n\_funded & exposure & exposure\_share & weighted\_default\_rate & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & weighted\_pd\_high & realized\_return \\ -\midrule -B & 1 & 4000.000000 & 0.004000 & 0.000000 & 0.000000 & 0.062075 & 0.101472 & 0.140869 & 643.200000 \\ -C & 91 & 313565.990445 & 0.313566 & 0.028622 & 0.028622 & 0.067801 & 0.108598 & 0.149395 & 47303.752880 \\ -D & 170 & 586834.009555 & 0.586834 & 0.045839 & 0.045839 & 0.088328 & 0.194926 & 0.301523 & 109618.077252 \\ -E & 33 & 80600.000000 & 0.080600 & 0.000000 & 0.000000 & 0.085242 & 0.179551 & 0.273860 & 19879.705000 \\ -F & 11 & 13000.000000 & 0.013000 & 0.269231 & 0.076923 & 0.110970 & 0.425687 & 0.740404 & 1266.550000 \\ -G & 2 & 2000.000000 & 0.002000 & 0.000000 & 0.000000 & 0.146801 & 0.573401 & 1.000000 & 616.300000 \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv deleted file mode 100644 index ebd91b3..0000000 --- a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv +++ /dev/null @@ -1,11 +0,0 @@ -bootstrap_unit,metric,observed,boot_mean,boot_p025,boot_p50,boot_p975,n_units,n_draws,seed,note -origination_month,realized_return,179327.5851322598,179293.01466566528,163421.1394838545,179487.08382041534,193551.6460089551,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." -origination_month,weighted_default_rate,0.039375,0.03943546399994348,0.02051693505224659,0.03875339983492982,0.06262154165393485,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." -origination_month,weighted_miscoverage,0.036875,0.03691656808682405,0.01833810380778745,0.03613937487855996,0.060083696168230914,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." -origination_month,Gamma_CP,0.1761021788469351,0.17637453569205386,0.1321447744854007,0.1746349832245928,0.2308148609432896,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." -origination_month,endpoint_budget,0.2580510894234676,0.2583077550716801,0.21199608169774178,0.25666011731200084,0.3148701678446982,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." -funded_loan,realized_return,179327.5851322598,179075.4838443055,162706.17200230644,179416.98769601624,193924.73990027257,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -funded_loan,weighted_default_rate,0.039375,0.03963778218343555,0.020869102395698755,0.039043420065914713,0.0617419907402632,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -funded_loan,weighted_miscoverage,0.036875,0.03720263786105997,0.019357859801356386,0.03671487666893709,0.058812406466411934,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -funded_loan,Gamma_CP,0.1761021788469351,0.17602573881357803,0.13715911012280738,0.17480824046170865,0.2243084301758857,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -funded_loan,endpoint_budget,0.2580510894234676,0.25794196914296347,0.21730801774100114,0.2567618086877488,0.30857101650304447,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex deleted file mode 100644 index dda735b..0000000 --- a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex +++ /dev/null @@ -1,16 +0,0 @@ -\begin{tabular}{llrrrrrrrrl} -\toprule -bootstrap\_unit & metric & observed & boot\_mean & boot\_p025 & boot\_p50 & boot\_p975 & n\_units & n\_draws & seed & note \\ -\midrule -origination\_month & realized\_return & 179327.585132 & 179293.014666 & 163421.139484 & 179487.083820 & 193551.646009 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ -origination\_month & weighted\_default\_rate & 0.039375 & 0.039435 & 0.020517 & 0.038753 & 0.062622 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ -origination\_month & weighted\_miscoverage & 0.036875 & 0.036917 & 0.018338 & 0.036139 & 0.060084 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ -origination\_month & Gamma\_CP & 0.176102 & 0.176375 & 0.132145 & 0.174635 & 0.230815 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ -origination\_month & endpoint\_budget & 0.258051 & 0.258308 & 0.211996 & 0.256660 & 0.314870 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ -funded\_loan & realized\_return & 179327.585132 & 179075.483844 & 162706.172002 & 179416.987696 & 193924.739900 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -funded\_loan & weighted\_default\_rate & 0.039375 & 0.039638 & 0.020869 & 0.039043 & 0.061742 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -funded\_loan & weighted\_miscoverage & 0.036875 & 0.037203 & 0.019358 & 0.036715 & 0.058812 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -funded\_loan & Gamma\_CP & 0.176102 & 0.176026 & 0.137159 & 0.174808 & 0.224308 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -funded\_loan & endpoint\_budget & 0.258051 & 0.257942 & 0.217308 & 0.256762 & 0.308571 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -\bottomrule -\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv deleted file mode 100644 index e07feb7..0000000 --- a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv +++ /dev/null @@ -1,4 +0,0 @@ -policy,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,endpoint_budget,markov_loss_threshold,return_delta_vs_selected,default_delta_vs_selected,threshold_delta_vs_selected -Calibration-selected 50/50 CRPTO,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.2580510894234676,0.5742788554403055,0.0,0.0,0.0 -More-conservative 75% blend,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.2003962108788766,0.5166239768957146,-6388.080277992645,-0.003500000000000003,-0.05765487854459095 -Point-PD matched-tau,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.9213174385285344,1.2375452045453723,17041.554867740255,0.07902500000000001,0.6632663491050668 diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex deleted file mode 100644 index 7653717..0000000 --- a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex +++ /dev/null @@ -1,9 +0,0 @@ -\begin{tabular}{lrrrrrrrrrr} -\toprule -policy & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & endpoint\_budget & markov\_loss\_threshold & return\_delta\_vs\_selected & default\_delta\_vs\_selected & threshold\_delta\_vs\_selected \\ -\midrule -Calibration-selected 50/50 CRPTO & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.258051 & 0.574279 & 0.000000 & 0.000000 & 0.000000 \\ -More-conservative 75\% blend & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.200396 & 0.516624 & -6388.080278 & -0.003500 & -0.057655 \\ -Point-PD matched-tau & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.921317 & 1.237545 & 17041.554868 & 0.079025 & 0.663266 \\ -\bottomrule -\end{tabular} diff --git a/reports/mrm/.gitignore b/reports/mrm/.gitignore deleted file mode 100644 index b28a470..0000000 --- a/reports/mrm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/mrm_validation_report.json diff --git a/reports/mrm/corepd_model_card.html b/reports/mrm/corepd_model_card.html deleted file mode 100644 index 57e2b15..0000000 --- a/reports/mrm/corepd_model_card.html +++ /dev/null @@ -1,190 +0,0 @@ -
-# Model description
-
-[More Information Needed]
-
-## Intended uses & limitations
-
-[More Information Needed]
-
-## Training Procedure
-
-[More Information Needed]
-
-### Hyperparameters
-
-<details>
-<summary> Click to expand </summary>
-
-|   Hyperparameter  |   Value    |
-| :---------------: | :--------: |
-|         C         |    1.0     |
-|    class_weight   |  balanced  |
-|        dual       |   False    |
-|   fit_intercept   |    True    |
-| intercept_scaling |     1      |
-|      l1_ratio     |    0.0     |
-|      max_iter     |    1000    |
-|       n_jobs      |    None    |
-|      penalty      | deprecated |
-|    random_state   |    None    |
-|       solver      |   lbfgs    |
-|        tol        |   0.0001   |
-|      verbose      |     0      |
-|     warm_start    |   False    |
-
-</details>
-
-### Model Plot
-
-<style>#sk-container-id-1 {/* Definition of color scheme common for light and dark mode */--sklearn-color-text: #000;--sklearn-color-text-muted: #666;--sklearn-color-line: gray;/* Definition of color scheme for unfitted estimators */--sklearn-color-unfitted-level-0: #fff5e6;--sklearn-color-unfitted-level-1: #f6e4d2;--sklearn-color-unfitted-level-2: #ffe0b3;--sklearn-color-unfitted-level-3: chocolate;/* Definition of color scheme for fitted estimators */--sklearn-color-fitted-level-0: #f0f8ff;--sklearn-color-fitted-level-1: #d4ebff;--sklearn-color-fitted-level-2: #b3dbfd;--sklearn-color-fitted-level-3: cornflowerblue;
-}#sk-container-id-1.light {/* Specific color for light theme */--sklearn-color-text-on-default-background: black;--sklearn-color-background: white;--sklearn-color-border-box: black;--sklearn-color-icon: #696969;
-}#sk-container-id-1.dark {--sklearn-color-text-on-default-background: white;--sklearn-color-background: #111;--sklearn-color-border-box: white;--sklearn-color-icon: #878787;
-}#sk-container-id-1 {color: var(--sklearn-color-text);
-}#sk-container-id-1 pre {padding: 0;
-}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;
-}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed var(--sklearn-color-line);margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: var(--sklearn-color-background);
-}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }`but bootstrap.min.css set `[hidden] { display: none !important; }`so we also need the `!important` here to be able to override thedefault hidden behavior on the sphinx rendered scikit-learn.org.See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;
-}#sk-container-id-1 div.sk-text-repr-fallback {display: none;
-}div.sk-parallel-item,
-div.sk-serial,
-div.sk-item {/* draw centered vertical line to link estimators */background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));background-size: 2px 100%;background-repeat: no-repeat;background-position: center center;
-}/* Parallel-specific style estimator block */#sk-container-id-1 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 2px solid var(--sklearn-color-text-on-default-background);flex-grow: 1;
-}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: var(--sklearn-color-background);position: relative;
-}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;
-}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;
-}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;
-}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;
-}/* Serial-specific style estimator block */#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: var(--sklearn-color-background);padding-right: 1em;padding-left: 1em;
-}/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is
-clickable and can be expanded/collapsed.
-- Pipeline and ColumnTransformer use this feature and define the default style
-- Estimators will overwrite some part of the style using the `sk-estimator` class
-*//* Pipeline and ColumnTransformer style (default) */#sk-container-id-1 div.sk-toggleable {/* Default theme specific background. It is overwritten whether we have aspecific estimator or a Pipeline/ColumnTransformer */background-color: var(--sklearn-color-background);
-}/* Toggleable label */
-#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: flex;width: 100%;margin-bottom: 0;padding: 0.5em;box-sizing: border-box;text-align: center;align-items: center;justify-content: center;gap: 0.5em;
-}#sk-container-id-1 label.sk-toggleable__label .caption {font-size: 0.6rem;font-weight: lighter;color: var(--sklearn-color-text-muted);
-}#sk-container-id-1 label.sk-toggleable__label-arrow:before {/* Arrow on the left of the label */content: "▸";float: left;margin-right: 0.25em;color: var(--sklearn-color-icon);
-}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: var(--sklearn-color-text);
-}/* Toggleable content - dropdown */#sk-container-id-1 div.sk-toggleable__content {display: none;text-align: left;/* unfitted */background-color: var(--sklearn-color-unfitted-level-0);
-}#sk-container-id-1 div.sk-toggleable__content.fitted {/* fitted */background-color: var(--sklearn-color-fitted-level-0);
-}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;border-radius: 0.25em;color: var(--sklearn-color-text);/* unfitted */background-color: var(--sklearn-color-unfitted-level-0);
-}#sk-container-id-1 div.sk-toggleable__content.fitted pre {/* unfitted */background-color: var(--sklearn-color-fitted-level-0);
-}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {/* Expand drop-down */display: block;width: 100%;overflow: visible;
-}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";
-}/* Pipeline/ColumnTransformer-specific style */#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {color: var(--sklearn-color-text);background-color: var(--sklearn-color-unfitted-level-2);
-}#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: var(--sklearn-color-fitted-level-2);
-}/* Estimator-specific style *//* Colorize estimator box */
-#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {/* unfitted */background-color: var(--sklearn-color-unfitted-level-2);
-}#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {/* fitted */background-color: var(--sklearn-color-fitted-level-2);
-}#sk-container-id-1 div.sk-label label.sk-toggleable__label,
-#sk-container-id-1 div.sk-label label {/* The background is the default theme color */color: var(--sklearn-color-text-on-default-background);
-}/* On hover, darken the color of the background */
-#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {color: var(--sklearn-color-text);background-color: var(--sklearn-color-unfitted-level-2);
-}/* Label box, darken color on hover, fitted */
-#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {color: var(--sklearn-color-text);background-color: var(--sklearn-color-fitted-level-2);
-}/* Estimator label */#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;line-height: 1.2em;
-}#sk-container-id-1 div.sk-label-container {text-align: center;
-}/* Estimator-specific */
-#sk-container-id-1 div.sk-estimator {font-family: monospace;border: 1px dotted var(--sklearn-color-border-box);border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;/* unfitted */background-color: var(--sklearn-color-unfitted-level-0);
-}#sk-container-id-1 div.sk-estimator.fitted {/* fitted */background-color: var(--sklearn-color-fitted-level-0);
-}/* on hover */
-#sk-container-id-1 div.sk-estimator:hover {/* unfitted */background-color: var(--sklearn-color-unfitted-level-2);
-}#sk-container-id-1 div.sk-estimator.fitted:hover {/* fitted */background-color: var(--sklearn-color-fitted-level-2);
-}/* Specification for estimator info (e.g. "i" and "?") *//* Common style for "i" and "?" */.sk-estimator-doc-link,
-a:link.sk-estimator-doc-link,
-a:visited.sk-estimator-doc-link {float: right;font-size: smaller;line-height: 1em;font-family: monospace;background-color: var(--sklearn-color-unfitted-level-0);border-radius: 1em;height: 1em;width: 1em;text-decoration: none !important;margin-left: 0.5em;text-align: center;/* unfitted */border: var(--sklearn-color-unfitted-level-3) 1pt solid;color: var(--sklearn-color-unfitted-level-3);
-}.sk-estimator-doc-link.fitted,
-a:link.sk-estimator-doc-link.fitted,
-a:visited.sk-estimator-doc-link.fitted {/* fitted */background-color: var(--sklearn-color-fitted-level-0);border: var(--sklearn-color-fitted-level-3) 1pt solid;color: var(--sklearn-color-fitted-level-3);
-}/* On hover */
-div.sk-estimator:hover .sk-estimator-doc-link:hover,
-.sk-estimator-doc-link:hover,
-div.sk-label-container:hover .sk-estimator-doc-link:hover,
-.sk-estimator-doc-link:hover {/* unfitted */background-color: var(--sklearn-color-unfitted-level-3);border: var(--sklearn-color-fitted-level-0) 1pt solid;color: var(--sklearn-color-unfitted-level-0);text-decoration: none;
-}div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,
-.sk-estimator-doc-link.fitted:hover,
-div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,
-.sk-estimator-doc-link.fitted:hover {/* fitted */background-color: var(--sklearn-color-fitted-level-3);border: var(--sklearn-color-fitted-level-0) 1pt solid;color: var(--sklearn-color-fitted-level-0);text-decoration: none;
-}/* Span, style for the box shown on hovering the info icon */
-.sk-estimator-doc-link span {display: none;z-index: 9999;position: relative;font-weight: normal;right: .2ex;padding: .5ex;margin: .5ex;width: min-content;min-width: 20ex;max-width: 50ex;color: var(--sklearn-color-text);box-shadow: 2pt 2pt 4pt #999;/* unfitted */background: var(--sklearn-color-unfitted-level-0);border: .5pt solid var(--sklearn-color-unfitted-level-3);
-}.sk-estimator-doc-link.fitted span {/* fitted */background: var(--sklearn-color-fitted-level-0);border: var(--sklearn-color-fitted-level-3);
-}.sk-estimator-doc-link:hover span {display: block;
-}/* "?"-specific style due to the `<a>` HTML tag */#sk-container-id-1 a.estimator_doc_link {float: right;font-size: 1rem;line-height: 1em;font-family: monospace;background-color: var(--sklearn-color-unfitted-level-0);border-radius: 1rem;height: 1rem;width: 1rem;text-decoration: none;/* unfitted */color: var(--sklearn-color-unfitted-level-1);border: var(--sklearn-color-unfitted-level-1) 1pt solid;
-}#sk-container-id-1 a.estimator_doc_link.fitted {/* fitted */background-color: var(--sklearn-color-fitted-level-0);border: var(--sklearn-color-fitted-level-1) 1pt solid;color: var(--sklearn-color-fitted-level-1);
-}/* On hover */
-#sk-container-id-1 a.estimator_doc_link:hover {/* unfitted */background-color: var(--sklearn-color-unfitted-level-3);color: var(--sklearn-color-background);text-decoration: none;
-}#sk-container-id-1 a.estimator_doc_link.fitted:hover {/* fitted */background-color: var(--sklearn-color-fitted-level-3);
-}.estimator-table {font-family: monospace;
-}.estimator-table summary {padding: .5rem;cursor: pointer;
-}.estimator-table summary::marker {font-size: 0.7rem;
-}.estimator-table details[open] {padding-left: 0.1rem;padding-right: 0.1rem;padding-bottom: 0.3rem;
-}.estimator-table .parameters-table {margin-left: auto !important;margin-right: auto !important;margin-top: 0;
-}.estimator-table .parameters-table tr:nth-child(odd) {background-color: #fff;
-}.estimator-table .parameters-table tr:nth-child(even) {background-color: #f6f6f6;
-}.estimator-table .parameters-table tr:hover {background-color: #e0e0e0;
-}.estimator-table table td {border: 1px solid rgba(106, 105, 104, 0.232);
-}/*`table td`is set in notebook with right text-align.We need to overwrite it.
-*/
-.estimator-table table td.param {text-align: left;position: relative;padding: 0;
-}.user-set td {color:rgb(255, 94, 0);text-align: left !important;
-}.user-set td.value {color:rgb(255, 94, 0);background-color: transparent;
-}.default td {color: black;text-align: left !important;
-}.user-set td i,
-.default td i {color: black;
-}/*Styles for parameter documentation linksWe need styling for visited so jupyter doesn't overwrite it
-*/
-a.param-doc-link,
-a.param-doc-link:link,
-a.param-doc-link:visited {text-decoration: underline dashed;text-underline-offset: .3em;color: inherit;display: block;padding: .5em;
-}/* "hack" to make the entire area of the cell containing the link clickable */
-a.param-doc-link::before {position: absolute;content: "";inset: 0;
-}.param-doc-description {display: none;position: absolute;z-index: 9999;left: 0;padding: .5ex;margin-left: 1.5em;color: var(--sklearn-color-text);box-shadow: .3em .3em .4em #999;width: max-content;text-align: left;max-height: 10em;overflow-y: auto;/* unfitted */background: var(--sklearn-color-unfitted-level-0);border: thin solid var(--sklearn-color-unfitted-level-3);
-}/* Fitted state for parameter tooltips */
-.fitted .param-doc-description {/* fitted */background: var(--sklearn-color-fitted-level-0);border: thin solid var(--sklearn-color-fitted-level-3);
-}.param-doc-link:hover .param-doc-description {display: block;
-}.copy-paste-icon {background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0NDggNTEyIj48IS0tIUZvbnQgQXdlc29tZSBGcmVlIDYuNy4yIGJ5IEBmb250YXdlc29tZSAtIGh0dHBzOi8vZm9udGF3ZXNvbWUuY29tIExpY2Vuc2UgLSBodHRwczovL2ZvbnRhd2Vzb21lLmNvbS9saWNlbnNlL2ZyZWUgQ29weXJpZ2h0IDIwMjUgRm9udGljb25zLCBJbmMuLS0+PHBhdGggZD0iTTIwOCAwTDMzMi4xIDBjMTIuNyAwIDI0LjkgNS4xIDMzLjkgMTQuMWw2Ny45IDY3LjljOSA5IDE0LjEgMjEuMiAxNC4xIDMzLjlMNDQ4IDMzNmMwIDI2LjUtMjEuNSA0OC00OCA0OGwtMTkyIDBjLTI2LjUgMC00OC0yMS41LTQ4LTQ4bDAtMjg4YzAtMjYuNSAyMS41LTQ4IDQ4LTQ4ek00OCAxMjhsODAgMCAwIDY0LTY0IDAgMCAyNTYgMTkyIDAgMC0zMiA2NCAwIDAgNDhjMCAyNi41LTIxLjUgNDgtNDggNDhMNDggNTEyYy0yNi41IDAtNDgtMjEuNS00OC00OEwwIDE3NmMwLTI2LjUgMjEuNS00OCA0OC00OHoiLz48L3N2Zz4=);background-repeat: no-repeat;background-size: 14px 14px;background-position: 0;display: inline-block;width: 14px;height: 14px;cursor: pointer;
-}
-</style><body><div id="sk-container-id-1" class="sk-top-container" style="overflow: auto;"><div class="sk-text-repr-fallback"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;, max_iter=1000)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator fitted sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-1" type="checkbox" checked><label for="sk-estimator-id-1" class="sk-toggleable__label fitted sk-toggleable__label-arrow"><div><div>LogisticRegression</div></div><div><a class="sk-estimator-doc-link fitted" rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html">?<span>Documentation for LogisticRegression</span></a><span class="sk-estimator-doc-link fitted">i<span>Fitted</span></span></div></label><div class="sk-toggleable__content fitted" data-param-prefix=""><div class="estimator-table"><details><summary>Parameters</summary><table class="parameters-table"><tbody><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('penalty',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=penalty,-%7B%27l1%27%2C%20%27l2%27%2C%20%27elasticnet%27%2C%20None%7D%2C%20default%3D%27l2%27">penalty<span class="param-doc-description">penalty: {'l1', 'l2', 'elasticnet', None}, default='l2'<br><br>Specify the norm of the penalty:<br><br>- `None`: no penalty is added;<br>- `'l2'`: add a L2 penalty term and it is the default choice;<br>- `'l1'`: add a L1 penalty term;<br>- `'elasticnet'`: both L1 and L2 penalty terms are added.<br><br>.. warning::<br>   Some penalties may not work with some solvers. See the parameter<br>   `solver` below, to know the compatibility between the penalty and<br>   solver.<br><br>.. versionadded:: 0.19<br>   l1 penalty with SAGA solver (allowing 'multinomial' + L1)<br><br>.. deprecated:: 1.8<br>   `penalty` was deprecated in version 1.8 and will be removed in 1.10.<br>   Use `l1_ratio` instead. `l1_ratio=0` for `penalty='l2'`, `l1_ratio=1` for<br>   `penalty='l1'` and `l1_ratio` set to any float between 0 and 1 for<br>   `'penalty='elasticnet'`.</span></a></td><td class="value">&#x27;deprecated&#x27;</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('C',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=C,-float%2C%20default%3D1.0">C<span class="param-doc-description">C: float, default=1.0<br><br>Inverse of regularization strength; must be a positive float.<br>Like in support vector machines, smaller values specify stronger<br>regularization. `C=np.inf` results in unpenalized logistic regression.<br>For a visual example on the effect of tuning the `C` parameter<br>with an L1 penalty, see:<br>:ref:`sphx_glr_auto_examples_linear_model_plot_logistic_path.py`.</span></a></td><td class="value">1.0</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('l1_ratio',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=l1_ratio,-float%2C%20default%3D0.0">l1_ratio<span class="param-doc-description">l1_ratio: float, default=0.0<br><br>The Elastic-Net mixing parameter, with `0 <= l1_ratio <= 1`. Setting<br>`l1_ratio=1` gives a pure L1-penalty, setting `l1_ratio=0` a pure L2-penalty.<br>Any value between 0 and 1 gives an Elastic-Net penalty of the form<br>`l1_ratio * L1 + (1 - l1_ratio) * L2`.<br><br>.. warning::<br>   Certain values of `l1_ratio`, i.e. some penalties, may not work with some<br>   solvers. See the parameter `solver` below, to know the compatibility between<br>   the penalty and solver.<br><br>.. versionchanged:: 1.8<br>    Default value changed from None to 0.0.<br><br>.. deprecated:: 1.8<br>    `None` is deprecated and will be removed in version 1.10. Always use<br>    `l1_ratio` to specify the penalty type.</span></a></td><td class="value">0.0</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('dual',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=dual,-bool%2C%20default%3DFalse">dual<span class="param-doc-description">dual: bool, default=False<br><br>Dual (constrained) or primal (regularized, see also<br>:ref:`this equation <regularized-logistic-loss>`) formulation. Dual formulation<br>is only implemented for l2 penalty with liblinear solver. Prefer `dual=False`<br>when n_samples > n_features.</span></a></td><td class="value">False</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('tol',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=tol,-float%2C%20default%3D1e-4">tol<span class="param-doc-description">tol: float, default=1e-4<br><br>Tolerance for stopping criteria.</span></a></td><td class="value">0.0001</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('fit_intercept',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=fit_intercept,-bool%2C%20default%3DTrue">fit_intercept<span class="param-doc-description">fit_intercept: bool, default=True<br><br>Specifies if a constant (a.k.a. bias or intercept) should be<br>added to the decision function.</span></a></td><td class="value">True</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('intercept_scaling',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=intercept_scaling,-float%2C%20default%3D1">intercept_scaling<span class="param-doc-description">intercept_scaling: float, default=1<br><br>Useful only when the solver `liblinear` is used<br>and `self.fit_intercept` is set to `True`. In this case, `x` becomes<br>`[x, self.intercept_scaling]`,<br>i.e. a "synthetic" feature with constant value equal to<br>`intercept_scaling` is appended to the instance vector.<br>The intercept becomes<br>``intercept_scaling * synthetic_feature_weight``.<br><br>.. note::<br>    The synthetic feature weight is subject to L1 or L2<br>    regularization as all other features.<br>    To lessen the effect of regularization on synthetic feature weight<br>    (and therefore on the intercept) `intercept_scaling` has to be increased.</span></a></td><td class="value">1</td></tr><tr class="user-set"><td><i class="copy-paste-icon"onclick="copyToClipboard('class_weight',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=class_weight,-dict%20or%20%27balanced%27%2C%20default%3DNone">class_weight<span class="param-doc-description">class_weight: dict or 'balanced', default=None<br><br>Weights associated with classes in the form ``{class_label: weight}``.<br>If not given, all classes are supposed to have weight one.<br><br>The "balanced" mode uses the values of y to automatically adjust<br>weights inversely proportional to class frequencies in the input data<br>as ``n_samples / (n_classes * np.bincount(y))``.<br><br>Note that these weights will be multiplied with sample_weight (passed<br>through the fit method) if sample_weight is specified.<br><br>.. versionadded:: 0.17<br>   *class_weight='balanced'*</span></a></td><td class="value">&#x27;balanced&#x27;</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('random_state',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=random_state,-int%2C%20RandomState%20instance%2C%20default%3DNone">random_state<span class="param-doc-description">random_state: int, RandomState instance, default=None<br><br>Used when ``solver`` == 'sag', 'saga' or 'liblinear' to shuffle the<br>data. See :term:`Glossary <random_state>` for details.</span></a></td><td class="value">None</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('solver',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=solver,-%7B%27lbfgs%27%2C%20%27liblinear%27%2C%20%27newton-cg%27%2C%20%27newton-cholesky%27%2C%20%27sag%27%2C%20%27saga%27%7D%2C%20%20%20%20%20%20%20%20%20%20%20%20%20default%3D%27lbfgs%27">solver<span class="param-doc-description">solver: {'lbfgs', 'liblinear', 'newton-cg', 'newton-cholesky', 'sag', 'saga'},             default='lbfgs'<br><br>Algorithm to use in the optimization problem. Default is 'lbfgs'.<br>To choose a solver, you might want to consider the following aspects:<br><br>- 'lbfgs' is a good default solver because it works reasonably well for a wide<br>  class of problems.<br>- For :term:`multiclass` problems (`n_classes >= 3`), all solvers except<br>  'liblinear' minimize the full multinomial loss, 'liblinear' will raise an<br>  error.<br>- 'newton-cholesky' is a good choice for<br>  `n_samples` >> `n_features * n_classes`, especially with one-hot encoded<br>  categorical features with rare categories. Be aware that the memory usage<br>  of this solver has a quadratic dependency on `n_features * n_classes`<br>  because it explicitly computes the full Hessian matrix.<br>- For small datasets, 'liblinear' is a good choice, whereas 'sag'<br>  and 'saga' are faster for large ones;<br>- 'liblinear' can only handle binary classification by default. To apply a<br>  one-versus-rest scheme for the multiclass setting one can wrap it with the<br>  :class:`~sklearn.multiclass.OneVsRestClassifier`.<br><br>.. warning::<br>   The choice of the algorithm depends on the penalty chosen (`l1_ratio=0`<br>   for L2-penalty, `l1_ratio=1` for L1-penalty and `0 < l1_ratio < 1` for<br>   Elastic-Net) and on (multinomial) multiclass support:<br><br>   ================= ======================== ======================<br>   solver            l1_ratio                 multinomial multiclass<br>   ================= ======================== ======================<br>   'lbfgs'           l1_ratio=0               yes<br>   'liblinear'       l1_ratio=1 or l1_ratio=0 no<br>   'newton-cg'       l1_ratio=0               yes<br>   'newton-cholesky' l1_ratio=0               yes<br>   'sag'             l1_ratio=0               yes<br>   'saga'            0<=l1_ratio<=1           yes<br>   ================= ======================== ======================<br><br>.. note::<br>   'sag' and 'saga' fast convergence is only guaranteed on features<br>   with approximately the same scale. You can preprocess the data with<br>   a scaler from :mod:`sklearn.preprocessing`.<br><br>.. seealso::<br>   Refer to the :ref:`User Guide <Logistic_regression>` for more<br>   information regarding :class:`LogisticRegression` and more specifically the<br>   :ref:`Table <logistic_regression_solvers>`<br>   summarizing solver/penalty supports.<br><br>.. versionadded:: 0.17<br>   Stochastic Average Gradient (SAG) descent solver. Multinomial support in<br>   version 0.18.<br>.. versionadded:: 0.19<br>   SAGA solver.<br>.. versionchanged:: 0.22<br>   The default solver changed from 'liblinear' to 'lbfgs' in 0.22.<br>.. versionadded:: 1.2<br>   newton-cholesky solver. Multinomial support in version 1.6.</span></a></td><td class="value">&#x27;lbfgs&#x27;</td></tr><tr class="user-set"><td><i class="copy-paste-icon"onclick="copyToClipboard('max_iter',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=max_iter,-int%2C%20default%3D100">max_iter<span class="param-doc-description">max_iter: int, default=100<br><br>Maximum number of iterations taken for the solvers to converge.</span></a></td><td class="value">1000</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('verbose',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=verbose,-int%2C%20default%3D0">verbose<span class="param-doc-description">verbose: int, default=0<br><br>For the liblinear and lbfgs solvers set verbose to any positive<br>number for verbosity.</span></a></td><td class="value">0</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('warm_start',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=warm_start,-bool%2C%20default%3DFalse">warm_start<span class="param-doc-description">warm_start: bool, default=False<br><br>When set to True, reuse the solution of the previous call to fit as<br>initialization, otherwise, just erase the previous solution.<br>Useless for liblinear solver. See :term:`the Glossary <warm_start>`.<br><br>.. versionadded:: 0.17<br>   *warm_start* to support *lbfgs*, *newton-cg*, *sag*, *saga* solvers.</span></a></td><td class="value">False</td></tr><tr class="default"><td><i class="copy-paste-icon"onclick="copyToClipboard('n_jobs',this.parentElement.nextElementSibling)"></i></td><td class="param"><a class="param-doc-link"rel="noreferrer" target="_blank" href="https://scikit-learn.org/1.8/modules/generated/sklearn.linear_model.LogisticRegression.html#:~:text=n_jobs,-int%2C%20default%3DNone">n_jobs<span class="param-doc-description">n_jobs: int, default=None<br><br>Does not have any effect.<br><br>.. deprecated:: 1.8<br>   `n_jobs` is deprecated in version 1.8 and will be removed in 1.10.</span></a></td><td class="value">None</td></tr></tbody></table></details></div></div></div></div></div></div><script>function copyToClipboard(text, element) {// Get the parameter prefix from the closest toggleable contentconst toggleableContent = element.closest('.sk-toggleable__content');const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';const fullParamName = paramPrefix ? `${paramPrefix}${text}` : text;const originalStyle = element.style;const computedStyle = window.getComputedStyle(element);const originalWidth = computedStyle.width;const originalHTML = element.innerHTML.replace('Copied!', '');navigator.clipboard.writeText(fullParamName).then(() => {element.style.width = originalWidth;element.style.color = 'green';element.innerHTML = "Copied!";setTimeout(() => {element.innerHTML = originalHTML;element.style = originalStyle;}, 2000);}).catch(err => {console.error('Failed to copy:', err);element.style.color = 'red';element.innerHTML = "Failed!";setTimeout(() => {element.innerHTML = originalHTML;element.style = originalStyle;}, 2000);});return false;
-}document.querySelectorAll('.copy-paste-icon').forEach(function(element) {const toggleableContent = element.closest('.sk-toggleable__content');const paramPrefix = toggleableContent ? toggleableContent.dataset.paramPrefix : '';const paramName = element.parentElement.nextElementSibling.textContent.trim().split(' ')[0];const fullParamName = paramPrefix ? `${paramPrefix}${paramName}` : paramName;element.setAttribute('title', fullParamName);
-});/*** Adapted from Skrub* https://github.com/skrub-data/skrub/blob/403466d1d5d4dc76a7ef569b3f8228db59a31dc3/skrub/_reporting/_data/templates/report.js#L789* @returns "light" or "dark"*/
-function detectTheme(element) {const body = document.querySelector('body');// Check VSCode themeconst themeKindAttr = body.getAttribute('data-vscode-theme-kind');const themeNameAttr = body.getAttribute('data-vscode-theme-name');if (themeKindAttr && themeNameAttr) {const themeKind = themeKindAttr.toLowerCase();const themeName = themeNameAttr.toLowerCase();if (themeKind.includes("dark") || themeName.includes("dark")) {return "dark";}if (themeKind.includes("light") || themeName.includes("light")) {return "light";}}// Check Jupyter themeif (body.getAttribute('data-jp-theme-light') === 'false') {return 'dark';} else if (body.getAttribute('data-jp-theme-light') === 'true') {return 'light';}// Guess based on a parent element's colorconst color = window.getComputedStyle(element.parentNode, null).getPropertyValue('color');const match = color.match(/^rgb\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\)\s*$/i);if (match) {const [r, g, b] = [parseFloat(match[1]),parseFloat(match[2]),parseFloat(match[3])];// https://en.wikipedia.org/wiki/HSL_and_HSV#Lightnessconst luma = 0.299 * r + 0.587 * g + 0.114 * b;if (luma > 180) {// If the text is very bright we have a dark themereturn 'dark';}if (luma < 75) {// If the text is very dark we have a light themereturn 'light';}// Otherwise fall back to the next heuristic.}// Fallback to system preferencereturn window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
-}function forceTheme(elementId) {const estimatorElement = document.querySelector(`#${elementId}`);if (estimatorElement === null) {console.error(`Element with id ${elementId} not found.`);} else {const theme = detectTheme(estimatorElement);estimatorElement.classList.add(theme);}
-}forceTheme('sk-container-id-1');</script></body>
-
-## Evaluation Results
-
-[More Information Needed]
-
-# How to Get Started with the Model
-
-[More Information Needed]
-
-# Model Card Authors
-
-This model card is written by following authors:
-
-[More Information Needed]
-
-# Model Card Contact
-
-You can contact the model card authors through following channels:
-[More Information Needed]
-
-# Citation
-
-Below you can find information related to citation.
-
-**BibTeX:**
-```
-[More Information Needed]
-```
-
-# Validation
-
-Pipeline-level subsystem gates consumed by MRM.
-
-|  Metric |                                                              Value                                                               |
-| :-----: | :------------------------------------------------------------------------------------------------------------------------------: |
-| metrics | {'pipeline_overall_pass': False, 'conformal_overall_pass': True, 'governance_overall_pass': True, 'fairness_overall_pass': True} |
-
diff --git a/reports/mrm/corepd_model_card.json b/reports/mrm/corepd_model_card.json deleted file mode 100644 index eaf94e7..0000000 --- a/reports/mrm/corepd_model_card.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "model_name": "CorePDCanonical", - "model_version": "1.0", - "champion_artifact": "models/pd_canonical.cbm", - "subsystem_pass": { - "pipeline": false, - "conformal": true, - "governance": true, - "fairness": true - }, - "limitations": [ - "Champion artifact remains CatBoost/CBM; skops export is limited to sklearn-compatible artifacts.", - "Fairlearn is integrated as a sidecar audit and does not replace the canonical fairness gate in this phase.", - "Observational causal outputs remain insights_only and are not promotion-eligible." - ], - "exports": [ - { - "name": "pd_logreg_baseline", - "source_path": "models/pd_logreg_baseline.pkl", - "export_path": "reports/mrm/skops/pd_logreg_baseline.skops", - "status": "exported" - } - ], - "card_render_status": "rendered" -} diff --git a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/baseline_snapshot.json b/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/baseline_snapshot.json deleted file mode 100644 index 44f0293..0000000 --- a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/baseline_snapshot.json +++ /dev/null @@ -1,3635 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "generated_at_utc": "2026-04-07T14:25:39.057029+00:00", - "pipeline_family": "crpto_e2e", - "pipeline_profile": "crpto_e2e_default", - "artifact_scope": "paper", - "promotion_state": "paper_grade", - "upstream_canonical_run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "writes_canonical_artifacts": false, - "git": { - "head": "7c55db2b57fb0994f09640eeab6710aac492dd48", - "branch": "main", - "status_short": "M .gitignore\n M README.md\n M SESSION_STATE.md\n M book/_quarto.yml\n M book/assets/figures/editorial/pipeline_ecl_by_scenario.png\n M book/assets/figures/publication/p3_fig1_variant_scatter.pdf\n M book/assets/figures/publication/p3_fig1_variant_scatter.png\n M book/assets/figures/publication/p3_fig2_grade_coverage_heatmap.pdf\n M book/assets/figures/publication/p3_fig2_grade_coverage_heatmap.png\n M book/assets/figures/publication/p3_fig3_temporal_stability.pdf\n M book/assets/figures/publication/p3_fig3_temporal_stability.png\n M book/assets/figures/publication/p3_fig_calibration_stat_tests.png\n M book/assets/figures/publication/p3_fig_nonconformity_scores_by_grade.pdf\n M book/assets/figures/publication/p3_fig_nonconformity_scores_by_grade.png\n M book/chapters/00b-abstract.qmd\n M book/chapters/01-executive-map.qmd\n M book/chapters/02-glossary/02c-uncertainty-conformal.qmd\n M book/chapters/06-pd-modeling/06b-catboost-tuned.qmd\n M book/chapters/06-pd-modeling/06c-calibration-selection.qmd\n M book/chapters/06-pd-modeling/06d-model-comparison-champion.qmd\n M book/chapters/07-conformal/07c-cqr-variants-benchmark.qmd\n M book/chapters/07-conformal/07d-backtest-monitoring.qmd\n M book/chapters/07-conformal/07e-lgd-ead-conformal.qmd\n M book/chapters/09-portfolio/09b-robust-portfolio.qmd\n M book/chapters/09-portfolio/09c-policy-selection.qmd\n M book/chapters/09-portfolio/09e-efficient-frontier.qmd\n M book/chapters/09-portfolio/index.qmd\n M book/chapters/10-ifrs9-governance/10e-model-risk-management.qmd\n M book/chapters/13-advanced-topics/13a-uncertainty-baselines.qmd\n M book/chapters/13-advanced-topics/13f-gpu-edge-research.qmd\n M book/chapters/14a-introduction-motivation.qmd\n M book/chapters/14b-theoretical-framework.qmd\n M book/chapters/14c-methodology.qmd\n M book/chapters/14d-results.qmd\n M book/chapters/14e-discussion-conclusions.qmd\n M book/chapters/index.qmd\n M book/chapters/16-paper-mondrian/16a-introduction.qmd\n M book/chapters/16-paper-mondrian/16c-results.qmd\n M book/chapters/16-paper-mondrian/index.qmd\n M book/chapters/17-specialization-bridge.qmd\n M book/chapters/18-research-agenda/18a-state-of-the-art.qmd\n M book/chapters/D-configuration-reference.qmd\n M book/references.bib\n M configs/pd_model.champion.yaml\n D configs/pd_model.paper_grade_final.yaml\n M configs/pipeline_registry/pipeline_matrix.yaml\n M configs/pipeline_registry/script_role_registry.yaml\n M configs/pipeline_registry/search_registry.yaml\n M configs/pipelines/crpto_e2e.yaml\n D configs/profiles/paper_grade_final.yaml\n D configs/run_profiles/paper_grade_final.yaml\n M data/processed/brier_score_decomposition.json\n M data/processed/conformal_diagnostic_metrics.json\n M data/processed/statistical_calibration_tests.json\n M docs/CANONICAL_DOCUMENTATION_AND_QUARTO_TRACEABILITY_2026-03-30.md\n M docs/DOCUMENTATION_MAP.md\n M docs/MODEL_RISK_MANAGEMENT.md\n M docs/PIPELINE_FIRST_TOPOLOGY_2026-03-31.md\n M docs/RUNBOOK.md\n M docs/backlog-papers-unified.md\n M dvc.lock\n M dvc.yaml\n M models/bootstrap_validation_status.json\n M models/calibration_mapping_status.json\n M models/challenger_promotion_report.json\n M models/champion_portfolio_policy.json\n M models/champion_registry.json\n M models/champion_search_bundle.json\n M models/conformal_variant_selection_status.json\n M models/dependency_audit_last_valid_artifact.json\n M models/encoding_stability_status.json\n M models/fairlearn_fairness_status.json\n M models/fairness_decision_policy.json\n M models/feature_artifacts_last_valid_artifact.json\n M models/feature_artifacts_runtime_status.json\n M models/ifrs9_diagnostics_status.json\n M models/ifrs9_last_valid_artifact.json\n M models/ifrs9_runtime_status.json\n M models/model_shift_status.json\n M models/monotonicity_audit_status.json\n M models/mrm_report_status.json\n M models/paper_figures_status.json\n D models/paper_grade_protocol_status.json\n M models/pd_backtesting_status.json\n M models/pd_conformal_width_attribution_status.json\n M models/pd_hpo_seed_replay_status.json\n M models/pd_model_contract.json\n M models/pd_rare_event_calibration_status.json\n M models/pd_training_checkpoints/calibration_selection.json\n M models/pd_training_checkpoints/config_validation.json\n M models/pd_training_checkpoints/hpo_summary.json\n M models/pd_training_status.json\n M models/pd_validation_interpretation_status.json\n M models/pipeline_registry/pipeline_matrix.json\n M models/pipeline_registry/script_role_registry.json\n M models/pipeline_registry/search_registry.json\n M models/portfolio_optimization_last_valid_artifact.json\n M models/portfolio_optimization_runtime_status.json\n M models/portfolio_tradeoff_last_valid_artifact.json\n M models/portfolio_tradeoff_runtime_status.json\n M models/shap_fairness_status.json\n M models/threshold_semantics.json\n M models/time_series_forecastability_status.json\n M models/time_series_hierarchy_status.json\n M models/time_series_status.json\n M reports/dependency_summary.json\n M reports/figures/calibration/murphy_diagram.png\n M reports/figures/shap/shap_feature_importance.json\n M reports/mrm/corepd_model_card.html\n M reports/mrm/corepd_model_card.json\n M reports/mrm/skops/pd_logreg_baseline.skops\n M reports/crpto/figures/p3_fig1_variant_scatter.pdf\n M reports/crpto/figures/p3_fig1_variant_scatter.png\n M reports/crpto/figures/p3_fig2_grade_coverage_heatmap.pdf\n M reports/crpto/figures/p3_fig2_grade_coverage_heatmap.png\n M reports/crpto/figures/p3_fig3_temporal_stability.pdf\n M reports/crpto/figures/p3_fig3_temporal_stability.png\n M reports/crpto/figures/p3_fig_calibration_stat_tests.pdf\n M reports/crpto/figures/p3_fig_calibration_stat_tests.png\n M reports/crpto/figures/p3_fig_nonconformity_scores_by_grade.pdf\n M reports/crpto/figures/p3_fig_nonconformity_scores_by_grade.png\n D scripts/analyze_time_series_forecastability.py\n M scripts/benchmark_conformal_variants.py\n M scripts/benchmark_pd_set_prediction.py\n M scripts/build_champion_search_bundle.py\n D scripts/build_pd_model_contract.py\n M scripts/build_pipeline_results.py\n D scripts/build_student_thesis_presentation_elegant_v3.py\n D scripts/compat/end_to_end_pipeline.py\n D scripts/compat/run_long_pipeline.py\n D scripts/core/run_canonical_rebuild.py\n D scripts/export_pr_auc_metrics.py\n D scripts/finalize_paper_grade_run.py\n M scripts/generate_conformal_intervals.py\n M scripts/generate_crpto_figures.py\n D scripts/generate_paper_grade_protocol.py\n D scripts/generate_slice_anomaly_report.py\n D scripts/generate_thesis_document.py\n D scripts/generate_thesis_word.py\n M scripts/history/export_pr_auc_metrics.py\n M scripts/history/generate_slice_anomaly_report.py\n D scripts/history/run_paper_grade_pre_quarto.sh\n M scripts/history/update_cost_matrix_threshold.py\n M scripts/monitor_pipeline_eta.py\n M scripts/optimize_portfolio_tradeoff.py\n D scripts/papers/run_crpto_e2e.py\n M scripts/prepare_streamlit_deploy.py\n M scripts/run_comparison.py\n M scripts/run_long_pipeline.py\n D scripts/run_paper_grade_final.py\n D scripts/run_paper_grade_pre_quarto.sh\n M scripts/search_monotonic_competitor.py\n M scripts/train_pd_model.py\n D scripts/update_cost_matrix_threshold.py\n M scripts/validate_alpha_gamma_bound.py\n M src/data/build_datasets.py\n M src/models/conformal.py\n M src/models/conformal_artifacts.py\n M src/models/optuna_tuning.py\n M src/models/pd_contract.py\n M src/optimization/cuopt_adapter.py\n M src/optimization/portfolio_model.py\n M src/utils/pipeline_topology.py\n M tests/test_config_consistency.py\n M tests/test_data/test_build_datasets.py\n M tests/test_models/test_conformal.py\n M tests/test_models/test_pd_contract.py\n M tests/test_models/test_pd_model.py\n M tests/test_optimization/test_portfolio_model.py\n M tests/test_scripts/test_build_champion_search_bundle.py\n D tests/test_scripts/test_finalize_paper_grade_run.py\n D tests/test_scripts/test_generate_paper_grade_protocol.py\n M tests/test_scripts/test_generate_pipeline_registries.py\n D tests/test_scripts/test_historical_script_wrappers.py\n M tests/test_scripts/test_legacy_guardrails.py\n M tests/test_scripts/test_pipeline_entrypoints.py\n M tests/test_scripts/test_run_comparison.py\n M tests/test_scripts/test_run_long_pipeline.py\n D \"thesis_poster/Presentaci\\303\\263n Tesis Carlos inicial.pptx\"\n D \"thesis_poster/Propuesta proyecto Especializaci\\303\\263n.pdf\"\n D thesis_poster/applied_conformal_prediction_121_168.claude.md\n D thesis_poster/build_thesis_pptx.py\n D thesis_poster/figures/figure1.png\n D thesis_poster/figures/figure2.png\n D thesis_poster/figures/figure3.png\n D thesis_poster/figures/logo-utp.jpg\n D thesis_poster/figures/methods_flowchart.png\n D thesis_poster/generate_ifrs9_figure.py\n D thesis_poster/poster_tesis.tex\n D thesis_poster/review_poster.sh\n D thesis_poster/sdam_2nd_ed_feb_9_2026_12_64.claude.md\n?? book/_freeze/chapters/\n?? book/assets/figures/publication/p3_fig2_global_vs_mondrian.pdf\n?? book/assets/figures/publication/p3_fig2_global_vs_mondrian.png\n?? book/assets/figures/publication/p3_fig6_efficiency_paradox.pdf\n?? book/assets/figures/publication/p3_fig6_efficiency_paradox.png\n?? book/assets/figures/publication/p3_figA1_monthly_coverage_by_grade.pdf\n?? book/assets/figures/publication/p3_figA1_monthly_coverage_by_grade.png\n?? book/assets/figures/publication/p3_figA2_coverage_floor_multiplier.pdf\n?? book/assets/figures/publication/p3_figA2_coverage_floor_multiplier.png\n?? book/chapters/19-specialization-snapshot.qmd\n?? canonical_alpha_gamma_bound.pdf\n?? canonical_alpha_gamma_bound.png\n?? configs/monotonic_competitor_blockwise_exhaustive.yaml\n?? configs/pipelines/search_paper2_ifrs9.yaml\n?? configs/profiles/crpto_e2e_default.yaml\n?? configs/profiles/search_conformal_exhaustive.yaml\n?? configs/profiles/search_conformal_reopen_exhaustive.yaml\n?? configs/profiles/search_paper2_ifrs9_exhaustive.yaml\n?? configs/profiles/search_pd_blockwise_exhaustive.yaml\n?? configs/profiles/search_pd_hpo_local_exhaustive.yaml\n?? configs/time_series_vnext.yaml\n?? data/processed/.test_fe.parquet.tmp-314234\n?? data/processed/alpha_gamma_bound/\n?? data/processed/alpha_gamma_bound_finalist_sweep.json\n?? data/processed/portfolio_bound_aware/\n?? data/processed/portfolio_tradeoff/\n?? data/processed/search_pd/\n?? docs/TIME_SERIES_VNEXT_DECISION_2026-04-02.md\n?? docs/research/chapter3_classification_conformal_memo_2026-04-03.md\n?? models/final_project_promotion.json\n?? models/portfolio_bound_aware/\n?? models/portfolio_tradeoff/\n?? models/search_pd/\n?? models/time_series_policy_review.json\n?? models/time_series_vnext_status.json\n?? papers/\n?? reports/figures/search_pd/\n?? reports/figures/shap/shap_values_test.npz\n?? reports/history/kaggle_faressayah_lendingclub.html\n?? reports/crpto/figures/p3_fig2_global_vs_mondrian.pdf\n?? reports/crpto/figures/p3_fig2_global_vs_mondrian.png\n?? reports/crpto/figures/p3_fig6_efficiency_paradox.pdf\n?? reports/crpto/figures/p3_fig6_efficiency_paradox.png\n?? reports/crpto/figures/p3_figA1_monthly_coverage_by_grade.pdf\n?? reports/crpto/figures/p3_figA1_monthly_coverage_by_grade.png\n?? reports/crpto/figures/p3_figA2_coverage_floor_multiplier.pdf\n?? reports/crpto/figures/p3_figA2_coverage_floor_multiplier.png\n?? reports/crpto/figures/rank1_score_decile_raw_bins5_mgs100_alpha_gamma_bound.pdf\n?? reports/crpto/figures/rank1_score_decile_raw_bins5_mgs100_alpha_gamma_bound.png\n?? reports/crpto/figures/rank2_grade_cal_bins10_mgs100_alpha_gamma_bound.pdf\n?? reports/crpto/figures/rank2_grade_cal_bins10_mgs100_alpha_gamma_bound.png\n?? reports/crpto/figures/rank3_grade_cal_bins10_mgs1000_alpha_gamma_bound.pdf\n?? reports/crpto/figures/rank3_grade_cal_bins10_mgs1000_alpha_gamma_bound.png\n?? scripts/build_pd_hpo_local_config.py\n?? scripts/export_final_project_promotion.py\n?? scripts/run_time_series_vnext.py\n?? scripts/search/compare_conformal_portfolio_finalists.py\n?? scripts/search/resume_conformal_reopen_closure.py\n?? scripts/search/run_conformal_reopen_search.py\n?? scripts/search/run_paper2_ifrs9_search.py\n?? scripts/search/run_pd_hpo_local.py\n?? scripts/search/run_portfolio_bound_aware_search.py\n?? scripts/search/run_portfolio_bound_exact_eval.py\n?? scripts/search/sweep_alpha_gamma_bound_finalists.py\n?? src/models/time_series_vnext.py\n?? tests/test_models/test_time_series_vnext.py\n?? tests/test_scripts/test_benchmark_pd_set_prediction.py\n?? tests/test_scripts/test_build_pd_hpo_local_config.py\n?? tests/test_scripts/test_monitor_pipeline_eta.py\n?? tests/test_scripts/test_run_conformal_reopen_search.py\n?? tests/test_scripts/test_run_portfolio_bound_aware_search.py\n?? tests/test_scripts/test_run_time_series_vnext.py" - }, - "versions": { - "main_env": { - "absl-py": "2.4.0", - "adagio": "0.2.6", - "aiobotocore": "3.2.0", - "aiohappyeyeballs": "2.6.1", - "aiohttp": "3.13.3", - "aiohttp-retry": "2.9.1", - "aioitertools": "0.13.0", - "aiosignal": "1.4.0", - "alembic": "1.18.4", - "altair": "4.2.2", - "amqp": "5.3.1", - "annotated-doc": "0.0.4", - "annotated-types": "0.7.0", - "antlr4-python3-runtime": "4.9.3", - "anyio": "4.12.1", - "appdirs": "1.4.4", - "argon2-cffi": "25.1.0", - "argon2-cffi-bindings": "25.1.0", - "arrow": "1.4.0", - "asttokens": "3.0.1", - "async-lru": "2.2.0", - "asyncssh": "2.22.0", - "atpublic": "7.0.0", - "attrs": "25.4.0", - "autograd": "1.8.0", - "autograd-gamma": "0.5.0", - "babel": "2.18.0", - "backoff": "2.2.1", - "beautifulsoup4": "4.14.3", - "billiard": "4.2.4", - "bleach": "6.3.0", - "blinker": "1.9.0", - "boto3": "1.42.55", - "botocore": "1.42.55", - "cachetools": "6.2.6", - "catboost": "1.2.10", - "causal-learn": "0.1.4.4", - "celery": "5.6.2", - "certifi": "2026.2.25", - "cffi": "2.0.0", - "cfgv": "3.5.0", - "charset-normalizer": "3.4.4", - "clarabel": "0.11.1", - "click": "8.3.1", - "click-didyoumean": "0.3.1", - "click-plugins": "1.1.1.2", - "click-repl": "0.3.0", - "cloudpickle": "3.1.2", - "colorama": "0.4.6", - "colorlog": "6.10.1", - "comm": "0.2.3", - "configobj": "5.0.9", - "contourpy": "1.3.3", - "coreforecast": "0.0.17", - "coverage": "7.13.4", - "cryptography": "46.0.5", - "cvxpy": "1.4.4", - "cycler": "0.12.1", - "cython": "3.2.4", - "dacite": "1.6.0", - "dagshub": "0.6.9", - "dagshub-annotation-converter": "0.1.16", - "databricks-sdk": "0.94.0", - "dataclasses-json": "0.6.7", - "debugpy": "1.8.20", - "decorator": "5.2.1", - "defusedxml": "0.7.1", - "dictdiffer": "0.9.0", - "diskcache": "5.6.3", - "distlib": "0.4.0", - "distro": "1.9.0", - "docker": "7.1.0", - "dowhy": "0.13", - "dpath": "2.2.0", - "duckdb": "1.5.1", - "dulwich": "1.1.0", - "dvc": "3.66.1", - "dvc-data": "3.18.3", - "dvc-http": "2.32.0", - "dvc-objects": "5.2.0", - "dvc-render": "1.0.2", - "dvc-s3": "3.3.0", - "dvc-studio-client": "0.22.0", - "dvc-task": "0.40.2", - "ecos": "2.0.14", - "entrypoints": "0.4", - "executing": "2.2.1", - "fairlearn": "0.13.0", - "fastapi": "0.135.2", - "fastjsonschema": "2.21.2", - "filelock": "3.25.0", - "flask": "3.1.3", - "flask-cors": "6.0.2", - "flatten-dict": "0.4.2", - "flufl-lock": "9.0.0", - "fonttools": "4.61.1", - "formulaic": "1.2.1", - "fqdn": "1.5.1", - "frozenlist": "1.8.0", - "fsspec": "2026.2.0", - "fugue": "0.9.7", - "funcy": "2.0", - "gitdb": "4.0.12", - "gitpython": "3.1.46", - "google-auth": "2.48.0", - "gql": "4.0.0", - "grandalf": "0.8", - "graphene": "3.4.3", - "graphql-core": "3.2.7", - "graphql-relay": "3.2.0", - "graphviz": "0.21", - "greenlet": "3.3.2", - "gto": "1.9.0", - "gunicorn": "25.1.0", - "h11": "0.16.0", - "hierarchicalforecast": "1.5.1", - "highspy": "1.13.1", - "httpcore": "1.0.9", - "httpx": "0.28.1", - "httpx-sse": "0.4.3", - "huey": "2.6.0", - "hydra-core": "1.3.2", - "identify": "2.6.17", - "idna": "3.11", - "immutabledict": "4.3.1", - "importlib-metadata": "8.7.1", - "iniconfig": "2.3.0", - "intel-cmplr-lib-rt": "2025.3.2", - "interface-meta": "1.3.0", - "ipykernel": "7.2.0", - "ipython": "9.10.0", - "ipython-pygments-lexers": "1.1.1", - "ipywidgets": "8.1.8", - "isoduration": "20.11.0", - "iterative-telemetry": "0.0.10", - "itsdangerous": "2.2.0", - "jedi": "0.19.2", - "jinja2": "3.1.6", - "jmespath": "1.1.0", - "joblib": "1.5.3", - "json5": "0.13.0", - "jsonpointer": "3.0.0", - "jsonschema": "4.26.0", - "jsonschema-specifications": "2025.9.1", - "jupyter": "1.1.1", - "jupyter-client": "8.8.0", - "jupyter-console": "6.6.3", - "jupyter-core": "5.9.1", - "jupyter-events": "0.12.0", - "jupyter-lsp": "2.3.0", - "jupyter-server": "2.17.0", - "jupyter-server-terminals": "0.5.4", - "jupyterlab": "4.5.5", - "jupyterlab-pygments": "0.3.0", - "jupyterlab-server": "2.28.0", - "jupyterlab-widgets": "3.0.16", - "jupytext": "1.19.1", - "kiwisolver": "1.4.9", - "kombu": "5.6.2", - "lark": "1.3.1", - "lending-club-risk": "0.1.0", - "lifelines": "0.30.3", - "lightgbm": "4.6.0", - "llvmlite": "0.46.0", - "loguru": "0.7.3", - "lxml": "6.0.2", - "mako": "1.3.10", - "mapie": "1.3.0", - "markdown-it-py": "4.0.0", - "markupsafe": "3.0.3", - "marshmallow": "3.26.2", - "matplotlib": "3.10.8", - "matplotlib-inline": "0.2.1", - "mcp": "1.26.0", - "mdit-py-plugins": "0.5.0", - "mdurl": "0.1.2", - "mistune": "3.2.0", - "mlflow": "3.10.1", - "mlflow-skinny": "3.10.1", - "mlflow-tracing": "3.10.1", - "mlforecast": "1.0.31", - "momentchi2": "0.1.8", - "mpmath": "1.3.0", - "multidict": "6.7.1", - "mypy-extensions": "1.1.0", - "narwhals": "2.17.0", - "nbclient": "0.10.4", - "nbconvert": "7.17.0", - "nbformat": "5.10.4", - "nbstripout": "0.9.1", - "nest-asyncio": "1.6.0", - "networkx": "3.6.1", - "nodeenv": "1.10.0", - "notebook": "7.5.4", - "notebook-shim": "0.2.4", - "numba": "0.64.0", - "numexpr": "2.14.1", - "numpy": "2.4.3", - "omegaconf": "2.3.0", - "opentelemetry-api": "1.39.1", - "opentelemetry-proto": "1.39.1", - "opentelemetry-sdk": "1.39.1", - "opentelemetry-semantic-conventions": "0.60b1", - "optbinning": "0.21.0", - "optuna": "4.8.0", - "optuna-integration": "4.8.0", - "orjson": "3.11.7", - "ortools": "9.10.4067", - "osqp": "1.1.1", - "packaging": "26.0", - "pandas": "2.3.3", - "pandera": "0.30.1", - "pandocfilters": "1.5.1", - "parso": "0.8.6", - "pathspec": "0.12.1", - "pathvalidate": "3.3.1", - "patsy": "1.0.2", - "pexpect": "4.9.0", - "pillow": "12.1.1", - "platformdirs": "4.9.2", - "plotly": "6.6.0", - "pluggy": "1.6.0", - "pre-commit": "4.5.1", - "prettytable": "3.17.0", - "prometheus-client": "0.24.1", - "prompt-toolkit": "3.0.52", - "propcache": "0.4.1", - "protobuf": "6.33.5", - "psutil": "7.2.2", - "ptyprocess": "0.7.0", - "pure-eval": "0.2.3", - "pyarrow": "23.0.1", - "pyasn1": "0.6.2", - "pyasn1-modules": "0.4.2", - "pybind11": "3.0.2", - "pycparser": "3.0", - "pydantic": "2.12.5", - "pydantic-core": "2.41.5", - "pydantic-settings": "2.13.1", - "pydeck": "0.9.1", - "pydot": "4.0.1", - "pygit2": "1.19.1", - "pygments": "2.19.2", - "pygtrie": "2.5.0", - "pyjwt": "2.11.0", - "pyomo": "6.10.0", - "pyparsing": "3.3.2", - "pytest": "9.0.2", - "pytest-cov": "7.1.0", - "python-dateutil": "2.9.0.post0", - "python-discovery": "1.1.0", - "python-docx": "1.2.0", - "python-dotenv": "1.2.2", - "python-json-logger": "4.0.0", - "python-multipart": "0.0.22", - "python-pptx": "1.0.2", - "pytz": "2025.2", - "pyyaml": "6.0.3", - "pyzmq": "27.1.0", - "qpsolvers": "4.8.2", - "referencing": "0.37.0", - "requests": "2.32.5", - "requests-toolbelt": "1.0.0", - "rfc3339-validator": "0.1.4", - "rfc3986-validator": "0.1.1", - "rfc3987-syntax": "1.1.0", - "rich": "14.3.3", - "ropwr": "1.1.0", - "rpds-py": "0.30.0", - "rsa": "4.9.1", - "ruamel-yaml": "0.19.1", - "ruff": "0.15.7", - "s3fs": "2026.2.0", - "s3transfer": "0.16.0", - "scikit-learn": "1.8.0", - "scikit-survival": "0.27.0", - "scipy": "1.15.3", - "scmrepo": "3.6.1", - "scs": "3.2.11", - "seaborn": "0.13.2", - "semver": "3.0.4", - "send2trash": "2.1.0", - "setuptools": "75.9.1", - "shellingham": "1.5.4", - "shortuuid": "1.0.13", - "shtab": "1.8.0", - "six": "1.17.0", - "skops": "0.13.0", - "smmap": "5.0.2", - "soupsieve": "2.8.3", - "sqlalchemy": "2.0.47", - "sqlparse": "0.5.5", - "sqltrie": "0.11.2", - "sse-starlette": "3.3.2", - "stack-data": "0.6.3", - "starlette": "0.52.1", - "statsforecast": "2.0.3", - "statsmodels": "0.14.6", - "streamlit": "1.55.0", - "streamlit-flow-component": "1.6.1", - "streamlit-mermaid": "0.3.0", - "sympy": "1.14.0", - "tabulate": "0.9.0", - "tenacity": "9.1.4", - "terminado": "0.18.1", - "threadpoolctl": "3.6.0", - "tinycss2": "1.4.0", - "toml": "0.10.2", - "tomlkit": "0.14.0", - "toolz": "1.1.0", - "tornado": "6.5.4", - "tqdm": "4.67.3", - "traitlets": "5.14.3", - "treelib": "1.8.0", - "triad": "1.0.2", - "typeguard": "4.5.1", - "typer": "0.24.1", - "typing-extensions": "4.15.0", - "typing-inspect": "0.9.0", - "typing-inspection": "0.4.2", - "tzdata": "2025.3", - "tzlocal": "5.3.1", - "uri-template": "1.3.0", - "urllib3": "2.6.3", - "utilsforecast": "0.2.15", - "uvicorn": "0.42.0", - "venn-abers": "1.5.1", - "vine": "5.1.0", - "virtualenv": "21.1.0", - "voluptuous": "0.16.0", - "watchdog": "6.0.0", - "wcwidth": "0.6.0", - "webcolors": "25.10.0", - "webencodings": "0.5.1", - "websocket-client": "1.9.0", - "werkzeug": "3.1.6", - "widgetsnbextension": "4.0.15", - "wrapt": "2.1.1", - "xlsxwriter": "3.2.9", - "yarl": "1.23.0", - "zc-lockfile": "4.0", - "zipp": "3.23.0" - }, - "rapids_env": { - "_libgcc_mutex": "0.1", - "_openmp_mutex": "4.5", - "_python_abi3_support": "1.0", - "aiohappyeyeballs": "2.6.1", - "aiohttp": "3.13.3", - "aiosignal": "1.4.0", - "anyio": "4.12.1", - "aom": "3.9.1", - "argon2-cffi": "25.1.0", - "argon2-cffi-bindings": "25.1.0", - "arrow": "1.4.0", - "asttokens": "3.0.1", - "async-lru": "2.1.0", - "attr": "2.5.2", - "attrs": "25.4.0", - "aws-c-auth": "0.9.3", - "aws-c-cal": "0.9.13", - "aws-c-common": "0.12.6", - "aws-c-compression": "0.3.1", - "aws-c-event-stream": "0.5.7", - "aws-c-http": "0.10.7", - "aws-c-io": "0.23.3", - "aws-c-mqtt": "0.13.3", - "aws-c-s3": "0.11.3", - "aws-c-sdkutils": "0.2.4", - "aws-checksums": "0.2.7", - "aws-crt-cpp": "0.35.4", - "aws-sdk-cpp": "1.11.606", - "azure-core-cpp": "1.16.2", - "azure-identity-cpp": "1.13.3", - "azure-storage-blobs-cpp": "12.16.0", - "azure-storage-common-cpp": "12.12.0", - "azure-storage-files-datalake-cpp": "12.14.0", - "babel": "2.18.0", - "backports.zstd": "1.3.0", - "beautifulsoup4": "4.14.3", - "bleach": "6.3.0", - "bleach-with-css": "6.3.0", - "blosc": "1.21.6", - "bokeh": "3.8.2", - "boost": "1.82.0", - "branca": "0.8.2", - "brotli": "1.2.0", - "brotli-bin": "1.2.0", - "brotli-python": "1.2.0", - "brunsli": "0.1", - "bzip2": "1.0.8", - "c-ares": "1.34.6", - "c-blosc2": "2.23.0", - "ca-certificates": "2026.1.4", - "cached-property": "1.5.2", - "cached_property": "1.5.2", - "cachetools": "7.0.0", - "certifi": "2026.1.4", - "cffi": "2.0.0", - "charls": "2.4.2", - "charset-normalizer": "3.4.4", - "click": "8.3.1", - "cloudpickle": "3.1.2", - "colorcet": "3.1.0", - "comm": "0.2.3", - "contourpy": "1.3.3", - "cpython": "3.11.14", - "cucim": "25.12.00", - "cuda-bindings": "12.9.6", - "cuda-cccl_linux-64": "13.1.115", - "cuda-core": "0.3.2", - "cuda-crt-dev_linux-64": "13.1.115", - "cuda-crt-tools": "13.1.115", - "cuda-cudart": "13.1.80", - "cuda-cudart-dev": "13.1.80", - "cuda-cudart-dev_linux-64": "13.1.80", - "cuda-cudart-static": "13.1.80", - "cuda-cudart-static_linux-64": "13.1.80", - "cuda-cudart_linux-64": "13.1.80", - "cuda-nvcc-dev_linux-64": "13.1.115", - "cuda-nvcc-impl": "13.1.115", - "cuda-nvcc-tools": "13.1.115", - "cuda-nvrtc": "13.1.115", - "cuda-nvvm-dev_linux-64": "13.1.115", - "cuda-nvvm-impl": "13.1.115", - "cuda-nvvm-tools": "13.1.115", - "cuda-pathfinder": "1.5.1", - "cuda-profiler-api": "13.1.115", - "cuda-python": "12.9.6", - "cuda-toolkit": "12.9.1", - "cuda-version": "13.1", - "cudf": "25.12.00", - "cudf-cu12": "26.2.1", - "cudf-polars": "25.12.00", - "cudf_kafka": "25.12.00", - "cugraph": "25.12.02", - "cuml": "25.12.00", - "cuopt": "25.12.00", - "cuopt-cu12": "26.2.0", - "cuopt-mps-parser": "26.2.0", - "cupy": "13.6.0", - "cupy-cuda12x": "14.0.1", - "custreamz": "25.12.00", - "cuvs": "25.12.00", - "cuxfilter": "25.12.00", - "cycler": "0.12.1", - "cyrus-sasl": "2.1.28", - "cytoolz": "1.1.0", - "dask": "2025.9.2", - "dask-core": "2025.9.2", - "dask-cuda": "25.12.00", - "dask-cudf": "25.12.00", - "datashader": "0.18.2", - "dav1d": "1.2.1", - "debugpy": "1.8.20", - "decorator": "5.2.1", - "defusedxml": "0.7.1", - "distributed": "2025.9.2", - "distributed-ucxx": "0.47.00", - "dlpack": "0.8", - "exceptiongroup": "1.3.1", - "executing": "2.2.1", - "fastrlock": "0.8.3", - "folium": "0.20.0", - "fonttools": "4.61.1", - "fqdn": "1.5.1", - "freetype": "2.14.1", - "freexl": "2.0.0", - "frozenlist": "1.7.0", - "fsspec": "2026.2.0", - "geopandas": "1.1.2", - "geopandas-base": "1.1.2", - "geos": "3.14.1", - "gflags": "2.2.2", - "giflib": "5.2.2", - "glog": "0.7.1", - "h11": "0.16.0", - "h2": "4.3.0", - "h5py": "3.15.1", - "hdf5": "1.14.6", - "holoviews": "1.22.1", - "hpack": "4.1.0", - "httpcore": "1.0.9", - "httpx": "0.28.1", - "hupper": "1.12.1", - "hyperframe": "6.1.0", - "icu": "78.2", - "idna": "3.11", - "imagecodecs": "2026.1.14", - "imageio": "2.37.0", - "importlib-metadata": "8.7.0", - "importlib_resources": "6.5.2", - "ipykernel": "7.2.0", - "ipython": "9.10.0", - "ipython_pygments_lexers": "1.1.1", - "isoduration": "20.11.0", - "jedi": "0.19.2", - "jinja2": "3.1.6", - "joblib": "1.5.3", - "json-c": "0.18", - "json5": "0.13.0", - "jsonpointer": "3.0.0", - "jsonschema": "4.26.0", - "jsonschema-specifications": "2025.9.1", - "jsonschema-with-format-nongpl": "4.26.0", - "jupyter-lsp": "2.3.0", - "jupyter-server-proxy": "4.4.0", - "jupyter_client": "8.8.0", - "jupyter_core": "5.9.1", - "jupyter_events": "0.12.0", - "jupyter_server": "2.17.0", - "jupyter_server_terminals": "0.5.4", - "jupyterlab": "4.5.3", - "jupyterlab_pygments": "0.3.0", - "jupyterlab_server": "2.28.0", - "jxrlib": "1.1", - "keyutils": "1.6.3", - "kiwisolver": "1.4.9", - "krb5": "1.21.3", - "lark": "1.3.1", - "lazy-loader": "0.4", - "lazy_loader": "0.4", - "lcms2": "2.18", - "ld_impl_linux-64": "2.45.1", - "legacy-cgi": "2.6.4", - "lerc": "4.0.0", - "libabseil": "20260107.0", - "libaec": "1.1.5", - "libarchive": "3.8.5", - "libarrow": "21.0.0", - "libarrow-acero": "21.0.0", - "libarrow-compute": "21.0.0", - "libarrow-dataset": "21.0.0", - "libarrow-substrait": "21.0.0", - "libavif16": "1.3.0", - "libblas": "3.11.0", - "libboost-headers": "1.82.0", - "libboost-python": "1.82.0", - "libboost-python-devel": "1.82.0", - "libbrotlicommon": "1.2.0", - "libbrotlidec": "1.2.0", - "libbrotlienc": "1.2.0", - "libcap": "2.77", - "libcblas": "3.11.0", - "libcrc32c": "1.1.2", - "libcublas": "13.2.1.1", - "libcublas-dev": "13.2.1.1", - "libcucim": "25.12.00", - "libcudf": "25.12.00", - "libcudf-cu12": "26.2.1", - "libcudf_kafka": "25.12.00", - "libcudss": "0.7.1.4", - "libcufft": "12.1.0.78", - "libcufile": "1.16.1.26", - "libcufile-dev": "1.16.1.26", - "libcugraph": "25.12.02", - "libcugraph_etl": "25.12.02", - "libcuml": "25.12.00", - "libcumlprims": "25.12.00", - "libcuopt": "25.12.00", - "libcuopt-cu12": "26.2.0", - "libcurand": "10.4.1.81", - "libcurand-dev": "10.4.1.81", - "libcurl": "8.18.0", - "libcusolver": "12.0.9.81", - "libcusolver-dev": "12.0.9.81", - "libcusparse": "12.7.3.1", - "libcusparse-dev": "12.7.3.1", - "libcuvs": "25.12.00", - "libcuvs-headers": "25.12.00", - "libdeflate": "1.25", - "libedit": "3.1.20250104", - "libev": "4.33", - "libevent": "2.1.12", - "libexpat": "2.7.3", - "libffi": "3.5.2", - "libfreetype": "2.14.1", - "libfreetype6": "2.14.1", - "libgcc": "15.2.0", - "libgcc-ng": "15.2.0", - "libgdal-core": "3.12.1", - "libgfortran": "15.2.0", - "libgfortran5": "15.2.0", - "libgomp": "15.2.0", - "libgoogle-cloud": "2.39.0", - "libgoogle-cloud-storage": "2.39.0", - "libgrpc": "1.78.0", - "libhwloc": "2.12.2", - "libhwy": "1.3.0", - "libiconv": "1.18", - "libjpeg-turbo": "3.1.2", - "libjxl": "0.11.1", - "libkml": "1.3.0", - "libkvikio": "25.12.00", - "libkvikio-cu12": "26.2.0", - "liblapack": "3.11.0", - "liblzma": "5.8.2", - "libmps-parser": "25.12.00", - "libnghttp2": "1.67.0", - "libnl": "3.11.0", - "libnsl": "2.0.1", - "libntlm": "1.8", - "libnuma": "2.0.18", - "libnvcomp": "5.0.0.6", - "libnvcomp-dev": "5.0.0.6", - "libnvimgcodec": "0.6.0", - "libnvimgcodec0": "0.6.0", - "libnvjitlink": "13.1.115", - "libnvjpeg": "13.0.3.75", - "libnvjpeg2k0": "0.9.0.43", - "libnvptxcompiler-dev": "13.1.115", - "libnvptxcompiler-dev_linux-64": "13.1.115", - "libnvtiff": "0.5.1.75", - "libnvtiff0": "0.5.1.75", - "libopenblas": "0.3.30", - "libopentelemetry-cpp": "1.21.0", - "libopentelemetry-cpp-headers": "1.21.0", - "libparquet": "21.0.0", - "libpng": "1.6.54", - "libprotobuf": "6.33.5", - "libraft": "25.12.00", - "libraft-cu12": "26.2.0", - "libraft-headers": "25.12.00", - "libraft-headers-only": "25.12.00", - "librdkafka": "2.8.0", - "libre2-11": "2025.11.05", - "librmm": "25.12.00", - "librmm-cu12": "26.2.0", - "librttopo": "1.1.0", - "libsodium": "1.0.20", - "libspatialite": "5.1.0", - "libsqlite": "3.51.2", - "libssh2": "1.11.1", - "libstdcxx": "15.2.0", - "libstdcxx-ng": "15.2.0", - "libsystemd0": "258.3", - "libthrift": "0.22.0", - "libtiff": "4.7.1", - "libucxx": "0.47.00", - "libudev1": "258.3", - "libutf8proc": "2.11.3", - "libuuid": "2.41.3", - "libuv": "1.51.0", - "libwebp-base": "1.6.0", - "libxcb": "1.17.0", - "libxcrypt": "4.4.36", - "libxgboost": "3.1.2", - "libxml2": "2.15.1", - "libxml2-16": "2.15.1", - "libxml2-devel": "2.15.1", - "libzlib": "1.3.1", - "libzopfli": "1.0.3", - "linkify-it-py": "2.0.3", - "llvmlite": "0.44.0", - "locket": "1.0.0", - "loguru": "0.7.3", - "lz4": "4.4.5", - "lz4-c": "1.10.0", - "lzo": "2.10", - "mapclassify": "2.10.0", - "markdown": "3.10.1", - "markdown-it-py": "4.0.0", - "markupsafe": "3.0.3", - "matplotlib-base": "3.10.8", - "matplotlib-inline": "0.2.1", - "mdit-py-plugins": "0.5.0", - "mdurl": "0.1.2", - "minizip": "4.0.10", - "mistune": "3.2.0", - "msgpack-python": "1.1.2", - "multidict": "6.7.0", - "multipledispatch": "0.6.0", - "munkres": "1.1.4", - "muparser": "2.3.5", - "narwhals": "2.16.0", - "nbclient": "0.10.4", - "nbconvert-core": "7.17.0", - "nbformat": "5.10.4", - "nccl": "2.29.2.1", - "ncurses": "6.5", - "nest-asyncio": "1.6.0", - "networkx": "3.6.1", - "nlohmann_json": "3.12.0", - "nodejs": "25.6.0", - "notebook": "7.5.3", - "notebook-shim": "0.2.4", - "numba": "0.61.2", - "numba-cuda": "0.22.2", - "numpy": "2.2.6", - "nvidia-cublas-cu12": "12.9.1.4", - "nvidia-cuda-cccl-cu12": "12.9.27", - "nvidia-cuda-nvcc-cu12": "12.9.86", - "nvidia-cuda-nvrtc-cu12": "12.9.86", - "nvidia-cuda-runtime-cu12": "12.9.79", - "nvidia-cudss-cu12": "0.7.1.6", - "nvidia-curand-cu12": "10.3.10.19", - "nvidia-cusolver-cu12": "11.7.5.82", - "nvidia-cusparse-cu12": "12.5.10.65", - "nvidia-libnvcomp-cu12": "5.1.0.21", - "nvidia-ml-py": "13.590.48", - "nvidia-nccl-cu12": "2.29.7", - "nvidia-nvjitlink-cu12": "12.9.86", - "nvidia-nvtx-cu12": "12.9.79", - "nvtx": "0.2.14", - "nx-cugraph": "25.12.00", - "openjpeg": "2.5.4", - "openjph": "0.26.0", - "openssl": "3.6.1", - "orc": "2.2.2", - "overrides": "7.7.0", - "packaging": "26.0", - "pandas": "2.3.3", - "pandocfilters": "1.5.0", - "panel": "1.8.7", - "param": "2.3.2", - "parso": "0.8.5", - "partd": "1.4.2", - "pastedeploy": "3.1.0", - "pcre2": "10.47", - "pexpect": "4.9.0", - "pillow": "12.1.0", - "pip": "26.0.1", - "plaster": "1.0", - "plaster_pastedeploy": "0.7", - "platformdirs": "4.5.1", - "polars": "1.34.0", - "polars-runtime-32": "1.34.0", - "proj": "9.7.1", - "prometheus-cpp": "1.3.0", - "prometheus_client": "0.24.1", - "prompt-toolkit": "3.0.52", - "propcache": "0.3.1", - "psutil": "7.2.2", - "pthread-stubs": "0.4", - "ptyprocess": "0.7.0", - "pure_eval": "0.2.3", - "py-xgboost": "3.1.2", - "pyarrow": "21.0.0", - "pyarrow-core": "21.0.0", - "pycparser": "2.22", - "pyct": "0.6.0", - "pygments": "2.19.2", - "pylibcudf": "25.12.00", - "pylibcudf-cu12": "26.2.1", - "pylibcugraph": "25.12.02", - "pylibraft": "25.12.00", - "pylibraft-cu12": "26.2.0", - "pynndescent": "0.6.0", - "pyogrio": "0.12.1", - "pyomo": "6.10.0", - "pyparsing": "3.3.2", - "pyproj": "3.7.2", - "pyramid": "2.0.2", - "pysocks": "1.7.1", - "python": "3.11.14", - "python-confluent-kafka": "2.8.0", - "python-dateutil": "2.9.0.post0", - "python-fastjsonschema": "2.21.2", - "python-gil": "3.11.14", - "python-json-logger": "2.0.7", - "python-tzdata": "2025.3", - "python_abi": "3.11", - "pytz": "2025.2", - "pyviz_comms": "3.0.6", - "pywavelets": "1.9.0", - "pyyaml": "6.0.3", - "pyzmq": "27.1.0", - "qhull": "2020.2", - "raft-dask": "25.12.00", - "rapids": "0.0.1", - "rapids-dask-dependency": "25.12.01", - "rapids-logger": "0.2.3", - "rapids-xgboost": "26.02.00", - "rav1e": "0.7.1", - "rdma-core": "61.0", - "re2": "2025.11.05", - "readline": "8.3", - "referencing": "0.37.0", - "requests": "2.32.5", - "rfc3339-validator": "0.1.4", - "rfc3986-validator": "0.1.1", - "rfc3987-syntax": "1.1.0", - "rich": "14.3.2", - "rmm": "25.12.00", - "rmm-cu12": "26.2.0", - "rpds-py": "0.30.0", - "s2n": "1.6.2", - "scikit-image": "0.24.0", - "scikit-learn": "1.8.0", - "scipy": "1.16.3", - "send2trash": "2.1.0", - "setuptools": "82.0.0", - "shapely": "2.1.2", - "simpervisor": "1.0.0", - "six": "1.17.0", - "snappy": "1.2.2", - "sniffio": "1.3.1", - "sortedcontainers": "2.4.0", - "soupsieve": "2.8.3", - "sqlite": "3.51.2", - "stack_data": "0.6.3", - "streamz": "0.6.5", - "svt-av1": "4.0.0", - "tbb": "2022.3.0", - "tblib": "3.2.2", - "terminado": "0.18.1", - "threadpoolctl": "3.6.0", - "tifffile": "2026.1.28", - "tinycss2": "1.5.1", - "tk": "8.6.13", - "tomli": "2.4.0", - "toolz": "1.1.0", - "tornado": "6.5.4", - "tqdm": "4.67.3", - "traitlets": "5.14.3", - "translationstring": "1.4", - "treelite": "4.6.1", - "typing-extensions": "4.15.0", - "typing_extensions": "4.15.0", - "typing_utils": "0.1.0", - "tzdata": "2025c", - "uc-micro-py": "1.0.3", - "ucx": "1.19.1", - "ucxx": "0.47.00", - "umap-learn": "0.5.11", - "unicodedata2": "17.0.0", - "uri-template": "1.3.0", - "uriparser": "0.9.8", - "urllib3": "2.6.3", - "venusian": "3.1.1", - "wcwidth": "0.6.0", - "webcolors": "25.10.0", - "webencodings": "0.5.1", - "webob": "1.8.9", - "websocket-client": "1.9.0", - "wheel": "0.46.3", - "xarray": "2026.1.0", - "xerces-c": "3.3.0", - "xgboost": "3.1.2", - "xorg-libxau": "1.0.12", - "xorg-libxdmcp": "1.1.5", - "xyzservices": "2025.11.0", - "yaml": "0.2.5", - "yarl": "1.22.0", - "zeromq": "4.3.5", - "zfp": "1.0.1", - "zict": "3.0.0", - "zipp": "3.23.0", - "zlib": "1.3.1", - "zlib-ng": "2.3.3", - "zope.deprecation": "4.4.0", - "zope.interface": "8.0.1", - "zstd": "1.5.7" - } - }, - "artifacts": { - "reports/dvc/metrics_summary.json": { - "exists": true, - "sha256": "8478836af28cd6cfaf350243a727ae2c54c78c1e6f9206b7035f9bfa4823bb85", - "size_bytes": 1828 - }, - "data/processed/model_comparison.json": { - "exists": true, - "sha256": "ab2321771a00f1c9e539d7338965c304c010633abba37a1d3083e217fb09b6f3", - "size_bytes": 15059 - }, - "data/processed/pipeline_summary.json": { - "exists": true, - "sha256": "89417baa5fb081855cbb3bd84052ee82270ea5f710decf72e9f15868728bdc56", - "size_bytes": 4674 - }, - "models/conformal_policy_status.json": { - "exists": true, - "sha256": "b5ed38e85d94c9ab7dff44f0aebc6d52599148732028203059b64be378bbb554", - "size_bytes": 12473 - }, - "models/fairness_audit_status.json": { - "exists": true, - "sha256": "f2c3d2ac7125ebd5679da92968c8f02375680a073cc9bcada987f65097ec1e18", - "size_bytes": 3343 - }, - "models/governance_status.json": { - "exists": true, - "sha256": "301d7109e7369c323bd9e4cd5464dd86a190eb76f4613de2bff0f4fa96c7b35c", - "size_bytes": 9580 - }, - "models/conformal_lgd_ead_status.json": { - "exists": true, - "sha256": "c54f40c6b440015648eebf0b17449df76a05fe833ce4b617fa928268b5fb09e0", - "size_bytes": 7815 - }, - "models/survival_summary.pkl": { - "exists": true, - "sha256": "6369f4ec3cbfafb00f761e8543a0379bf44194470d2a6d465ea459c893cff31a", - "size_bytes": 1557 - }, - "data/processed/portfolio_robustness_summary.parquet": { - "exists": true, - "sha256": "02c89e002a5b3f9f492b5748f189d077dc86d34b21caf8354f2392884da2c629", - "size_bytes": 14930 - }, - "data/processed/portfolio_robustness_frontier.parquet": { - "exists": true, - "sha256": "a2d7653efbec2e55acfdf0a5a11b4f1d0b9be5b356fbd379a76a371c62eaf85c", - "size_bytes": 127884 - }, - "data/processed/ifrs9_scenario_summary.parquet": { - "exists": true, - "sha256": "e0f1c752df9233f1c40dde832e65685dea2dbfdb12c64db2d0f9f52decfe728d", - "size_bytes": 17684 - }, - "reports/gpu_benchmark/gpu_bench_meta.json": { - "exists": true, - "sha256": "a5da2aebfedd61f6c4cfbca1a8a4b4fdf89a932b57efce065de5fc8df1858cd4", - "size_bytes": 960 - }, - "reports/gpu_benchmark/cuml_benchmark.csv": { - "exists": true, - "sha256": "d8d9ca67782d48cfca3fe684bf611e018c6559d14f508288dc83b533b0fe623b", - "size_bytes": 1106 - }, - "reports/gpu_benchmark/cugraph_benchmark.csv": { - "exists": true, - "sha256": "9567aa78fdd5d6d24c9ace43694b761cb920c049671039b805c21f84e5813291", - "size_bytes": 401 - }, - "reports/gpu_benchmark/cuopt_benchmark.csv": { - "exists": true, - "sha256": "81735d629576b3aefcb30bfa90f0b67d9434ec5377cfee92bbb7758e7868c3b3", - "size_bytes": 871 - }, - "reports/gpu_benchmark/cudf_polars_benchmark.csv": { - "exists": true, - "sha256": "33dc933fff824dfc126a58a41670984d411676605324d7d7c13ece80066fe4d0", - "size_bytes": 340 - }, - "reports/gpu_benchmark/cupy_benchmark.csv": { - "exists": true, - "sha256": "382e3755ae73fe987665c1dbde2c6003187f858e58c3cc3a03ee18ea4806c5c1", - "size_bytes": 383 - } - }, - "metrics": { - "dvc_metrics": { - "conformal.avg_width90": 0.7527708083842941, - "conformal.coverage90": 0.9241590788423406, - "conformal.coverage95": 0.9596018333580141, - "conformal.min_group_coverage90": 0.8900073882526782, - "conformal.overall_pass": 1.0, - "ifrs9.ecl_baseline": 875935946.2736094, - "ifrs9.ecl_severe": 1487900897.2686803, - "ifrs9.severe_uplift_pct": 69.86412118357295, - "optimization.nonrobust_funded": 152.0, - "optimization.nonrobust_return": 73159.34466177782, - "optimization.price_of_robustness": -66048.30591090144, - "optimization.robust_funded": 235.0, - "optimization.robust_return": 139207.65057267927, - "pd.auc": 0.7126809898257414, - "pd.brier": 0.15456155709655775, - "pd.d2_brier": 0.09864138358717767, - "pd.ece": 0.006731531345207922, - "pd.gini": 0.4253619796514827, - "pd.ks": 0.31268447874554145 - }, - "dvc_metrics_meta": { - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T05:07:52.494864+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2" - }, - "model_comparison": { - "models": [ - { - "model": "Logistic Regression", - "auc": 0.6789, - "gini": 0.3579, - "brier": 0.2325, - "d2_brier": -0.356 - }, - { - "model": "CatBoost (default)", - "auc": 0.7128, - "gini": 0.4256, - "brier": 0.2095, - "d2_brier": -0.2217 - }, - { - "model": "CatBoost (tuned)", - "auc": 0.7128, - "gini": 0.4256, - "brier": 0.2095, - "d2_brier": -0.2217 - }, - { - "model": "CatBoost (tuned + calibrated)", - "auc": 0.7127, - "gini": 0.4254, - "brier": 0.1546, - "d2_brier": 0.0986 - } - ], - "best_model": "CatBoost (tuned + calibrated)", - "best_calibration": "Venn-Abers", - "optuna_best_auc": 0.7128, - "optuna_n_trials": 0, - "hpo_trials_executed": 0, - "hpo_best_validation_auc": 0.7221, - "validation_scheme": "temporal_train_val_cal_test", - "feature_count_default": 42, - "feature_count_tuned": 42, - "calibration_selection_report": { - "selected_method": "venn_abers", - "selection_reason": "feasible_multi_metric", - "auc_drop_limit": 0.0015, - "candidates": [ - { - "method": "platt", - "folds_used": 4, - "mean_brier": 0.15958740589258152, - "mean_log_loss": 0.48904405217674785, - "mean_ece": 0.021825964776485386, - "mean_auc_drop": 0.0, - "brier_variance": 2.2417204457832686e-05, - "ece_variance": 3.0612803544634866e-06, - "stability": 2.5478484812296173e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.7162750060014119, - "auc_drop": 0.0, - "brier": 0.15210592246344298, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.4711275847227073, - "ece": 0.023798878871191257 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7137578123443502, - "auc_drop": 0.0, - "brier": 0.1595655785898165, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.4888525156352012, - "ece": 0.021642260782289733 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7196508805224282, - "auc_drop": 0.0, - "brier": 0.16500428063296432, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5016630965290716, - "ece": 0.02276416465903103 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7131331724667808, - "auc_drop": 0.0, - "brier": 0.16167384188410222, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4945330118200113, - "ece": 0.019098554793429527 - } - ] - }, - { - "method": "isotonic", - "folds_used": 4, - "mean_brier": 0.1595832553554798, - "mean_log_loss": 0.4896474838692051, - "mean_ece": 0.023323024517531266, - "mean_auc_drop": 0.00019330039380807684, - "brier_variance": 2.1720345584669146e-05, - "ece_variance": 6.450997363829596e-06, - "stability": 2.8171342948498743e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.7159637371312912, - "auc_drop": 0.0003112688701206734, - "brier": 0.15221129127429892, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.47394017921687165, - "ece": 0.026971999727171802 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7134764625309519, - "auc_drop": 0.000281349813398335, - "brier": 0.15965480510372984, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.48879216922395263, - "ece": 0.023471723262675024 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7195381424223168, - "auc_drop": 0.00011273810011136476, - "brier": 0.16494845287785834, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5011860111552263, - "ece": 0.02304525112996469 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7130653276751788, - "auc_drop": 6.784479160193424e-05, - "brier": 0.16151847216603205, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4946715758807695, - "ece": 0.019803123950313542 - } - ] - }, - { - "method": "venn_abers", - "folds_used": 4, - "mean_brier": 0.15956506166210385, - "mean_log_loss": 0.4887236629247538, - "mean_ece": 0.02307409187208089, - "mean_auc_drop": 0.00016360264421683146, - "brier_variance": 2.1781167975507866e-05, - "ece_variance": 6.108658616092844e-06, - "stability": 2.7889826591600712e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.71603162066236, - "auc_drop": 0.00024338533905188253, - "brier": 0.15218331951570335, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.4711075736416691, - "ece": 0.026605278647181308 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7135137022124884, - "auc_drop": 0.00024411013186176334, - "brier": 0.15962815651316078, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.4886894900545325, - "ece": 0.02320141751288713 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7195530155068415, - "auc_drop": 9.786501558672889e-05, - "brier": 0.164934542423762, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5011176637036296, - "ece": 0.022866067611790976 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7130641223764138, - "auc_drop": 6.905009036695109e-05, - "brier": 0.16151422819578928, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4939799242991838, - "ece": 0.019623603716464152 - } - ] - } - ], - "feasible_candidates": [ - { - "method": "platt", - "folds_used": 4, - "mean_brier": 0.15958740589258152, - "mean_log_loss": 0.48904405217674785, - "mean_ece": 0.021825964776485386, - "mean_auc_drop": 0.0, - "brier_variance": 2.2417204457832686e-05, - "ece_variance": 3.0612803544634866e-06, - "stability": 2.5478484812296173e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.7162750060014119, - "auc_drop": 0.0, - "brier": 0.15210592246344298, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.4711275847227073, - "ece": 0.023798878871191257 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7137578123443502, - "auc_drop": 0.0, - "brier": 0.1595655785898165, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.4888525156352012, - "ece": 0.021642260782289733 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7196508805224282, - "auc_drop": 0.0, - "brier": 0.16500428063296432, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5016630965290716, - "ece": 0.02276416465903103 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7131331724667808, - "auc_drop": 0.0, - "brier": 0.16167384188410222, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4945330118200113, - "ece": 0.019098554793429527 - } - ] - }, - { - "method": "isotonic", - "folds_used": 4, - "mean_brier": 0.1595832553554798, - "mean_log_loss": 0.4896474838692051, - "mean_ece": 0.023323024517531266, - "mean_auc_drop": 0.00019330039380807684, - "brier_variance": 2.1720345584669146e-05, - "ece_variance": 6.450997363829596e-06, - "stability": 2.8171342948498743e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.7159637371312912, - "auc_drop": 0.0003112688701206734, - "brier": 0.15221129127429892, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.47394017921687165, - "ece": 0.026971999727171802 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7134764625309519, - "auc_drop": 0.000281349813398335, - "brier": 0.15965480510372984, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.48879216922395263, - "ece": 0.023471723262675024 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7195381424223168, - "auc_drop": 0.00011273810011136476, - "brier": 0.16494845287785834, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5011860111552263, - "ece": 0.02304525112996469 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7130653276751788, - "auc_drop": 6.784479160193424e-05, - "brier": 0.16151847216603205, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4946715758807695, - "ece": 0.019803123950313542 - } - ] - }, - { - "method": "venn_abers", - "folds_used": 4, - "mean_brier": 0.15956506166210385, - "mean_log_loss": 0.4887236629247538, - "mean_ece": 0.02307409187208089, - "mean_auc_drop": 0.00016360264421683146, - "brier_variance": 2.1781167975507866e-05, - "ece_variance": 6.108658616092844e-06, - "stability": 2.7889826591600712e-05, - "degradation_rate": 0.0, - "folds": [ - { - "fold": 1, - "n_fit": 47516, - "n_eval": 47516, - "raw_auc": 0.7162750060014119, - "cal_auc": 0.71603162066236, - "auc_drop": 0.00024338533905188253, - "brier": 0.15218331951570335, - "brier_raw": 0.2111677009676567, - "brier_degraded": false, - "log_loss": 0.4711075736416691, - "ece": 0.026605278647181308 - }, - { - "fold": 2, - "n_fit": 95032, - "n_eval": 47516, - "raw_auc": 0.7137578123443502, - "cal_auc": 0.7135137022124884, - "auc_drop": 0.00024411013186176334, - "brier": 0.15962815651316078, - "brier_raw": 0.214796281715785, - "brier_degraded": false, - "log_loss": 0.4886894900545325, - "ece": 0.02320141751288713 - }, - { - "fold": 3, - "n_fit": 142548, - "n_eval": 47516, - "raw_auc": 0.7196508805224282, - "cal_auc": 0.7195530155068415, - "auc_drop": 9.786501558672889e-05, - "brier": 0.164934542423762, - "brier_raw": 0.21634688780879766, - "brier_degraded": false, - "log_loss": 0.5011176637036296, - "ece": 0.022866067611790976 - }, - { - "fold": 4, - "n_fit": 190064, - "n_eval": 47516, - "raw_auc": 0.7131331724667808, - "cal_auc": 0.7130641223764138, - "auc_drop": 6.905009036695109e-05, - "brier": 0.16151422819578928, - "brier_raw": 0.21067119337229187, - "brier_degraded": false, - "log_loss": 0.4939799242991838, - "ece": 0.019623603716464152 - } - ] - } - ] - }, - "final_test_metrics": { - "auc_roc": 0.7126809898257414, - "gini": 0.4253619796514827, - "brier_score": 0.15456155709655775, - "log_loss": 0.4769128786416889, - "ece": 0.006731531345207926, - "ks_statistic": 0.3126844787455415, - "pr_auc": 0.3974083356465099, - "recall_at_0p35": 0.3469679539852095, - "f1_at_0p35": 0.3852561470735824, - "d2_brier_score": 0.09864138358717767 - }, - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T05:06:36.012034+00:00", - "dataset_scope": "full_data", - "export_warnings": [ - "HPO trials not found in pd_training_record; exported as 0." - ], - "missing_fields": [ - "hpo_trials_executed" - ] - }, - "pipeline_summary": { - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "pipeline": { - "batch_size": 276869, - "pd_mean": 0.21608988807287652, - "pd_auc": 0.7126809898257414, - "interval_width_mean": 0.7527708083842941, - "stages": { - "S1": 81459, - "S2": 133159, - "S3": 62251 - }, - "ecl_expected": 875935946.2736094, - "ecl_conservative": 1487900897.2686803, - "ecl_range": 611964950.9950709, - "robust_return": 139207.65057267927, - "robust_funded": 235, - "nonrobust_return": 73159.34466177782, - "nonrobust_funded": 152, - "price_of_robustness": -66048.30591090144, - "pipeline_time_s": 274.79 - }, - "pd_model": { - "final_auc": 0.7126809898257414, - "final_gini": 0.4253619796514827, - "final_brier": 0.15456155709655775, - "final_ece": 0.006731531345207926, - "final_d2_brier": 0.09864138358717767, - "calibration_method": "Venn-Abers" - }, - "conformal": { - "coverage_90": 0.9241590788423406, - "coverage_95": 0.9596018333580141, - "overall_pass": true, - "n_checks_passed": 9 - }, - "survival": { - "cox_concordance": 0.6947846585338557, - "rsf_concordance": 0.6948291667286338 - }, - "dataset": { - "n_loans": 500000, - "n_events": 91989, - "event_rate": 0.183978 - }, - "causal": { - "ate": 0.008349399779009324, - "ate_ci": [ - 0.007376764960351769, - 0.009322034597666878 - ], - "cate_mean": 0.008261572107039006, - "cate_std": 0.018584600775105254, - "selected_rule": "discount_100_only", - "total_net_value": 24255084.366237763, - "bootstrap_p05_net": 24170337.916558467, - "avg_action_rate": 0.2054802605581573, - "oot_p05_monthly_net": 1697.3359334449603, - "official_method": { - "identification": "LinearDML with DoWhy audit/refutation", - "heterogeneity": "causal_forest_dml", - "policy_semantics": "research_grade_pricing_intervention" - }, - "run_tag_coherence": { - "expected_run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "observed_run_tags": [ - "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - ], - "observed_by_artifact": { - "causal_effect_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "causal_policy_rule": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "causal_policy_oot_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "cate_portfolio_status": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - }, - "missing_run_tag_artifacts": [], - "mismatched_artifacts": [ - "cate_portfolio_status", - "causal_effect_status", - "causal_policy_oot_status", - "causal_policy_rule" - ], - "coherent": false - } - }, - "explainability": { - "top_global_drivers": [ - "int_rate", - "term", - "home_ownership", - "fico_score", - "emp_length_num" - ], - "n_global_features": 42, - "n_local_cases": 4, - "n_ale_features": 7 - }, - "flattened_summary": { - "pd_auc": 0.7126809898257414, - "pd_gini": 0.4253619796514827, - "pd_brier": 0.15456155709655775, - "pd_ece": 0.006731531345207926, - "pd_d2_brier": 0.09864138358717767, - "calibration_method": "Venn-Abers", - "conformal_coverage_90": 0.9241590788423406, - "conformal_coverage_95": 0.9596018333580141, - "conformal_overall_pass": true, - "survival_cox_concordance": 0.6947846585338557, - "survival_rsf_concordance": 0.6948291667286338, - "dataset_n_loans": 500000, - "dataset_n_events": 91989, - "dataset_event_rate": 0.183978, - "causal_ate": 0.008349399779009324, - "causal_cate_mean": 0.008261572107039006, - "causal_total_net_value": 24255084.366237763, - "causal_bootstrap_p05_net": 24170337.916558467, - "causal_oot_p05_monthly_net": 1697.3359334449603 - }, - "pd_auc": 0.7126809898257414, - "pd_gini": 0.4253619796514827, - "pd_brier": 0.15456155709655775, - "pd_ece": 0.006731531345207926, - "pd_d2_brier": 0.09864138358717767, - "calibration_method": "Venn-Abers", - "conformal_coverage_90": 0.9241590788423406, - "conformal_coverage_95": 0.9596018333580141, - "conformal_overall_pass": true, - "survival_cox_concordance": 0.6947846585338557, - "survival_rsf_concordance": 0.6948291667286338, - "dataset_n_loans": 500000, - "dataset_n_events": 91989, - "dataset_event_rate": 0.183978, - "causal_ate": 0.008349399779009324, - "causal_cate_mean": 0.008261572107039006, - "causal_total_net_value": 24255084.366237763, - "causal_bootstrap_p05_net": 24170337.916558467, - "causal_oot_p05_monthly_net": 1697.3359334449603, - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T05:07:14.529613+00:00", - "dataset_scope": "full_data" - }, - "conformal_status": { - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9293384235866059, - "coverage_95": 0.9623395902033092, - "avg_width_90": 0.764155715329014, - "min_group_coverage_90": 0.9003509420022164, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.193742537952152, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.1305594074149068, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2919.8495778179786, - "p_value": 0.0, - "reject": true, - "n_violations": 19564, - "n_total": 276869, - "violation_rate": 0.07066157641339406, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 966.7782855685655, - "p_value": 0.0, - "reject": true, - "n_violations": 10427, - "n_total": 276869, - "violation_rate": 0.03766040979669085, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2919.8495778179786, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 2.290243347553769, - "p_ind": 0.13018950970882026, - "reject_ind": false, - "lr_cc": 2922.1398211655323, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239175, - "n01": 18129, - "n10": 18129, - "n11": 1435 - } - }, - "christoffersen_95": { - "lr_uc": 966.7782855685655, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.07731377833988518, - "p_ind": 0.7809712495852495, - "reject_ind": false, - "lr_cc": 966.8555993469054, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 256412, - "n01": 10029, - "n10": 10029, - "n11": 398 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07066157641339406, - "violation_rate_95": 0.03766040979669085, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.13018950970882026, - "christoffersen_independence_pvalue_95": 0.7809712495852495 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.029338423586605877, - "coverage_deviation_95": 0.012339590203309214, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "configs/conformal_policy.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T06:13:27.444684+00:00", - "run_tag": "conformal-reopen-rank1-promotion-2026-04-07" - }, - "fairness_status": { - "overall_pass": true, - "n_attributes": 6, - "n_base_attributes": 3, - "n_intersectional_attributes": 3, - "n_passed": 6, - "attributes": [ - { - "attribute": "home_ownership", - "dpd": 0.006889830359128268, - "eo_gap": 0.01770489363145389, - "dir": 0.9931052556725157, - "tpr_gap": 0.0026810291101042605, - "fpr_gap": 0.01770489363145389, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "annual_inc_quartile", - "dpd": 0.006470025746612151, - "eo_gap": 0.017440762137960086, - "dir": 0.9935278114659133, - "tpr_gap": 0.0025382505294188062, - "fpr_gap": 0.017440762137960086, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "verification_status", - "dpd": 0.008941886950530664, - "eo_gap": 0.019226656076939297, - "dir": 0.9910447580509565, - "tpr_gap": 0.004008505944615881, - "fpr_gap": 0.019226656076939297, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "base" - }, - { - "attribute": "home_ownership__x__annual_inc_quartile", - "dpd": 0.011476463365722056, - "eo_gap": 0.029276578737010395, - "dir": 0.9885232793779202, - "tpr_gap": 0.0042916403722097485, - "fpr_gap": 0.029276578737010395, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "home_ownership__x__verification_status", - "dpd": 0.026822101739420012, - "eo_gap": 0.05026883471246557, - "dir": 0.9731729626463925, - "tpr_gap": 0.01270121157775772, - "fpr_gap": 0.05026883471246557, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - }, - { - "attribute": "annual_inc_quartile__x__verification_status", - "dpd": 0.01626364901860511, - "eo_gap": 0.036263736263736246, - "dir": 0.9837351461327805, - "tpr_gap": 0.007530002041792994, - "fpr_gap": 0.036263736263736246, - "passed_dpd": true, - "passed_eo": true, - "passed_dir": true, - "passed_all": true, - "attribute_type": "intersectional" - } - ], - "prediction_threshold": 0.35, - "primary_threshold": 0.35, - "prediction_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval", - "thresholds": { - "dpd": 0.1, - "eo_gap": 0.11, - "dir": 0.8 - }, - "threshold_frontier": { - "path": "data/processed/fairness_threshold_frontier.parquet", - "thresholds": [ - 0.35, - 0.4, - 0.45, - 0.5 - ], - "worst_primary_attribute": "home_ownership__x__verification_status", - "selected_threshold": 0.35, - "all_primary_pass": true - }, - "decision_policy": { - "path": "models/fairness_decision_policy.json", - "global_threshold": 0.35, - "n_overrides": 0 - }, - "policy_config": "configs/fairness_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T04:03:18.399296+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2" - }, - "fairness_decision_policy": { - "global_threshold": 0.35, - "overrides": [], - "selection": { - "source": "fairness_frontier_auto_select", - "n_passed": 6, - "worst_eo_gap": 0.05026883471246557, - "approval_rate": 0.9966373989142879 - }, - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T04:03:16.014535+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2" - }, - "governance_status": { - "overall_pass": true, - "checks": { - "pass_psi": true, - "pass_breach_ratio": true, - "pass_score_psi": true, - "pass_auc_delta": true, - "pass_brier_increase": true, - "pass_calibration_gap_delta": true, - "pass_predictive_drift": true, - "pass_fairness": true, - "pass_c2st": false, - "pass_explainability": true, - "pass_reason_code_stability": true, - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "thresholds": { - "psi_threshold": 0.25, - "ks_pvalue_min": 0.01, - "cvm_pvalue_min": 0.01, - "c2st_auc_max": 0.6, - "max_feature_breach_ratio": 0.15, - "score_psi_max": 0.15, - "auc_delta_max": 0.05, - "brier_increase_max": 0.02, - "calibration_gap_delta_max": 0.02, - "explanation_rank_overlap_top10_min": 0.6, - "explanation_shap_psi_max": 0.25, - "reason_code_stability_min": 0.55 - }, - "summary": { - "n_features": 33, - "max_psi": 0.13944722694322031, - "mean_psi": 0.024499996173148176, - "min_ks_pvalue": 0.0, - "min_cvm_pvalue": 8.80828621152574e-09, - "c2st_auc": 0.9853501044444442, - "psi_breaches": 0, - "ks_breaches": 32, - "cvm_breaches": 33, - "feature_breach_ratio": 0.0, - "distribution_warning_ratio": 0.9848484848484849, - "c2st_rows_used": 100000, - "c2st_materiality": "severe", - "c2st_effective_driver_count": 1, - "score_psi": 0.013842775323210633, - "auc_train_reference": 0.7452259354910183, - "auc_test_oot": 0.7118120844793607, - "auc_delta_train_to_test": 0.033413851011657614, - "brier_train_reference": 0.13811101140845075, - "brier_test_oot": 0.1550062316178897, - "brier_increase_train_to_test": 0.016895220209438955, - "calibration_gap_train_reference": 0.0303166024308566, - "calibration_gap_test_oot": 0.00439340612496672, - "calibration_gap_delta": 0.0, - "train_eval_rows": 100000, - "test_eval_rows": 100000, - "n_explanation_segments": 1, - "min_rank_overlap_top10": 0.9, - "max_explanation_shap_psi": 0.06104327913317865, - "min_reason_code_stability": 1.0, - "fairness_overall_pass": true, - "fairness_primary_threshold": 0.35, - "challenger_promotable": false, - "model_shift_type": "mixed_shift", - "governance_posture": "candidate_gate" - }, - "warnings": { - "warn_c2st": true, - "warn_distribution_tests": true, - "warn_explainability": false, - "warn_reason_code_stability": false - }, - "c2st": { - "auc": 0.9853501044444442, - "materiality": "severe", - "effective_driver_count": 1, - "top_drivers": [ - { - "feature": "int_rate", - "importance_auc": 0.4170757822222219 - }, - { - "feature": "loan_to_income", - "importance_auc": 0.00718282296296265 - }, - { - "feature": "pub_rec", - "importance_auc": 0.004748391851851608 - }, - { - "feature": "has_bankruptcy", - "importance_auc": 0.003179000740740475 - }, - { - "feature": "loan_amnt", - "importance_auc": 0.0025205829629626844 - }, - { - "feature": "fico_score", - "importance_auc": 0.0018507688888886875 - }, - { - "feature": "inq_last_6mths", - "importance_auc": 0.0005079955555552887 - }, - { - "feature": "term", - "importance_auc": 0.000483162962962654 - }, - { - "feature": "credit_age_years", - "importance_auc": 0.000386952592592323 - }, - { - "feature": "revol_bal_to_income", - "importance_auc": 0.000345855555555256 - } - ] - }, - "model_shift": { - "shift_type": "mixed_shift", - "structural_shift_level": "severe", - "predictive_degradation_level": "moderate", - "governance_posture": "candidate_gate", - "c2st_materiality": "severe", - "pvalue_interpretation": "Both structural shift and predictive degradation are present. This combination deserves the strongest governance posture." - }, - "artifacts": { - "drift_monitoring_path": "data/processed/drift_monitoring.parquet", - "explanation_drift_path": "data/processed/explanation_drift.parquet", - "fairness_status_path": "models/fairness_audit_status.json", - "fairness_frontier_path": "data/processed/fairness_threshold_frontier.parquet", - "challenger_promotion_report_path": "models/challenger_promotion_report.json", - "model_shift_status_path": "models/model_shift_status.json" - }, - "top_drift_features": [ - { - "feature": "fico_score", - "train_n": 798550, - "test_n": 276869, - "psi": 0.13944722694322031, - "ks_statistic": 0.15457282396927496, - "ks_pvalue": 0.0, - "cvm_statistic": 3230.128652537591, - "cvm_pvalue": 4.865294239708717e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "int_rate", - "train_n": 798550, - "test_n": 276869, - "psi": 0.10118060476101726, - "ks_statistic": 0.09481768715722516, - "ks_pvalue": 0.0, - "cvm_statistic": 398.7076133590017, - "cvm_pvalue": 1.135480649505638e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.05347961324875766, - "ks_statistic": 0.09122311383287474, - "ks_pvalue": 0.0, - "cvm_statistic": 909.8105939357774, - "cvm_pvalue": 2.1943404671098676e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_severity", - "train_n": 798550, - "test_n": 276869, - "psi": 0.05178332227166686, - "ks_statistic": 0.0916698979201469, - "ks_pvalue": 0.0, - "cvm_statistic": 44606.0478113642, - "cvm_pvalue": 2.134860395242022e-06, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "log_revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "revol_bal", - "train_n": 798550, - "test_n": 276869, - "psi": 0.045158456442640646, - "ks_statistic": 0.07714231680126768, - "ks_pvalue": 0.0, - "cvm_statistic": 581.7568173016189, - "cvm_pvalue": 1.863552576697458e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670529338066, - "cvm_pvalue": 4.4769498996899415e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "loan_to_income_sq", - "train_n": 798482, - "test_n": 276157, - "psi": 0.040709068313852514, - "ks_statistic": 0.05937315877269134, - "ks_pvalue": 0.0, - "cvm_statistic": 200.07670714583946, - "cvm_pvalue": 4.476950465903684e-08, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "delinq_recency", - "train_n": 798550, - "test_n": 276869, - "psi": 0.038966862981116755, - "ks_statistic": 0.08135488943770064, - "ks_pvalue": 0.0, - "cvm_statistic": 11755.072386931657, - "cvm_pvalue": 9.791550346083255e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - }, - { - "feature": "credit_age_years", - "train_n": 798550, - "test_n": 276869, - "psi": 0.029775277868375027, - "ks_statistic": 0.054973967117968536, - "ks_pvalue": 0.0, - "cvm_statistic": 290.1511620187666, - "cvm_pvalue": 1.0223037416601244e-07, - "pass_psi": true, - "pass_ks": false, - "pass_cvm": false - } - ], - "top_explanation_breaches": [ - { - "segment_type": "overall", - "segment": "all", - "reference_period": "2018Q1|2018Q2|2018Q3|2018Q4|2019Q1|2019Q2|2019Q3", - "comparison_period": "2019Q4|2020Q1|2020Q2|2020Q3", - "reference_n": 4763, - "comparison_n": 237, - "rank_overlap_top10": 0.9, - "avg_shap_psi_top5": 0.044564149041987314, - "max_shap_psi_top5": 0.06104327913317865, - "reason_code_match_rate": 1.0, - "pass_rank_overlap": true, - "pass_distribution_shift": true, - "pass_reason_code_stability": true, - "passed_all": true, - "feature_psi_details": "[{\"feature\": \"int_rate\", \"psi\": 0.0571152296148128}, {\"feature\": \"term\", \"psi\": 0.05398847136247015}, {\"feature\": \"fico_score\", \"psi\": 0.06104327913317865}, {\"feature\": \"home_ownership\", \"psi\": 0.03675845025796413}, {\"feature\": \"many_recent_opens\", \"psi\": 0.013915314841510859}]", - "reason_code_details": "[{\"band\": \"low\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}, {\"band\": \"near_threshold\", \"reference_reason\": \"int_rate\", \"comparison_reason\": \"int_rate\", \"matched\": true}]" - } - ], - "primary_threshold": 0.35, - "explainability_pass": true, - "explanation_drift_pass": true, - "reason_code_stability_pass": true, - "challenger_promotable": false, - "policy_config": "configs/mrm_policy.yaml", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T04:04:49.130469+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2" - }, - "survival_summary": { - "cox_concordance_index": 0.6947846585338557, - "rsf_c_index_test": 0.6948291667286338, - "cox_training_time": 18.711559661023784, - "rsf_training_time": 6457.35460821999, - "n_loans": 500000, - "n_events": 91989, - "event_rate": 0.183978, - "median_time_default": 17.017082785808146, - "median_time_censored": 28.02233902759527, - "cox_features": [ - "loan_amnt", - "int_rate", - "installment", - "annual_inc", - "dti", - "loan_to_income", - "installment_burden", - "rev_utilization", - "revol_bal_to_income", - "open_acc_ratio", - "fico_score", - "credit_age_years", - "emp_length_num", - "open_acc", - "total_acc", - "revol_bal", - "pub_rec", - "delinq_severity", - "delinq_recency", - "il_ratio", - "high_util_pct", - "log_annual_inc", - "log_revol_bal", - "loan_to_income_sq", - "fico_x_dti", - "has_delinq_2yrs", - "has_pub_rec", - "has_bankruptcy", - "has_recent_inq", - "has_mortgage", - "many_recent_opens", - "recent_chargeoff", - "bc_util", - "bc_open_to_buy", - "percent_bc_gt_75", - "acc_open_past_24mths", - "tot_cur_bal", - "tot_hi_cred_lim", - "total_bal_ex_mort", - "total_bc_limit", - "total_il_high_credit_limit", - "avg_cur_bal", - "pct_tl_nvr_dlq", - "mths_since_recent_bc", - "num_accts_ever_120_pd", - "num_actv_bc_tl", - "num_actv_rev_tl", - "num_bc_sats", - "num_bc_tl", - "num_il_tl", - "num_op_rev_tl", - "num_rev_accts", - "num_rev_tl_bal_gt_0", - "num_sats", - "num_tl_30dpd", - "num_tl_90g_dpd_24m", - "num_tl_op_past_12m", - "mo_sin_old_il_acct", - "mo_sin_old_rev_tl_op", - "mo_sin_rcnt_rev_tl_op", - "mo_sin_rcnt_tl" - ], - "rsf_sample_size": 500000, - "rsf_artifact": { - "saved": false, - "size_mb": 23932.23, - "placeholder": true, - "reason": "max_size_exceeded" - }, - "dataset_scope": "full_data", - "sample_size_requested": null, - "rsf_params": { - "n_estimators": 200, - "min_samples_leaf": 5, - "max_depth": null, - "max_samples": 0.5, - "n_jobs": 12 - } - }, - "ifrs9_summary": { - "baseline": { - "total_ecl": 875935946.2736094 - }, - "mild_stress": { - "total_ecl": 1033031290.9377306 - }, - "adverse": { - "total_ecl": 1220234278.8044748 - }, - "severe": { - "total_ecl": 1487900897.2686803 - } - }, - "portfolio_robustness_summary": [ - { - "risk_tolerance": 0.05, - "baseline_nonrobust_return": 14673.299659652834, - "baseline_nonrobust_realized_return": -641.9403223462718, - "best_robust_return": 102800.36040543186, - "best_robust_realized_return": 86867.44040545436, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.05, - "best_robust_pd_cap_slack_penalty": 1.5, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.2342473067307094, - "best_robust_funded": 194, - "baseline_nonrobust_funded": 34, - "price_of_robustness": -88127.06074577902, - "price_of_robustness_pct": -600.5947045578031, - "ab_diff_total_return": 87509.38072780063, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.06, - "baseline_nonrobust_return": 60610.43389418064, - "baseline_nonrobust_realized_return": 36903.007122105235, - "best_robust_return": 110822.13617489807, - "best_robust_realized_return": 96191.33117492504, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.05, - "best_robust_pd_cap_slack_penalty": 1.5, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.27412392884735237, - "best_robust_funded": 203, - "baseline_nonrobust_funded": 137, - "price_of_robustness": -50211.70228071743, - "price_of_robustness_pct": -82.84333084886552, - "ab_diff_total_return": 59288.32405281981, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.08, - "baseline_nonrobust_return": 67801.50513800641, - "baseline_nonrobust_realized_return": 40007.78523131351, - "best_robust_return": 125722.68685574384, - "best_robust_realized_return": 103432.75185577983, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.05, - "best_robust_pd_cap_slack_penalty": 1.5, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.35559441154158145, - "best_robust_funded": 223, - "baseline_nonrobust_funded": 140, - "price_of_robustness": -57921.18171773743, - "price_of_robustness_pct": -85.42757509436942, - "ab_diff_total_return": 63424.966624466324, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.1, - "baseline_nonrobust_return": 73159.34466177782, - "baseline_nonrobust_realized_return": 47373.34739448527, - "best_robust_return": 139207.65057267927, - "best_robust_realized_return": 121036.16557272426, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.05, - "best_robust_pd_cap_slack_penalty": 1.5, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.43170023737439667, - "best_robust_funded": 235, - "baseline_nonrobust_funded": 152, - "price_of_robustness": -66048.30591090144, - "price_of_robustness_pct": -90.28006772797906, - "ab_diff_total_return": 73662.81817823899, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.12, - "baseline_nonrobust_return": 78261.32156249504, - "baseline_nonrobust_realized_return": 50141.63823237712, - "best_robust_return": 152497.42593257467, - "best_robust_realized_return": 150318.14093262874, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.05, - "best_robust_pd_cap_slack_penalty": 1.5, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.5123003000677335, - "best_robust_funded": 254, - "baseline_nonrobust_funded": 161, - "price_of_robustness": -74236.10437007963, - "price_of_robustness_pct": -94.85669662484595, - "ab_diff_total_return": 100176.50270025161, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.14, - "baseline_nonrobust_return": 83001.06004968996, - "baseline_nonrobust_realized_return": 62481.695735424415, - "best_robust_return": 165061.75044281548, - "best_robust_realized_return": 179647.6554428785, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.0, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.0, - "best_robust_pd_cap_slack_penalty": 0.0, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.5913972958158074, - "best_robust_funded": 267, - "baseline_nonrobust_funded": 173, - "price_of_robustness": -82060.69039312552, - "price_of_robustness_pct": -98.8670389787912, - "ab_diff_total_return": 117165.95970745408, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.16, - "baseline_nonrobust_return": 87518.8462970579, - "baseline_nonrobust_realized_return": 73263.73554135203, - "best_robust_return": 176249.98526942707, - "best_robust_realized_return": 181892.66526949903, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "segment_tail_blended_uncertainty", - "best_robust_gamma": 0.05, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.0, - "best_robust_pd_cap_slack_penalty": 0.0, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.6671546616077108, - "best_robust_funded": 285, - "baseline_nonrobust_funded": 175, - "price_of_robustness": -88731.13897236917, - "price_of_robustness_pct": -101.38517899354231, - "ab_diff_total_return": 108628.929728147, - "ab_pass": true, - "selected_for_champion": false - }, - { - "risk_tolerance": 0.18, - "baseline_nonrobust_return": 91981.8929666001, - "baseline_nonrobust_realized_return": 78726.36209958795, - "best_robust_return": 176129.73709686322, - "best_robust_realized_return": 188082.30554512228, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "segment_relative_tail_blended_uncertainty", - "best_robust_gamma": 0.05, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.0, - "best_robust_pd_cap_slack_penalty": 0.0, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.6661215620172041, - "best_robust_funded": 268, - "baseline_nonrobust_funded": 176, - "price_of_robustness": -84147.84413026311, - "price_of_robustness_pct": -91.48305325690951, - "ab_diff_total_return": 109355.94344553433, - "ab_pass": true, - "selected_for_champion": true - }, - { - "risk_tolerance": 0.2, - "baseline_nonrobust_return": 96270.04147595653, - "baseline_nonrobust_realized_return": 78205.96353334474, - "best_robust_return": 171435.95556487225, - "best_robust_realized_return": 179921.47296238175, - "best_robust_lambda": 0.25, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.1, - "best_robust_delta_cap_quantile": 1.0, - "best_robust_min_budget_utilization": 0.0, - "best_robust_pd_cap_slack_penalty": 0.0, - "best_robust_pd_cap_slack": 0.0, - "best_robust_worst_pd": 0.6324817520511766, - "best_robust_funded": 278, - "baseline_nonrobust_funded": 177, - "price_of_robustness": -75165.91408891571, - "price_of_robustness_pct": -78.07819850883479, - "ab_diff_total_return": 101715.50942903702, - "ab_pass": true, - "selected_for_champion": false - } - ], - "ab_simulation_status": { - "strategy_a": "non_robust", - "strategy_b": "robust_selected_for_champion", - "comparison": { - "mean_a": 29.281061874477036, - "mean_b": 25.866592800653414, - "diff": -3.4144690738236214, - "ci_low": -29.252002019950044, - "ci_high": 22.500691623795504, - "p_value": 0.3952, - "significant": false - }, - "metrics_a": { - "total_return": 87843.18562343111, - "n_funded": 143, - "total_allocated": 1000000.0, - "avg_return_per_funded": 614.2880113526651 - }, - "metrics_b": { - "total_return": 77599.77840196024, - "n_funded": 118, - "total_allocated": 1000000.0, - "avg_return_per_funded": 657.6252406945783 - }, - "n_candidates_available": 3000, - "n_candidates_used": 3000, - "max_candidates_requested": 150000, - "dataset_scope": "sampled_candidates", - "solver_backend": "highs", - "policy_selector": "explicit_champion_only", - "decision_scenario": "baseline", - "max_portfolio_pd_requested": 0.18, - "max_portfolio_pd_effective": 0.175, - "robust_policy": { - "source": "champion_policy_artifact::explicit_champion_only", - "risk_tolerance": 0.175, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0 - }, - "champion_policy_path": "models/champion_portfolio_policy.json", - "candidate_universe_path": "data/processed/champion_candidate_universe.parquet", - "gate_contract": { - "gate": "no_regression", - "significance_role": "diagnostic" - }, - "diagnostics": { - "p_value": 0.3952, - "significant": false, - "n_boot": 5000, - "seed": 42 - }, - "policy_search": [], - "frontier_path": "data/processed/portfolio_robustness_frontier.parquet", - "no_regression": { - "diff_total_return": -10243.407221470872, - "tolerance_total_return": 4392.159281171555, - "tolerance_pct_of_control": 0.05, - "passed": false - }, - "decision_scenario_meta": { - "decision_scenario": "baseline", - "rows_removed": 0, - "rows_remaining": 3000, - "ambiguity_rate_removed": 0.0 - }, - "baseline_comparison_context": { - "artifact_truth_role": "current_run_status", - "official_truth_may_live_in_comparison_json": true - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T04:02:03.889964+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2" - }, - "causal_effect_status": { - "dataset_scope": "full_data", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:12.527380+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "treatment": "int_rate", - "treatment_unit": "percentage_points", - "identified_estimand": "linear_dml_const_marginal_effect", - "identification_strategy": "orthogonal_dml_with_dowhy_audit", - "ate": 0.008349399779009324, - "ate_ci": [ - 0.007376764960351769, - 0.009322034597666878 - ], - "ate_audit_dowhy": { - "ate": 0.01393374819424989, - "ate_ci": [ - 0.013556213125054806, - 0.01431128326344496 - ], - "identified_estimand": "Estimand type: EstimandType.NONPARAMETRIC_ATE\n\n### Estimand : 1\nEstimand name: backdoor\nEstimand expression:\n d ↪\n──────────(E[default_flag|annual_inc,purpose_woe,home_ownership_woe,fico_range ↪\nd[intᵣₐₜₑ] ↪\n\n↪ \n↪ _low,loan_amnt,grade_woe,dti])\n↪ \nEstimand assumption 1, Unconfoundedness: If U→{int_rate} and U→default_flag then P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti,U) = P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti)\n\n### Estimand : 2\nEstimand name: iv\nNo such variable(s) found!\n\n### Estimand : 3\nEstimand name: frontdoor\nNo such variable(s) found!\n\n### Estimand : 4\nEstimand name: general_adjustment\nEstimand expression:\n d ↪\n──────────(E[default_flag|annual_inc,purpose_woe,home_ownership_woe,fico_range ↪\nd[intᵣₐₜₑ] ↪\n\n↪ \n↪ _low,loan_amnt,grade_woe,dti])\n↪ \nEstimand assumption 1, Unconfoundedness: If U→{int_rate} and U→default_flag then P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti,U) = P(default_flag|int_rate,annual_inc,purpose_woe,home_ownership_woe,fico_range_low,loan_amnt,grade_woe,dti)\n", - "identification_strategy": "backdoor" - }, - "cate_mean": 0.008261572107039006, - "cate_std": 0.018584600775105254, - "effect_modifiers": [ - "loan_amnt", - "annual_inc", - "dti", - "fico_range_low" - ], - "confounders": [ - "grade_woe", - "purpose_woe", - "home_ownership_woe" - ], - "ate_controls": [ - "annual_inc", - "dti", - "fico_range_low", - "grade_woe", - "home_ownership_woe", - "loan_amnt", - "purpose_woe" - ], - "refutation_summary": [ - { - "test": "placebo_treatment", - "estimated_effect": 0.01393374819424989, - "new_effect": 1.1313914378179579e-05, - "p_value": 0.8600000000000001, - "result": "Refute: Use a Placebo Treatment\nEstimated effect:0.01393374819424989\nNew effect:1.1313914378179579e-05\np value:0.8600000000000001\n" - }, - { - "test": "random_common_cause", - "estimated_effect": 0.01393374819424989, - "new_effect": 0.013933751461074069, - "p_value": 0.94, - "result": "Refute: Add a random common cause\nEstimated effect:0.01393374819424989\nNew effect:0.013933751461074069\np value:0.94\n" - }, - { - "test": "data_subset", - "estimated_effect": 0.01393374819424989, - "new_effect": 0.013939064210410735, - "p_value": 0.9199999999999999, - "result": "Refute: Use a subset of data\nEstimated effect:0.01393374819424989\nNew effect:0.013939064210410735\np value:0.9199999999999999\n" - } - ], - "n_obs": 1346311, - "n_rows_input": 1346311, - "n_rows_dropped_nonfinite": 0, - "drop_rate": 0.0, - "n_imputed_cells": 74, - "source_train_split": "data/processed/train_fe.parquet", - "source_test_split": "data/processed/test_fe.parquet", - "cate_artifact_path": "data/processed/cate_estimates.parquet", - "oot_cate_artifact_path": "data/processed/cate_estimates_oot.parquet", - "overlap_artifact_path": "data/processed/causal_overlap_diagnostics.parquet", - "overlap_pass": true, - "sensitivity_pass": false, - "identification_valid": true, - "missing_required_columns": [], - "continuous_treatment_semantics": { - "estimand": "const_marginal_effect", - "interpretation": "default_probability_delta_per_1pp_rate_change", - "policy_safe": false - }, - "policy_value_method": "local_cate_discrete_grid", - "policy_evaluation_consistent": false, - "role": "insights_only", - "promotion_eligible": false, - "promotion_state": "insights_only", - "estimator_family": "causal_forest_dml", - "ate_estimator_family": "linear_dml", - "official_method": { - "identification": "LinearDML with DoWhy audit/refutation", - "heterogeneity": "causal_forest_dml", - "policy_semantics": "research_grade_pricing_intervention" - }, - "environment_status": { - "environment": "causal_lane", - "compatible": true, - "packages": { - "dowhy": { - "installed": "0.14", - "expected": ">=0.14,<0.15", - "compatible": true - }, - "econml": { - "installed": "0.16.0", - "expected": ">=0.16,<0.17", - "compatible": true - }, - "statsmodels": { - "installed": "0.14.6", - "expected": ">=0.14,<0.15", - "compatible": true - }, - "scikit-learn": { - "installed": "1.6.1", - "expected": ">=1.0,<1.7", - "compatible": true - }, - "shap": { - "installed": "0.48.0", - "expected": ">=0.38.1,<0.49", - "compatible": true - } - } - } - }, - "causal_policy_rule_status": { - "selection_reason": "best_feasible", - "selected_rule": "discount_100_only", - "selected_metrics": { - "rule_name": "discount_100_only", - "n_selected": 286409.0, - "action_rate": 0.21273613600423677, - "total_net_value": 24255084.366237763, - "total_loss_reduction": 69817578.61623776, - "total_revenue_impact": -45562494.25, - "bootstrap_mean_net": 24256557.58078567, - "bootstrap_p05_net": 24170337.916558467, - "bootstrap_p95_net": 24340805.21273278, - "min_grade_total_net": 304163.96874709026, - "worst_grade": "G", - "pass_action_rate": true, - "pass_bootstrap": true, - "pass_grade_floor": true, - "pass_all": true - }, - "constraints": { - "max_action_rate": 0.35, - "min_bootstrap_p05_net": 0.0, - "min_grade_total_net": 0.0 - }, - "source_simulation_path": "data/processed/causal_policy_simulation.parquet", - "source_effect_status_path": "models/causal_effect_status.json", - "effect_status_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "policy_semantics": "research_grade_pricing_intervention", - "policy_value_method": "local_cate_discrete_grid", - "role": "insights_only", - "promotion_eligible": false, - "promotion_state": "validated_research_policy", - "promotion_decider": "validate_causal_policy.py", - "policy_evaluation_consistent": false, - "overlap_pass": true, - "sensitivity_pass": false, - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:21.052543+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - }, - "causal_policy_oot_status": { - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:27.886061+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "rule_name": "discount_100_only", - "min_history_months": 12, - "n_months_evaluated": 106, - "avg_action_rate": 0.2054802605581573, - "total_net_value": 24239407.18821239, - "p05_monthly_net": 1697.3359334449603, - "worst_month": "2008-09-01 00:00:00", - "best_month": "2016-03-01 00:00:00", - "selected_rule_path": "models/causal_policy_rule.json", - "effect_status_path": "models/causal_effect_status.json", - "policy_semantics": "research_grade_pricing_intervention", - "policy_value_method": "local_cate_discrete_grid", - "promotion_state": "validated_research_policy" - }, - "cate_portfolio_status": { - "delta_rate": -1.0, - "baseline_objective": null, - "cate_adjusted_objective": null, - "baseline_n_funded": null, - "cate_adjusted_n_funded": null, - "objective_change_pct": null, - "n_candidates_available": 276869, - "n_candidates_used": 0, - "max_candidates_requested": 150000, - "dataset_scope": "blocked_by_policy_gate", - "solver_backend": "highs", - "feasible_baseline": false, - "feasible_adjusted": false, - "cate_policy_mode": "research_blocked_by_policy_gate", - "promotion_eligible": false, - "promotion_state": "research_blocked_by_policy_gate", - "fallback_applied": true, - "raw_objective_change_pct": null, - "shrunk_objective_change_pct": null, - "alignment_strategy": null, - "source_cate_artifact": null, - "n_missing_cate": null, - "cate_shrink": {}, - "constraint_binding_reason": "research_blocked_by_policy_gate", - "role": "insights_only", - "promotion_decider": "causal_policy_rule.json", - "policy_evaluation_consistent": false, - "warning": "CATE portfolio blocked because the causal policy did not pass overlap/sensitivity/policy gates.", - "source_effect_status_path": "models/causal_effect_status.json", - "source_policy_rule_path": "models/causal_policy_rule.json", - "effect_status_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "policy_rule_run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:30.521517+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - }, - "conformal_lgd_ead_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "threshold_semantics": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-07T06:08:38.791418+00:00", - "run_tag": "champion-hpo-local-t56-promotion-2026-04-07", - "pd_internal_selected_threshold": 0.05, - "pd_internal_fallback_threshold": 0.35, - "fairness_primary_threshold": 0.35, - "decision_policy_global_threshold": 0.35, - "source_artifacts": { - "fairness_status": "models/fairness_audit_status.json", - "fairness_decision_policy": "models/fairness_decision_policy.json", - "fairness_frontier": "data/processed/fairness_threshold_frontier.parquet", - "decision_threshold": "models/decision_threshold.json", - "decision_threshold_v2": "models/decision_threshold_v2.json", - "test_predictions": "data/processed/test_predictions.parquet" - }, - "business_meaning": { - "pd_internal_selected_threshold": "Threshold interno de screening/seleccion PD usado en busqueda y analisis tecnico.", - "pd_internal_fallback_threshold": "Fallback interno tomado de la policy de fairness durante la busqueda de threshold PD.", - "fairness_primary_threshold": "Threshold operativo principal para auditoria de fairness y narrativa de aprobacion.", - "decision_policy_global_threshold": "Threshold operativo global aplicado por la policy de decision/aprobacion." - }, - "pd_internal_threshold_source": "validation_fairness_search", - "calibration_method": "venn_abers" - }, - "time_series_status": { - "schema_version": "2026-03-07.1", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "status": "warn", - "warnings": [ - "interval_champion_not_promotable", - "exogenous_available_but_disabled_by_policy" - ], - "config": { - "horizon": 12, - "freq": "MS", - "season_length": 12, - "rolling_origin": { - "min_train_periods": 72, - "step_months": 1, - "embargo_periods": 12, - "max_windows": 36 - }, - "exogenous_enabled": false - }, - "summary": { - "n_models_evaluated": 6, - "n_backtest_rows": 2592, - "point_model": "AutoARIMA", - "interval_model": "AutoARIMA", - "point_promotable": true, - "interval_promotable": false, - "recent_actual_mean_12m": 0.026911907006801378 - }, - "point_champion": { - "model": "AutoARIMA", - "promotable": true, - "reasons": [], - "mae": 0.049873139227617436, - "mase": 1.9066089897100431, - "rmsse": 1.5570058706800392, - "abs_bias": 0.048018836438593226 - }, - "interval_champion": { - "model": "AutoARIMA", - "promotable": false, - "reasons": [ - "no_model_within_coverage_gap_policy" - ], - "coverage_90": 0.8101851851851852, - "coverage_gap_90": 0.08981481481481479, - "winkler_90": 0.35975890823617446, - "wis_90": 0.07804457277384988, - "pinball_90": 0.008993972705904364, - "avg_interval_width_90": 0.17404434598567664, - "family": "statistical", - "interval_subfamily": "native_statistical" - }, - "panel_global_model": { - "available": true, - "method": "bottom_up_counts_with_global_catboost", - "n_bottom_series": 14, - "n_forecast_rows": 297 - }, - "diagnostics": { - "forecastability": { - "schema_version": "2026-03-07.1", - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "n_periods": 160, - "series_start": "2007-06-01", - "series_end": "2020-09-01", - "recent_actual_mean_12m": 0.026911907006801378, - "adf": { - "statistic": -1.9765602949487615, - "p_value": 0.2969442042501646, - "lags_used": 13 - }, - "kpss": { - "statistic": 0.25753454841249984, - "p_value": 0.1, - "lags_used": 8 - }, - "acf": { - "lag_1": 0.8349981729279056, - "lag_2": 0.7459033829126462, - "lag_3": 0.6874916772827628, - "lag_4": 0.6240369599316403, - "lag_5": 0.5635390436715605, - "lag_6": 0.5053567057440341, - "lag_7": 0.472785952224223, - "lag_8": 0.39282805197501364, - "lag_9": 0.3169792848261985, - "lag_10": 0.2531242623788613, - "lag_11": 0.24157832454873526, - "lag_12": 0.1604446770648665 - }, - "pacf": { - "lag_1": 0.8402497337639301, - "lag_2": 0.16778528544014598, - "lag_3": 0.11010702609123724, - "lag_4": 0.0069389561419676725, - "lag_5": -0.007538230205843105, - "lag_6": -0.020816650426219923, - "lag_7": 0.06304032265221003, - "lag_8": -0.1548066587007755, - "lag_9": -0.09023881373390952, - "lag_10": -0.05817758714140727, - "lag_11": 0.15329868994109364, - "lag_12": -0.2048329868372945 - }, - "stl": { - "seasonal_strength": 0.21165919268077193, - "trend_strength": 0.8078373840115486 - }, - "variance_ratio": { - "k": 12, - "value": 0.4236395667454792 - }, - "spectral_entropy": 0.5559378570303748, - "permutation_entropy": 0.9903056294119694 - }, - "residual_drift": { - "ks_statistic": 0.7731481481481481, - "p_value": 6.274375257076014e-64, - "early_mean": 0.022589284820660756, - "late_mean": -0.07240765274862622 - } - }, - "research_backlog": [ - "ACI", - "EnbPI", - "OnlineConformal" - ], - "artifacts": { - "config_path": "configs/time_series.yaml", - "time_series_full_path": "data/processed/time_series_full.parquet", - "time_series_panel_path": "data/processed/time_series_panel.parquet", - "backtest_predictions_path": "data/processed/ts_backtest_predictions.parquet", - "backtest_metrics_path": "data/processed/ts_backtest_metrics.parquet", - "forecasts_path": "data/processed/ts_forecasts.parquet", - "scenarios_path": "data/processed/ts_ifrs9_scenarios.parquet", - "diagnostics_path": "data/processed/ts_diagnostics.json", - "panel_forecasts_path": "data/processed/ts_panel_forecasts.parquet", - "status_path": "models/time_series_status.json", - "forecastability_report_path": "data/processed/ts_forecastability_report.parquet", - "forecastability_status_path": "models/time_series_forecastability_status.json", - "hierarchical_eval_path": "data/processed/ts_hierarchical_eval.parquet", - "hierarchy_status_path": "models/time_series_hierarchy_status.json", - "interval_eval_path": "data/processed/ts_interval_eval.parquet" - }, - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "forecastability_summary": { - "available": true, - "series_evaluated": 22, - "routes": { - "intermittent_counts": 14, - "exogenous_challenger": 5, - "structured_statistical": 3 - }, - "intermittency": { - "erratic": 21, - "lumpy": 1 - }, - "levels": { - "grade_term": 14, - "grade": 7, - "portfolio": 1 - } - }, - "interval_benchmark": { - "available": true, - "estimator_name": "linear", - "best_method": "enbpi", - "candidate_methods_tested": [ - "enbpi", - "aci" - ], - "results": [ - { - "method": "enbpi", - "n_eval": 24, - "coverage_90": 0.8333333333333334, - "coverage_gap_90": 0.06666666666666665, - "avg_interval_width_90": 0.08555778278379522, - "winkler_90": 0.1492063183872844, - "pinball_90": 0.0037301579596821116, - "wis_90": 0.03850617753044722, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - { - "method": "aci", - "n_eval": 24, - "coverage_90": 0.7083333333333334, - "coverage_gap_90": 0.19166666666666665, - "avg_interval_width_90": 0.09372322463850964, - "winkler_90": 0.17448460839509497, - "pinball_90": 0.004362115209877376, - "wis_90": 0.04080420389479363, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.0, - "last_rolling_coverage_6": 1.0 - } - } - ], - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - "exogenous_contract_available": true, - "exogenous_contract_version": "minimal_macro_covariates_v1", - "exogenous_active": false, - "ensemble_weights": { - "AutoARIMA": 0.2730550804361629, - "AutoETS": 0.27172184234396446, - "SARIMAX": 0.2282948755555333, - "STL_CatBoost": 0.22692820166433936 - }, - "candidate_methods_tested": [ - "AutoARIMA", - "AutoETS", - "AutoTheta", - "SARIMAX", - "STL_CatBoost", - "SeasonalNaive", - "enbpi", - "aci" - ], - "rolling_coverage_summary": { - "official_interval_model": "AutoARIMA", - "official_interval_coverage_gap_90": 0.08981481481481479, - "mapie_best_method": "enbpi", - "mapie_best_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - }, - "rolling_coverage_by_horizon": { - "official_interval_model": { - "coverage_gap_90": 0.08981481481481479, - "avg_interval_width_90": 0.17404434598567664, - "winkler_90": 0.35975890823617446, - "wis_90": 0.07804457277384988, - "pinball_90": 0.008993972705904364 - }, - "adaptive_best_method": { - "method": "enbpi", - "coverage_gap_90": 0.06666666666666665, - "avg_interval_width_90": 0.08555778278379522, - "winkler_90": 0.1492063183872844, - "wis_90": 0.03850617753044722, - "pinball_90": 0.0037301579596821116, - "rolling_coverage_summary": { - "min_rolling_coverage_6": 0.3333333333333333, - "last_rolling_coverage_6": 1.0 - } - } - }, - "hierarchy_reconciliation": { - "available": false, - "reason": "hierarchy_reconciliation_disabled" - }, - "interval_policy": { - "eligible_interval_families": [ - "statistical" - ], - "max_coverage_gap": 0.03, - "max_winkler_90": null - }, - "interval_selector_reason": "interval champion selected from eligible families under the governed coverage-gap policy; adaptive and conformal statistical candidates remain diagnostic unless they satisfy the same thresholds.", - "adaptive_method_status": { - "best_method": "enbpi", - "candidate_methods_tested": [ - "enbpi", - "aci" - ], - "promotion_ready": false, - "notes": "Adaptive methods remain diagnostic until they beat the official interval policy." - }, - "final_interval_decision": { - "status": "research_only", - "official_family": "statistical", - "reason": "interval champion selected from eligible families under the governed coverage-gap policy; adaptive and conformal statistical candidates remain diagnostic unless they satisfy the same thresholds." - } - }, - "storytelling_snapshot": { - "snapshot_name": "storytelling_snapshot", - "schema_version": "2026-03-13.2", - "generated_at_utc": "2026-04-07T05:07:47.912391+00:00", - "dataset_scope": "full_data", - "official_run_tag": "canonical-monotonic-promotion-2026-03-29-0929", - "threshold_semantics": { - "schema_version": "2026-03-13.1", - "generated_at_utc": "2026-04-07T04:03:55.242901+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "pd_internal_selected_threshold": 0.05, - "pd_internal_fallback_threshold": 0.35, - "fairness_primary_threshold": 0.35, - "decision_policy_global_threshold": 0.35, - "source_artifacts": { - "fairness_status": "models/fairness_audit_status.json", - "fairness_decision_policy": "models/fairness_decision_policy.json", - "fairness_frontier": "data/processed/fairness_threshold_frontier.parquet", - "decision_threshold": "models/decision_threshold.json", - "decision_threshold_v2": "models/decision_threshold_v2.json", - "test_predictions": "data/processed/test_predictions.parquet" - }, - "business_meaning": { - "pd_internal_selected_threshold": "Threshold interno de screening/seleccion PD usado en busqueda y analisis tecnico.", - "pd_internal_fallback_threshold": "Fallback interno tomado de la policy de fairness durante la busqueda de threshold PD.", - "fairness_primary_threshold": "Threshold operativo principal para auditoria de fairness y narrativa de aprobacion.", - "decision_policy_global_threshold": "Threshold operativo global aplicado por la policy de decision/aprobacion." - }, - "fairness_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval" - }, - "conformal_promotion_pass": true, - "conformal_strict_policy_pass": false, - "conformal_methodological_justification_pass": true, - "conformal_methodological_closure_candidate": true, - "conformal_methodological_closure_threshold": 1.22, - "conformal_statistical_warning": true, - "time_series_interval_promotable": false, - "time_series_final_interval_decision": "research_only", - "causal_status": { - "selected_rule": "discount_100_only", - "selection_reason": "best_feasible", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129" - }, - "ab_gate_mode": "no_regression", - "method_registry_status": "adopted", - "headline_metrics": { - "auc_oot": 0.7126809898257414, - "coverage_90": 0.9241590788423406, - "coverage_95": 0.9596018333580141, - "pd_pr_auc": 0.3974083356465099, - "pd_ambiguity_rate": 0.5473816136873395, - "price_of_robustness": -66048.30591090144, - "price_of_robustness_pct": -600.5947045578031, - "robust_return": 139207.65057267927, - "nonrobust_return": 73159.34466177782, - "baseline_ecl": 875935946.2736094, - "severe_ecl": 1487900897.2686803, - "severe_uplift_pct": 69.86412118357295, - "fairness_primary_threshold": 0.35 - }, - "artifact_health": [ - { - "path": "data/processed/pipeline_summary.json", - "exists": true, - "updated_at_utc": "2026-04-07T05:07:14.525665+00:00", - "size_bytes": 4674 - }, - { - "path": "models/conformal_results_mondrian.pkl", - "exists": true, - "updated_at_utc": "2026-04-07T03:59:04.913420+00:00", - "size_bytes": 3514 - }, - { - "path": "data/processed/conformal_intervals_mondrian.parquet", - "exists": true, - "updated_at_utc": "2026-04-07T03:59:04.881801+00:00", - "size_bytes": 5384431 - }, - { - "path": "models/conformal_policy_status.json", - "exists": true, - "updated_at_utc": "2026-04-07T03:59:40.893999+00:00", - "size_bytes": 12486 - }, - { - "path": "data/processed/portfolio_robustness_summary.parquet", - "exists": true, - "updated_at_utc": "2026-03-30T04:59:30.427667+00:00", - "size_bytes": 14930 - }, - { - "path": "data/processed/ifrs9_scenario_summary.parquet", - "exists": true, - "updated_at_utc": "2026-04-07T04:02:13.138413+00:00", - "size_bytes": 17684 - } - ], - "notes": [ - "Snapshot con métricas para storytelling reproducible.", - "Fuente conformal canónica: conformal_results_mondrian.pkl + conformal_intervals_mondrian.parquet.", - "Threshold PD interno y threshold operativo de fairness se reportan por separado.", - "Classification sets y rare-event calibration son sidecars diagnósticos; no forman parte del champion operativo actual." - ] - } - } -} diff --git a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.json b/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.json deleted file mode 100644 index 75e6d29..0000000 --- a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.json +++ /dev/null @@ -1,1323 +0,0 @@ -{ - "schema_version": "2026-03-16.1", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "generated_at_utc": "2026-04-07T17:48:44.349138+00:00", - "baseline_path": "reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/baseline_snapshot.json", - "overall_pass": false, - "operational_overall_pass": true, - "artifact_coherence_pass": false, - "artifact_coherence": { - "expected_run_tag": "crpto-e2e-all-champions-2026-04-07", - "critical_artifacts": [ - { - "artifact": "reports/dvc/metrics_summary.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T17:45:11.987190+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "data/processed/pipeline_summary.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:44:41.209275+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/conformal_policy_status.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T14:39:38.072894+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/fairness_audit_status.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:42:20.905091+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/governance_status.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:43:53.829393+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/ab_simulation_status.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T17:41:15.994982+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_effect_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:12.527380+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_policy_rule.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:21.052543+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_policy_oot_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:27.886061+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/cate_portfolio_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:30.521517+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/time_series_status.json", - "schema_version": "2026-03-07.1", - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "missing_metadata_fields": [] - } - ], - "all_have_metadata": true, - "missing_metadata_artifacts": [], - "run_tags_observed": [ - "canonical-audit-rebuild-2026-04-06-r2", - "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "crpto-e2e-all-champions-2026-04-07" - ], - "run_tag_consistent": false, - "run_tag_matches_expected": false, - "run_tag_matches_expected_operational": false, - "causal_only_mismatch": false, - "mismatched_run_tag_artifacts": [ - "models/causal_effect_status.json", - "models/causal_policy_rule.json", - "models/causal_policy_oot_status.json", - "models/cate_portfolio_status.json", - "models/time_series_status.json" - ], - "non_causal_mismatched_run_tag_artifacts": [ - "models/time_series_status.json" - ], - "timestamp_skew_seconds": 685079.45981, - "timestamp_coherent": false, - "timestamp_max_skew_seconds": 259200, - "passed": false - }, - "semantic_coherence_pass": true, - "semantic_coherence": { - "checks": { - "operational_thresholds_ok": true, - "threshold_role_separation_ok": true, - "time_series_storytelling_ok": true, - "storytelling_conformal_ok": true - }, - "operational_thresholds": { - "threshold_semantics.fairness_primary_threshold": 0.35, - "threshold_semantics.decision_policy_global_threshold": 0.35, - "fairness_status.primary_threshold": 0.35, - "fairness_status.prediction_threshold": 0.35, - "fairness_decision_policy.global_threshold": 0.35, - "storytelling.headline_metrics.fairness_primary_threshold": 0.35 - }, - "time_series": { - "status_interval_promotable": false, - "status_final_decision": "research_only", - "storytelling_interval_promotable": false, - "storytelling_final_decision": "research_only" - }, - "conformal": { - "status_strict_overall_pass": false, - "status_methodological_justification_pass": true, - "storytelling_strict_policy_pass": false, - "storytelling_methodological_justification_pass": true - } - }, - "conformal_promotion_pass": true, - "conformal_statistical_warning": true, - "conformal_failing_statistical_tests": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "ab_no_regression_pass": true, - "fairness_absolute_business_pass": true, - "ab_gate_mode": "no_regression", - "ab_significant": false, - "ab_significance_role": "diagnostic", - "gates": [ - { - "name": "artifact_coherence", - "passed": false, - "details": { - "expected_run_tag": "crpto-e2e-all-champions-2026-04-07", - "critical_artifacts": [ - { - "artifact": "reports/dvc/metrics_summary.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T17:45:11.987190+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "data/processed/pipeline_summary.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:44:41.209275+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/conformal_policy_status.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T14:39:38.072894+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/fairness_audit_status.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:42:20.905091+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/governance_status.json", - "schema_version": "2026-03-06.1", - "generated_at_utc": "2026-04-07T17:43:53.829393+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/ab_simulation_status.json", - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T17:41:15.994982+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_effect_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:12.527380+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_policy_rule.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:21.052543+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/causal_policy_oot_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:27.886061+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/cate_portfolio_status.json", - "schema_version": "2026-03-26.1", - "generated_at_utc": "2026-03-30T19:27:30.521517+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "missing_metadata_fields": [] - }, - { - "artifact": "models/time_series_status.json", - "schema_version": "2026-03-07.1", - "generated_at_utc": "2026-04-07T04:01:36.025011+00:00", - "run_tag": "canonical-audit-rebuild-2026-04-06-r2", - "missing_metadata_fields": [] - } - ], - "all_have_metadata": true, - "missing_metadata_artifacts": [], - "run_tags_observed": [ - "canonical-audit-rebuild-2026-04-06-r2", - "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "crpto-e2e-all-champions-2026-04-07" - ], - "run_tag_consistent": false, - "run_tag_matches_expected": false, - "run_tag_matches_expected_operational": false, - "causal_only_mismatch": false, - "mismatched_run_tag_artifacts": [ - "models/causal_effect_status.json", - "models/causal_policy_rule.json", - "models/causal_policy_oot_status.json", - "models/cate_portfolio_status.json", - "models/time_series_status.json" - ], - "non_causal_mismatched_run_tag_artifacts": [ - "models/time_series_status.json" - ], - "timestamp_skew_seconds": 685079.45981, - "timestamp_coherent": false, - "timestamp_max_skew_seconds": 259200, - "passed": false - } - }, - { - "name": "semantic_coherence", - "passed": true, - "details": { - "checks": { - "operational_thresholds_ok": true, - "threshold_role_separation_ok": true, - "time_series_storytelling_ok": true, - "storytelling_conformal_ok": true - }, - "operational_thresholds": { - "threshold_semantics.fairness_primary_threshold": 0.35, - "threshold_semantics.decision_policy_global_threshold": 0.35, - "fairness_status.primary_threshold": 0.35, - "fairness_status.prediction_threshold": 0.35, - "fairness_decision_policy.global_threshold": 0.35, - "storytelling.headline_metrics.fairness_primary_threshold": 0.35 - }, - "time_series": { - "status_interval_promotable": false, - "status_final_decision": "research_only", - "storytelling_interval_promotable": false, - "storytelling_final_decision": "research_only" - }, - "conformal": { - "status_strict_overall_pass": false, - "status_methodological_justification_pass": true, - "storytelling_strict_policy_pass": false, - "storytelling_methodological_justification_pass": true - } - } - }, - { - "name": "pd_quality", - "passed": true, - "details": { - "baseline": { - "auc": 0.7126809898257414, - "ece": 0.006731531345207922, - "d2_brier": 0.09864138358717767 - }, - "current": { - "auc": 0.712438241693818, - "ece": 0.0062482132744862825, - "d2_brier": 0.09823922439765564 - }, - "checks": { - "auc_ok": true, - "ece_ok": true, - "d2_brier_ok": true - }, - "thresholds": { - "auc_min_delta": -0.005, - "ece_max_multiplier": 1.2, - "d2_brier_tolerance": 0.002 - }, - "deltas": { - "auc_delta": -0.00024274813192337596, - "ece_delta": -0.0004833180707216394, - "d2_brier_delta": -0.00040215918952202223 - } - } - }, - { - "name": "conformal_policy", - "passed": true, - "details": { - "baseline": { - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9293384235866059, - "coverage_95": 0.9623395902033092, - "avg_width_90": 0.764155715329014, - "min_group_coverage_90": 0.9003509420022164, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.193742537952152, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.1305594074149068, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2919.8495778179786, - "p_value": 0.0, - "reject": true, - "n_violations": 19564, - "n_total": 276869, - "violation_rate": 0.07066157641339406, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 966.7782855685655, - "p_value": 0.0, - "reject": true, - "n_violations": 10427, - "n_total": 276869, - "violation_rate": 0.03766040979669085, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2919.8495778179786, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 2.290243347553769, - "p_ind": 0.13018950970882026, - "reject_ind": false, - "lr_cc": 2922.1398211655323, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239175, - "n01": 18129, - "n10": 18129, - "n11": 1435 - } - }, - "christoffersen_95": { - "lr_uc": 966.7782855685655, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.07731377833988518, - "p_ind": 0.7809712495852495, - "reject_ind": false, - "lr_cc": 966.8555993469054, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 256412, - "n01": 10029, - "n10": 10029, - "n11": 398 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07066157641339406, - "violation_rate_95": 0.03766040979669085, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.13018950970882026, - "christoffersen_independence_pvalue_95": 0.7809712495852495 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.029338423586605877, - "coverage_deviation_95": 0.012339590203309214, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "configs/conformal_policy.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T06:13:27.444684+00:00", - "run_tag": "conformal-reopen-rank1-promotion-2026-04-07" - }, - "current": { - "overall_pass": true, - "strict_overall_pass": false, - "non_statistical_checks_pass": true, - "methodological_justification_pass": true, - "methodological_justification_status": "eligible_statistical_warning_only", - "statistical_tests_role": "diagnostic_informational", - "checks_passed": 9, - "checks_total": 13, - "failing_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_statistical_checks": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "failing_non_statistical_checks": [], - "coverage_90": 0.9293384235866059, - "coverage_95": 0.9623395902033092, - "avg_width_90": 0.764155715329014, - "min_group_coverage_90": 0.9003509420022164, - "critical_alerts": 0, - "warning_alerts": 0, - "total_alerts": 0, - "winkler_90": 1.193742537952152, - "winkler_90_raw_pass": true, - "winkler_90_policy_pass": true, - "winkler_90_policy_mode": "strict", - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "winkler_95": 1.1305594074149068, - "mapie_mwi_90": null, - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0, - "statistical_tests": { - "kupiec_90": { - "lr_statistic": 2919.8495778179786, - "p_value": 0.0, - "reject": true, - "n_violations": 19564, - "n_total": 276869, - "violation_rate": 0.07066157641339406, - "nominal_alpha": 0.1 - }, - "kupiec_95": { - "lr_statistic": 966.7782855685655, - "p_value": 0.0, - "reject": true, - "n_violations": 10427, - "n_total": 276869, - "violation_rate": 0.03766040979669085, - "nominal_alpha": 0.05 - }, - "christoffersen_90": { - "lr_uc": 2919.8495778179786, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 2.290243347553769, - "p_ind": 0.13018950970882026, - "reject_ind": false, - "lr_cc": 2922.1398211655323, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 239175, - "n01": 18129, - "n10": 18129, - "n11": 1435 - } - }, - "christoffersen_95": { - "lr_uc": 966.7782855685655, - "p_uc": 0.0, - "reject_uc": true, - "lr_ind": 0.07731377833988518, - "p_ind": 0.7809712495852495, - "reject_ind": false, - "lr_cc": 966.8555993469054, - "p_cc": 0.0, - "reject_cc": true, - "transition_matrix": { - "n00": 256412, - "n01": 10029, - "n10": 10029, - "n11": 398 - } - } - }, - "sample_size_context": { - "n_total_90": 276869, - "n_total_95": 276869, - "violation_rate_90": 0.07066157641339406, - "violation_rate_95": 0.03766040979669085, - "nominal_alpha_90": 0.1, - "nominal_alpha_95": 0.05, - "christoffersen_independence_pvalue_90": 0.13018950970882026, - "christoffersen_independence_pvalue_95": 0.7809712495852495 - }, - "methodological_justification": { - "allowed": true, - "only_statistical_failures": true, - "coverage_materiality_ok": true, - "coverage_deviation_90": 0.029338423586605877, - "coverage_deviation_95": 0.012339590203309214, - "max_coverage_deviation_for_statistical_warning_90": 0.03, - "max_coverage_deviation_for_statistical_warning_95": 0.03, - "independence_ok_90": true, - "independence_ok_95": true, - "min_christoffersen_independence_pvalue_90": 0.01, - "min_christoffersen_independence_pvalue_95": 0.01, - "winkler_90_policy_mode": "strict", - "winkler_90_raw_pass": true, - "winkler_90_compensated_pass": false, - "winkler_90_compensated_threshold": 1.22, - "decision": true, - "justification_role": "strict_blocking" - }, - "latest_backtest_month": "2020-09-01 00:00:00", - "intervals_path": "data/processed/conformal_intervals_mondrian.parquet", - "artifact_namespace": "", - "policy_config": "configs/conformal_policy.yaml", - "lgd_ead_conformal_status_path": "models/conformal_lgd_ead_status.json", - "lgd_ead_conformal_status": { - "schema_version": "2026-03-02.1", - "generated_at_utc": "2026-03-30T18:35:03.994629+00:00", - "run_tag": "canonical-monotonic-confirmatory-adsfcr-2026-03-30-1129", - "lgd": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "schema_variant": "adaptive_lgd_v2", - "selected_variant": "direct_adaptive_grade_temporal", - "selection_reason": "selected_best_pass_by_width", - "model_metrics": { - "two_stage": { - "stage1_auc": 0.5063874461720522, - "lgd_mae": 0.18939613847779388, - "lgd_rmse": 0.2227476719236602 - }, - "direct": { - "lgd_mae": 0.18808771622019224, - "lgd_rmse": 0.2215532507528556, - "lgd_bias": -0.17443676881106576 - } - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.9049958915365653, - "target_coverage": 0.9, - "coverage_gap": 0.004995891536565322, - "avg_interval_width": 0.49480114425148514, - "median_interval_width": 0.48959281220209183 - }, - "metrics_95": { - "empirical_coverage": 0.9550041084634346, - "target_coverage": 0.95, - "coverage_gap": 0.005004108463434687, - "avg_interval_width": 0.5752545930192787, - "median_interval_width": 0.5674159705753884 - }, - "policy": { - "target_coverage_online_90": 0.905, - "target_coverage_online_95": 0.955, - "eta": 0.12, - "offset_rho": 0.03, - "maturity_uncertainty_lambda": 0.25, - "maturity_months_reference": 24.0, - "q_start_90": 0.2835716520127255, - "q_start_95": 0.3316821510110814, - "q_end_by_grade_90": { - "A": 0.34837165201264114, - "B": 0.2907716520125905, - "D": 0.233771652012689, - "C": 0.30877165201262874, - "E": 0.18277165201271306, - "G": 0.3297716520127261, - "F": 0.3207716520127274 - }, - "q_end_by_grade_95": { - "A": 0.4444821510109877, - "B": 0.29088215101086917, - "D": 0.26388215101082846, - "C": 0.2488821510107701, - "E": 0.30288215101100663, - "G": 0.35988215101107823, - "F": 0.3808821510110691 - }, - "offset_end_by_grade_90": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - }, - "offset_end_by_grade_95": { - "A": 0.2176863042129322, - "B": 0.18134115469751272, - "C": 0.18866322285336742, - "D": 0.1646305582022108, - "E": 0.09996860469526932, - "F": 0.08499397029718181, - "G": 0.07271691316580109 - } - } - }, - "guardrails": { - "overall_pass": true, - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "min_grade_coverage_90": 0.9046563192904656, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "abs_bias": 0.08741436133827732, - "status_path": "models/lgd_guardrails_status.json" - }, - "artifacts": { - "intervals_path": "data/processed/conformal_intervals_lgd.parquet", - "benchmark_path": "data/processed/lgd_variant_benchmark.parquet", - "diagnostics_path": "data/processed/lgd_coverage_diagnostics.parquet", - "coverage_by_grade_path": "data/processed/lgd_coverage_by_grade.parquet", - "coverage_by_year_path": "data/processed/lgd_coverage_by_year.parquet" - }, - "benchmark_variants": [ - { - "variant": "two_stage_split", - "coverage_90": 0.7856368118323747, - "coverage_95": 0.8702711585866886, - "avg_width_90": 0.5634431044058621, - "avg_width_95": 0.648907429645572, - "bias": -0.17596373121449316, - "min_grade_coverage_90": 0.5445118059987237, - "min_year_coverage_90": 0.5023958717287136, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 1.0, - "width_inflation_95": 1.0, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_split", - "coverage_90": 0.7884963023829088, - "coverage_95": 0.8717995069843878, - "avg_width_90": 0.562395655348415, - "avg_width_95": 0.6475756145327506, - "bias": -0.17443676881106576, - "min_grade_coverage_90": 0.5443522654754308, - "min_year_coverage_90": 0.5060081091043126, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9981409852223648, - "width_inflation_95": 0.9979476038461312, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_cqr", - "coverage_90": 0.7381101068200493, - "coverage_95": 0.8417584223500411, - "avg_width_90": 0.5374013738533742, - "avg_width_95": 0.6423866594148478, - "bias": -0.17805024176144146, - "min_grade_coverage_90": 0.7091505328876837, - "min_year_coverage_90": 0.38510873571691856, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.9537810821556715, - "width_inflation_95": 0.9899511549216107, - "check_coverage_90": false, - "check_coverage_95": false, - "check_min_grade_coverage_90": false, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": false, - "overall_pass": false - }, - { - "variant": "direct_adaptive_grade_temporal", - "coverage_90": 0.9049958915365653, - "coverage_95": 0.9550041084634346, - "avg_width_90": 0.49480114425148514, - "avg_width_95": 0.5752545930192787, - "bias": -0.08741436133827732, - "min_grade_coverage_90": 0.9046563192904656, - "min_year_coverage_90": 0.9049760412827129, - "n_test": 60850.0, - "n_grades_eval": 6.0, - "n_years_eval": 2.0, - "width_inflation_90": 0.8781741055705024, - "width_inflation_95": 0.8864971592843036, - "check_coverage_90": true, - "check_coverage_95": true, - "check_min_grade_coverage_90": true, - "check_width_inflation_90": true, - "check_width_inflation_95": true, - "check_abs_bias": true, - "overall_pass": true - } - ] - }, - "ead": { - "available": true, - "n_train": 249380, - "n_cal": 52751, - "n_test": 60850, - "model_metrics": { - "ead_mae": 35.71422510728435, - "ead_r2": 0.9998664166463509 - }, - "conformal": { - "metrics_90": { - "empirical_coverage": 0.912013147082991, - "target_coverage": 0.9, - "coverage_gap": 0.01201314708299095, - "avg_interval_width": 146.39793597617248, - "median_interval_width": 146.39793597617245 - }, - "metrics_95": { - "empirical_coverage": 0.9510106820049301, - "target_coverage": 0.95, - "coverage_gap": 0.0010106820049301524, - "avg_interval_width": 209.64425578343796, - "median_interval_width": 209.64425578343798 - } - }, - "intervals_path": "data/processed/conformal_intervals_ead.parquet" - }, - "benchmark_mode": { - "benchmark_short": false, - "effective_sample_size_per_split": null - } - }, - "schema_version": "2026-03-01.1", - "generated_at_utc": "2026-04-07T14:39:38.072894+00:00", - "run_tag": "crpto-e2e-all-champions-2026-04-07" - }, - "checks": { - "coverage90_ok": true, - "coverage95_ok": true, - "min_group_coverage90_ok": true, - "winkler90_ok": true, - "critical_alerts_ok": true, - "conformal_promotion_pass": true - }, - "diagnostics": { - "statistical_warning": true, - "statistical_pvalue_threshold": 0.01, - "statistical_tests": { - "kupiec_pvalue_90": 0.0, - "kupiec_pvalue_95": 0.0, - "christoffersen_pvalue_90": 0.0, - "christoffersen_pvalue_95": 0.0 - }, - "failing_statistical_tests": [ - "kupiec_pvalue_90", - "kupiec_pvalue_95", - "christoffersen_pvalue_90", - "christoffersen_pvalue_95" - ], - "policy_overall_pass_strict": true - } - } - }, - { - "name": "ab_no_regression", - "passed": true, - "details": { - "checks": { - "self_no_regression_ok": true, - "cross_scenario_gate_ok": false, - "control_vs_baseline_ok": true, - "robust_vs_baseline_ok": true, - "gap_vs_baseline_ok": true - }, - "warnings": { - "control_vs_baseline_warning": false, - "robust_vs_baseline_warning": false, - "gap_vs_baseline_warning": false - }, - "current": { - "control_total_return": 99126.67771029609, - "robust_total_return": 102207.5100557222, - "diff_total_return": 3080.8323454261117, - "tolerance_total_return": 4956.333885514805, - "n_candidates_used": 80000 - }, - "baseline": { - "control_total_return": 87843.18562343111, - "robust_total_return": 77599.77840196024, - "diff_total_return": -10243.407221470872 - }, - "diagnostics": { - "p_value": 0.4644, - "significant": false, - "significance_role": "diagnostic", - "gate_mode": "no_regression" - } - } - }, - { - "name": "fairness_relative", - "passed": true, - "details": { - "baseline_n_passed": 6, - "current_n_passed": 6, - "baseline_n_total": 6, - "current_n_total": 6, - "baseline_overall_pass": true, - "current_overall_pass": true - } - }, - { - "name": "fairness_absolute_business", - "passed": true, - "details": { - "policy_contract": { - "prediction_threshold": 0.35, - "outcome_mode": "approval", - "use_artifact": true, - "policy_path": "configs/fairness_policy.yaml" - }, - "current": { - "prediction_threshold": 0.35, - "prediction_threshold_source": "decision_policy_artifact_auto_selected", - "outcome_mode": "approval", - "n_passed": 6, - "n_total": 6, - "overall_pass": true - }, - "checks": { - "overall_pass_ok": true, - "all_attributes_ok": true, - "threshold_match_ok": true, - "threshold_source_ok": true, - "outcome_mode_ok": true - } - } - }, - { - "name": "survival_quality", - "passed": true, - "details": { - "baseline": { - "cox_cindex": 0.6947846585338557, - "rsf_cindex": 0.6948291667286338 - }, - "current": { - "cox_cindex": 0.6947846585338557, - "rsf_cindex": 0.6948291667286338 - }, - "checks": { - "cox_ok": true, - "rsf_ok": true - } - } - }, - { - "name": "export_contracts", - "passed": true, - "details": { - "missing_keys": [] - } - } - ], - "artifact_changes": { - "data/processed/ifrs9_scenario_summary.parquet": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 17684, - "size_bytes_current": 17684 - }, - "data/processed/model_comparison.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 15059, - "size_bytes_current": 15059 - }, - "data/processed/pipeline_summary.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 4674, - "size_bytes_current": 4656 - }, - "data/processed/portfolio_robustness_frontier.parquet": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 127884, - "size_bytes_current": 43654 - }, - "data/processed/portfolio_robustness_summary.parquet": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 14930, - "size_bytes_current": 14334 - }, - "models/conformal_lgd_ead_status.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 7815, - "size_bytes_current": 7815 - }, - "models/conformal_policy_status.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 12473, - "size_bytes_current": 12465 - }, - "models/fairness_audit_status.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 3343, - "size_bytes_current": 3330 - }, - "models/governance_status.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 9580, - "size_bytes_current": 9578 - }, - "models/survival_summary.pkl": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 1557, - "size_bytes_current": 1557 - }, - "reports/dvc/metrics_summary.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": true, - "size_bytes_baseline": 1828, - "size_bytes_current": 1826 - }, - "reports/gpu_benchmark/cudf_polars_benchmark.csv": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 340, - "size_bytes_current": 340 - }, - "reports/gpu_benchmark/cugraph_benchmark.csv": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 401, - "size_bytes_current": 401 - }, - "reports/gpu_benchmark/cuml_benchmark.csv": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 1106, - "size_bytes_current": 1106 - }, - "reports/gpu_benchmark/cuopt_benchmark.csv": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 871, - "size_bytes_current": 871 - }, - "reports/gpu_benchmark/cupy_benchmark.csv": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 383, - "size_bytes_current": 383 - }, - "reports/gpu_benchmark/gpu_bench_meta.json": { - "baseline_exists": true, - "current_exists": true, - "hash_changed": false, - "size_bytes_baseline": 960, - "size_bytes_current": 960 - } - }, - "baseline_head": "7c55db2b57fb0994f09640eeab6710aac492dd48", - "current_head": "7c55db2b57fb0994f09640eeab6710aac492dd48", - "quality_contract": { - "conformal_checks_required": 13, - "ab_gate_mode": "no_regression", - "ab_significance_role": "diagnostic", - "fairness_gates": [ - "fairness_relative", - "fairness_absolute_business" - ], - "fairness_policy_path": "configs/fairness_policy.yaml", - "artifact_coherence_required": true, - "semantic_coherence_required": true, - "required_status_metadata": [ - "schema_version", - "generated_at_utc", - "run_tag" - ] - } -} diff --git a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.md b/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.md deleted file mode 100644 index 8522673..0000000 --- a/reports/run_comparisons/crpto-e2e-all-champions-2026-04-07/comparison.md +++ /dev/null @@ -1,36 +0,0 @@ -# Run Comparison: crpto-e2e-all-champions-2026-04-07 - -- Generated: 2026-04-07T17:48:44.349138+00:00 -- Overall gates pass: `False` -- Conformal promotion pass: `True` -- Conformal statistical warning: `True` -- Artifact coherence pass: `False` -- Semantic coherence pass: `True` -- Fairness absolute (business) pass: `True` -- A/B gate mode: `no_regression` -- A/B no-regression pass: `True` -- A/B significance (diagnostic): `False` - -## Gates -- `artifact_coherence`: **FAIL** -- `semantic_coherence`: **PASS** -- `pd_quality`: **PASS** -- `conformal_policy`: **PASS** -- `ab_no_regression`: **PASS** -- `fairness_relative`: **PASS** -- `fairness_absolute_business`: **PASS** -- `survival_quality`: **PASS** -- `export_contracts`: **PASS** - -## Artifact Changes -- `data/processed/model_comparison.json`: hash_changed=True, baseline_exists=True, current_exists=True -- `data/processed/pipeline_summary.json`: hash_changed=True, baseline_exists=True, current_exists=True -- `data/processed/portfolio_robustness_frontier.parquet`: hash_changed=True, baseline_exists=True, current_exists=True -- `data/processed/portfolio_robustness_summary.parquet`: hash_changed=True, baseline_exists=True, current_exists=True -- `models/conformal_policy_status.json`: hash_changed=True, baseline_exists=True, current_exists=True -- `models/fairness_audit_status.json`: hash_changed=True, baseline_exists=True, current_exists=True -- `models/governance_status.json`: hash_changed=True, baseline_exists=True, current_exists=True -- `reports/dvc/metrics_summary.json`: hash_changed=True, baseline_exists=True, current_exists=True - -## Conformal Diagnostics -- Statistical warnings (non-blocking): `kupiec_pvalue_90`, `kupiec_pvalue_95`, `christoffersen_pvalue_90`, `christoffersen_pvalue_95` diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 4ba608f..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# CRPTO scripts map - -This folder contains both current IJDS publication tooling and historical -research/search entry points. Treat the current submission path as narrow on -purpose. - -## Current IJDS path - -Use these for the active submission workflow: - -- `check_publication_integrity.py` - checks that paper, supplement, README and - official-template docs agree on the active midpoint IJDS claim. -- `build_ijds_calibration_selected_evidence.py` - regenerates active A35--A40 - and governance from the exact-alpha and selected-policy outputs. -- `experiments/run_ijds_exact_alpha_grid_challenger.py` - exact replay into an - isolated experiment path. -- `experiments/run_ijds_calibration_selected_policy_challenger.py` - solves the - declared 3x3 calibration grid and fixed OOT evaluations. -- `compile_ijds_submission.py` - compiles the official INFORMS/IJDS LaTeX - handoff and scans `.log`/`.blg` for unresolved citations or references. -- `run_ty_advisory.py` - runs pinned `ty` in a focused advisory scope for daily - IJDS work or in a blocking full scope. Both scopes are currently clean; - `submission-check` enforces the full scope. -- `build_crpto_journal_package.py` - builds the journal evidence package from - frozen inputs. -- `export_crpto_tables.py` - exports paper tables from frozen artifacts. -- `generate_crpto_figures.py` - exports paper figures from frozen artifacts. -- `render_submission_pdf_previews.py` - creates local HTML-print preview PDFs - for body and supplement. - -The high-level commands are still the source of truth: - -```powershell -just smoke -just type-advisory -just hooks-check -just complexity-report -just ijds-evidence -just paper-submission -just paper-submission-official -just submission-check -``` - -Optional local inspection: - -```powershell -just api-docs-core -``` - -This builds `reports/api-docs/` with `pdoc` for the core optimization, -calibration and evaluation modules. The output is ignored by Git. - -`just complexity-report` runs `radon` over `src/` and `scripts/` and reports -D-or-higher blocks. Treat it as a refactor radar, not a submission gate: some -historical/protected search entry points remain intentionally long until a -post-submission cleanup lane justifies touching them. - -## Protected or historical search paths - -The large scripts under `scripts/search/` and most `scripts/experiments/` are -historical or governed research surfaces. The three active IJDS experiment -modules listed above are the narrow exception. Do not run HPO, conformal interval -generation, champion search, or protected portfolio search unless the work has -a fresh run tag, artifact sink, and drift/revalidation plan. - -`scripts/search/run_conformal_search.py` and -`scripts/search/run_portfolio_search.py` are intentionally retired wrappers. -They now return actionable messages instead of importing the removed generic -`scripts.run_long_pipeline` orchestrator. - -TabPFN, SPO+/PyEPO/Torch and cuOpt remain optional experiment stacks. The -scripts that need them use explicit optional imports so the base IJDS -environment stays light and full-tree type checks still remain useful. - -The active paper should cite A35--A40 and `ijds_policy_governance.json`, not an -ad hoc rerun or the historical policy frontier. - -## Refactor priority - -For pre-submission cleanup, prefer small changes that reduce publication risk: - -1. keep claim synchronization checks strict; -2. keep `mypy` green; -3. use `just type-advisory` for daily IJDS work and `just type-advisory-full` - for final local checks; -4. avoid broad rewrites of protected search code until after IJDS submission. diff --git a/scripts/archive/README.md b/scripts/archive/README.md deleted file mode 100644 index 5b01348..0000000 --- a/scripts/archive/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# `scripts/archive/` - -Scripts one-shot retirados del flujo activo (2026-06). No los invoca ningún -stage DVC, target de `justfile`, orquestador, test, capítulo del libro ni -documento; se conservan como evidencia histórica del proceso de búsqueda y -de auditorías ya congeladas en `models/*.json` y `reports/crpto/tables/`. - -Política: - -- Un script vive aquí solo si tiene **cero referencias** en el repo - (verificado contra `dvc.yaml`, `justfile`, `run_crpto_pipeline.py`, - `tests/`, `book/`, `docs/`, `notebooks/`, `.github/` y cross-imports). -- Los scripts cuyos outputs siguen siendo deps de DVC o cuyo rol en - `configs/pipeline_registry/script_role_registry.yaml` es `core`/`paper` - permanecen en `scripts/` aunque nadie los invoque hoy. -- No se garantiza que estos scripts corran con el stack de dependencias - actual; reproducen estados históricos del pipeline. - -| Script | Rol histórico | -| --- | --- | -| `build_concentration_bound_table.py` | Tabla exploratoria de bounds de concentración (no promovida al paper). | -| `run_crpto_notebook_suite.py` | Runner batch de la suite de notebooks exploratorios. | -| `search/compare_conformal_portfolio_finalists.py` | Comparación puntual de finalistas conformal durante la búsqueda. | -| `search/monitor_regret_auditability.py` | Monitor de progreso del sandbox de regret-auditability (cerrado 2026-05-28). | -| `search/resume_conformal_reopen_closure.py` | Reanudación del cierre de la búsqueda conformal reopen (run 2026-04-03/05). | -| `search/sweep_alpha_gamma_bound_finalists.py` | Sweep alpha-gamma sobre finalistas previo al exact eval. | diff --git a/scripts/archive/build_concentration_bound_table.py b/scripts/archive/build_concentration_bound_table.py deleted file mode 100644 index 4f8390a..0000000 --- a/scripts/archive/build_concentration_bound_table.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Build the concentration-bound comparison table (A21b). - -Compares the distribution-free Markov tail bound used in the main theorem against -Hoeffding and Bernstein tightenings under an *additional* independence assumption, -using the real frozen funded-set weights. The point is illustrative and honest: -it quantifies exactly how much Markov leaves on the table, and under what extra -assumption a sharper bound would hold -- without re-optimizing or re-promoting the -champion. - -All inputs are frozen: the funded-set loan export (weights) and the frozen -weighted miscoverage V = 0.028875. No champion search stage is re-run. - -Output: reports/crpto/tables/crpto_tableA21b_concentration_bounds.{csv,tex} -""" - -from __future__ import annotations - -from pathlib import Path - -import numpy as np -import pandas as pd -from loguru import logger - -REPO_ROOT = Path(__file__).resolve().parents[1] -TABLES = REPO_ROOT / "reports" / "crpto" / "tables" -FUNDED = TABLES / "crpto_tableA7_funded_set_loans.csv" - -# Frozen weighted miscoverage of the promoted champion (alpha = 0.01 exact eval). -EMPIRICAL_V = 0.028875 -ALPHAS = [0.01, 0.05, 0.10] - - -def _funded_weights() -> np.ndarray: - funded = pd.read_csv(FUNDED) - w = pd.to_numeric(funded["portfolio_weight"], errors="coerce").fillna(0.0).to_numpy(float) - total = float(w.sum()) - if total <= 0: - raise ValueError("Funded-set weights sum to zero.") - return w / total - - -def build_table() -> pd.DataFrame: - w = _funded_weights() - n = int(w.size) - sum_w2 = float((w**2).sum()) - n_eff = 1.0 / sum_w2 - max_w = float(w.max()) - - rows: list[dict[str, object]] = [] - for alpha in ALPHAS: - # Match the Markov tail probability delta = sqrt(alpha) for an apples-to-apples - # threshold comparison: each bound reports the t such that P(V > t) <= delta. - delta = float(np.sqrt(alpha)) - log_term = float(np.log(1.0 / delta)) - - markov_t = float(np.sqrt(alpha)) # Markov: P(V > sqrt(alpha)) <= sqrt(alpha) - - # Hoeffding (independent miscoverage indicators across loans): - # P(V - E[V] > eps) <= exp(-2 eps^2 / sum w_i^2) - eps_h = float(np.sqrt(sum_w2 * log_term / 2.0)) - hoeffding_t = alpha + eps_h - - # Bernstein (uses variance): Var(V) <= alpha * sum w_i^2; bounded difference ~ max weight. - var_v = alpha * sum_w2 - b = max_w - eps_b = float( - b * log_term / 3.0 + np.sqrt((b * log_term / 3.0) ** 2 + 2.0 * var_v * log_term) - ) - bernstein_t = alpha + eps_b - - rows.append( - { - "alpha": alpha, - "expected_V_leq": alpha, - "empirical_V": EMPIRICAL_V, - "markov_threshold": round(markov_t, 4), - "hoeffding_threshold_indep": round(hoeffding_t, 4), - "bernstein_threshold_indep": round(bernstein_t, 4), - "tail_prob_delta": round(delta, 4), - "empirical_V_below_all": bool( - min(markov_t, hoeffding_t, bernstein_t) > EMPIRICAL_V - ), - } - ) - - df = pd.DataFrame(rows) - df.attrs["n_funded"] = n - df.attrs["sum_w2"] = sum_w2 - df.attrs["n_eff"] = n_eff - return df - - -def main() -> None: - TABLES.mkdir(parents=True, exist_ok=True) - df = build_table() - csv_path = TABLES / "crpto_tableA21b_concentration_bounds.csv" - df.to_csv(csv_path, index=False) - - # Minimal LaTeX booktabs export for the supplement. - tex = df.to_latex(index=False, escape=True, float_format=lambda x: f"{x:.4f}") - (TABLES / "crpto_tableA21b_concentration_bounds.tex").write_text(tex, encoding="utf-8") - - logger.info( - "A21b concentration bounds: n_funded={} n_eff={:.1f} sum_w2={:.6f}", - df.attrs["n_funded"], - df.attrs["n_eff"], - df.attrs["sum_w2"], - ) - logger.info("Wrote {}", csv_path.relative_to(REPO_ROOT)) - - -if __name__ == "__main__": - main() diff --git a/scripts/archive/run_crpto_notebook_suite.py b/scripts/archive/run_crpto_notebook_suite.py deleted file mode 100644 index 3f0918d..0000000 --- a/scripts/archive/run_crpto_notebook_suite.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Execute CRPTO notebooks without mutating canonical artifacts.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import nbformat -from nbclient import NotebookClient - -NOTEBOOKS = ("10_crpto_cp_robust_opt.ipynb",) - - -def execute_notebook(input_path: Path, output_path: Path, *, timeout_s: int) -> None: - notebook = nbformat.read(input_path, as_version=4) - client = NotebookClient( - notebook, - timeout=timeout_s, - kernel_name="python3", - resources={"metadata": {"path": str(input_path.parent)}}, - ) - client.execute() - output_path.parent.mkdir(parents=True, exist_ok=True) - nbformat.write(notebook, output_path) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Execute CRPTO notebooks in a sandbox.") - parser.add_argument("--timeout", type=int, default=900) - parser.add_argument("--output-dir", default="reports/notebook_exec") - args = parser.parse_args() - - repo_root = Path(__file__).resolve().parents[1] - notebooks_dir = repo_root / "notebooks" - output_dir = repo_root / args.output_dir - - for notebook_name in NOTEBOOKS: - input_path = notebooks_dir / notebook_name - if not input_path.exists(): - raise FileNotFoundError(input_path) - output_path = output_dir / notebook_name - execute_notebook(input_path, output_path, timeout_s=int(args.timeout)) - print(f"[crpto-notebook-suite] saved {output_path}") - - -if __name__ == "__main__": - main() diff --git a/scripts/archive/search/compare_conformal_portfolio_finalists.py b/scripts/archive/search/compare_conformal_portfolio_finalists.py deleted file mode 100644 index e5f087b..0000000 --- a/scripts/archive/search/compare_conformal_portfolio_finalists.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Run focused portfolio tradeoff comparisons across conformal finalists.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import main as tradeoff_main # noqa: E402 - - -def _coerce_csv(raw: str | None) -> list[str]: - if not raw: - return [] - return [token.strip() for token in str(raw).split(",") if token.strip()] - - -def _default_label(path: str) -> str: - candidate = Path(path).stem or "finalist" - return candidate.replace("/", "_") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--interval-paths", required=True) - parser.add_argument("--labels", default=None) - parser.add_argument("--config", default="configs/optimization.yaml") - parser.add_argument("--risk-grid", default="0.06,0.08,0.10,0.12") - parser.add_argument("--aversion-grid", default="0.0,0.5,1.0,2.0") - parser.add_argument("--max-candidates", type=int, default=3000) - parser.add_argument("--random-state", type=int, default=42) - parser.add_argument("--robust-min-budget-utilization", type=float, default=0.05) - parser.add_argument("--strict-risk-threshold", type=float, default=0.12) - parser.add_argument("--robust-pd-slack-penalty", type=float, default=1.5) - parser.add_argument("--grid-profile", default="custom") - parser.add_argument("--solver-backend", choices=["highs", "cuopt"], default="highs") - parser.add_argument( - "--candidate-universe-path", - default="data/processed/champion_candidate_universe.parquet", - ) - parser.add_argument( - "--output-parquet", - default="data/processed/portfolio_tradeoff/conformal_finalist_comparison.parquet", - ) - parser.add_argument( - "--output-json", - default="models/portfolio_tradeoff/conformal_finalist_comparison.json", - ) - args = parser.parse_args(argv) - - interval_paths = _coerce_csv(args.interval_paths) - labels = _coerce_csv(args.labels) - if labels and len(labels) != len(interval_paths): - raise ValueError("labels and interval-paths must have the same number of entries") - if not labels: - labels = [_default_label(path) for path in interval_paths] - - rows: list[dict[str, object]] = [] - for label, interval_path in zip(labels, interval_paths, strict=True): - artifact_namespace = f"conformal-finalist-{label}" - tradeoff_main( - config_path=args.config, - risk_grid=args.risk_grid, - aversion_grid=args.aversion_grid, - max_candidates=args.max_candidates, - random_state=args.random_state, - robust_min_budget_utilization=args.robust_min_budget_utilization, - strict_risk_threshold=args.strict_risk_threshold, - robust_pd_slack_penalty=args.robust_pd_slack_penalty, - grid_profile=args.grid_profile, - solver_backend=args.solver_backend, - candidate_universe_path=args.candidate_universe_path, - conformal_intervals_path=interval_path, - artifact_namespace=artifact_namespace, - run_tag=f"portfolio-finalist-{label}", - ) - research_policy_path = ( - Path("models/portfolio_tradeoff") - / artifact_namespace - / "portfolio_research_policy.json" - ) - frontier_path = ( - Path("data/processed/portfolio_tradeoff") - / artifact_namespace - / "portfolio_robustness_frontier.parquet" - ) - summary_path = ( - Path("data/processed/portfolio_tradeoff") - / artifact_namespace - / "portfolio_robustness_summary.parquet" - ) - research_payload = json.loads(research_policy_path.read_text(encoding="utf-8")) - selection_metrics = dict(research_payload.get("selection_metrics", {}) or {}) - selected_policy = dict(research_payload.get("selected_policy", {}) or {}) - rows.append( - { - "label": label, - "conformal_intervals_path": interval_path, - "artifact_namespace": artifact_namespace, - "frontier_path": str(frontier_path), - "summary_path": str(summary_path), - "research_policy_path": str(research_policy_path), - "risk_tolerance": float(selected_policy.get("risk_tolerance", 0.0)), - "policy_mode": str(selected_policy.get("policy_mode", "")), - "gamma": float(selected_policy.get("gamma", 0.0)), - "delta_cap_quantile": float(selected_policy.get("delta_cap_quantile", 0.0)), - "tail_focus_quantile": float(selected_policy.get("tail_focus_quantile", 0.0)), - "uncertainty_aversion": float(selected_policy.get("uncertainty_aversion", 0.0)), - "ab_pass": bool(selection_metrics.get("ab_pass", False)), - "ab_diff_total_return": float(selection_metrics.get("ab_diff_total_return", 0.0)), - "realized_total_return": float(selection_metrics.get("realized_total_return", 0.0)), - "price_of_robustness": float(selection_metrics.get("price_of_robustness", 0.0)), - "price_of_robustness_pct": float( - selection_metrics.get("price_of_robustness_pct", 0.0) - ), - "n_funded": int(selection_metrics.get("n_funded", 0)), - } - ) - - comparison = pd.DataFrame(rows).sort_values( - by=["ab_pass", "realized_total_return", "price_of_robustness_pct"], - ascending=[False, False, False], - ) - output_parquet = Path(args.output_parquet) - output_json = Path(args.output_json) - output_parquet.parent.mkdir(parents=True, exist_ok=True) - output_json.parent.mkdir(parents=True, exist_ok=True) - comparison.to_parquet(output_parquet, index=False) - output_json.write_text( - json.dumps( - { - "n_finalists": len(comparison), - "rows": comparison.to_dict(orient="records"), - }, - indent=2, - ensure_ascii=False, - ) - + "\n", - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/archive/search/monitor_regret_auditability.py b/scripts/archive/search/monitor_regret_auditability.py deleted file mode 100644 index 4b8e150..0000000 --- a/scripts/archive/search/monitor_regret_auditability.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Command Prompt friendly monitor for the regret-auditability sandbox.""" - -from __future__ import annotations - -import argparse -import csv -import json -import re -import sqlite3 -import subprocess -import sys -import time -from collections import Counter -from pathlib import Path -from typing import Any - -DEFAULT_ARTIFACT_ROOT = Path( - r"D:\crpto_experiments\regret_auditability\regret_auditability_20260513_v3_resource_tuned" -) -PHASE_TRIAL_TARGETS = {"pd-smoke": 12, "pd-broad": 1000, "pd-refine": 500} - - -def emit(text: str = "") -> None: - """Write one display line to stdout.""" - sys.stdout.write(f"{text}\n") - - -def clear_screen() -> None: - """Clear the Windows console.""" - subprocess.run(["cmd", "/c", "cls"], check=False) - - -def load_json(path: Path) -> dict[str, Any]: - """Load a JSON object or return an empty dict.""" - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return payload if isinstance(payload, dict) else {} - - -def load_command_rows(path: Path) -> list[dict[str, str]]: - """Load command log rows.""" - if not path.exists(): - return [] - with path.open("r", newline="", encoding="utf-8") as fh: - return [dict(row) for row in csv.DictReader(fh)] - - -def latest_command_states(rows: list[dict[str, str]]) -> dict[str, dict[str, str]]: - """Return the latest row per command name.""" - latest: dict[str, dict[str, str]] = {} - for row in rows: - name = row.get("name", "") - if name: - latest[name] = row - return latest - - -def active_commands( - rows: list[dict[str, str]], - live_command_names: set[str] | None = None, -) -> list[dict[str, str]]: - """Return commands whose latest state is started.""" - latest = latest_command_states(rows) - active = [row for row in latest.values() if row.get("state") == "started"] - if live_command_names is None: - return active - return [row for row in active if row.get("name", "") in live_command_names] - - -def summarize_command_states( - rows: list[dict[str, str]], - live_command_names: set[str] | None = None, -) -> Counter[str]: - """Count latest command states.""" - latest = latest_command_states(rows) - states: list[str] = [] - for row in latest.values(): - state = row.get("state", "unknown") - if ( - live_command_names is not None - and state == "started" - and row.get("name", "") not in live_command_names - ): - state = "stale_started" - states.append(state) - return Counter(states) - - -def process_count_for_run(artifact_root: Path) -> int: - """Count live processes for the sandbox run using Windows process command lines.""" - token = artifact_root.name - command = ( - "Get-CimInstance Win32_Process | " - "Where-Object { " - "($_.CommandLine -like '*run_regret_auditability_sandbox.py*' -or " - "$_.CommandLine -like '*train_pd_model.py*' -or " - "$_.CommandLine -like '*generate_conformal_intervals.py*' -or " - "$_.CommandLine -like '*run_portfolio_bound_aware_search.py*') -and " - f"$_.CommandLine -notlike '*monitor_regret_auditability*' -and " - f"$_.CommandLine -like '*{token}*' " - "} | Measure-Object | Select-Object -ExpandProperty Count" - ) - try: - proc = subprocess.run( - ["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command], - capture_output=True, - text=True, - check=False, - timeout=10, - ) - except (OSError, subprocess.TimeoutExpired): - return -1 - try: - return int(proc.stdout.strip() or "0") - except ValueError: - return -1 - - -def live_process_command_lines_for_run(artifact_root: Path) -> list[str]: - """Return live sandbox command lines excluding the monitor itself.""" - token = artifact_root.name - command = ( - "Get-CimInstance Win32_Process | " - "Where-Object { " - "($_.CommandLine -like '*run_regret_auditability_sandbox.py*' -or " - "$_.CommandLine -like '*train_pd_model.py*' -or " - "$_.CommandLine -like '*generate_conformal_intervals.py*' -or " - "$_.CommandLine -like '*run_portfolio_bound_aware_search.py*') -and " - "$_.CommandLine -notlike '*monitor_regret_auditability*' -and " - f"$_.CommandLine -like '*{token}*' " - "} | Select-Object -ExpandProperty CommandLine | ConvertTo-Json -Compress" - ) - try: - proc = subprocess.run( - ["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command], - capture_output=True, - text=True, - check=False, - timeout=10, - ) - except (OSError, subprocess.TimeoutExpired): - return [] - raw = proc.stdout.strip() - if not raw: - return [] - try: - payload = json.loads(raw) - except json.JSONDecodeError: - return [line.strip() for line in proc.stdout.splitlines() if line.strip()] - if isinstance(payload, str): - return [payload] - if isinstance(payload, list): - return [str(item) for item in payload if str(item).strip()] - return [] - - -def live_command_names_for_run(artifact_root: Path) -> set[str]: - """Infer active command names from live process command lines.""" - names: set[str] = set() - pd_config_pattern = re.compile( - r"[\\/]configs[\\/]pd_(?P.+?)_(?Ppd-smoke|pd-broad|pd-refine)\.yaml" - ) - for command_line in live_process_command_lines_for_run(artifact_root): - match = pd_config_pattern.search(command_line) - if match: - names.add(f"{match.group('phase')}_{match.group('lane')}") - if "generate_conformal_intervals.py" in command_line: - names.add("conformal_extensive_grid") - if "run_portfolio_bound_aware_search.py" in command_line: - names.add("portfolio_extensive_frontier") - if "--phase" in command_line and "metrics" in command_line: - names.add("metrics_manifest") - return names - - -def optuna_db_from_checkpoint(checkpoint: str) -> Path | None: - """Resolve the Optuna DB path from a command checkpoint path.""" - if not checkpoint: - return None - checkpoint_path = Path(checkpoint) - phase_root = checkpoint_path.parent.parent - db_path = phase_root / "optuna_pd_catboost.db" - return db_path if db_path.exists() else None - - -def optuna_summary(db_path: Path | None) -> dict[str, Any]: - """Read the newest Optuna study summary from SQLite storage.""" - if db_path is None: - return {"counts": Counter()} - try: - con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) - study_row = con.execute( - "select study_id, study_name from studies order by study_id desc limit 1" - ).fetchone() - if study_row is None: - con.close() - return {"counts": Counter()} - study_id = int(study_row[0]) - rows = con.execute( - "select state, count(*) from trials where study_id = ? group by state", - (study_id,), - ).fetchall() - best_row = con.execute( - """ - select tv.value - from trial_values tv - join trials t on tv.trial_id = t.trial_id - where t.study_id = ? and t.state = 'COMPLETE' - order by tv.value desc - limit 1 - """, - (study_id,), - ).fetchone() - con.close() - except sqlite3.Error: - return {"counts": Counter()} - return { - "study_id": study_id, - "study_name": str(study_row[1]), - "counts": Counter({str(state): int(count) for state, count in rows}), - "best_value": None if best_row is None else float(best_row[0]), - } - - -def command_subphase(name: str) -> str: - """Make command names easier to scan.""" - if "_" not in name: - return name - phase, lane = name.split("_", 1) - return f"{phase} | {lane}" - - -def tail_lines(path: Path, n_lines: int) -> list[str]: - """Read the last n lines from a text file.""" - if not path.exists(): - return [] - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - return lines[-n_lines:] - - -def tail_lines_from_last_start(path: Path, n_lines: int) -> list[str]: - """Read recent lines from the latest command START marker.""" - if not path.exists(): - return [] - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - start_indexes = [index for index, line in enumerate(lines) if "] START " in line] - if start_indexes: - lines = lines[start_indexes[-1] :] - return lines[-n_lines:] - - -def row_stderr_path(row: dict[str, str], artifact_root: Path) -> Path: - """Resolve per-command stderr path, including legacy command_log rows.""" - raw = row.get("stderr_log", "") - if raw: - return Path(raw) - phase = row.get("phase", "unknown") - name = row.get("name", "unknown").replace("/", "_").replace("\\", "_") - return artifact_root / "logs" / phase / f"{name}.err.log" - - -def classify_failure_reason(row: dict[str, str], artifact_root: Path) -> str: - """Classify the latest known failure from stderr text.""" - stderr_path = row_stderr_path(row, artifact_root) - lines = tail_lines(stderr_path, 120) - start_indexes = [index for index, line in enumerate(lines) if "] START " in line] - if start_indexes: - lines = lines[start_indexes[-1] :] - tail = "\n".join(lines) - if "MissingRunTagError" in tail or "PIPELINE_RUN_TAG" in tail: - return "missing PIPELINE_RUN_TAG before patch" - if "fairness_policy.yaml" in tail: - return "missing fairness policy before decision-threshold patch" - if row.get("returncode") == "4294967295": - return "interrupted/killed during relaunch" - if "FileNotFoundError" in tail: - return "FileNotFoundError" - if "Traceback" in tail: - return "Python exception; see stderr" - return "see stderr" - - -def render(artifact_root: Path) -> None: - """Render one monitor frame.""" - heartbeat = load_json(artifact_root / "heartbeat.json") - manifest = load_json(artifact_root / "sandbox_manifest.json") - rows = load_command_rows(artifact_root / "command_log.csv") - live_command_names = live_command_names_for_run(artifact_root) - active = active_commands(rows, live_command_names) - state_counts = summarize_command_states(rows, live_command_names) - process_count = process_count_for_run(artifact_root) - running = process_count > 1 - - emit("CRPTO REGRET-AUDITABILITY SANDBOX MONITOR") - emit("=" * 78) - emit(f"Artifact root : {artifact_root}") - emit(f"Running : {'YES' if running else 'NO'} ({process_count} matching processes)") - emit( - "Phase/state : {} / {}".format( - heartbeat.get("phase", "unknown"), - heartbeat.get("state", "unknown"), - ) - ) - emit( - "Units : {}/{}".format( - heartbeat.get("completed_units", "?"), - heartbeat.get("total_units", "?"), - ) - ) - emit( - "Resources : CPU {}% | RAM free {} GB | Disk free {} GB".format( - heartbeat.get("cpu_percent", "?"), - _round_or_unknown(heartbeat.get("ram_available_gb")), - _round_or_unknown(heartbeat.get("disk_free_gb")), - ) - ) - emit(f"Last beat UTC : {heartbeat.get('captured_at_utc', 'missing')}") - emit(f"Checkpoint : {heartbeat.get('last_checkpoint_path', '')}") - mlflow_tracking = manifest.get("mlflow_tracking", {}) - if isinstance(mlflow_tracking, dict) and mlflow_tracking: - emit(f"MLflow : {mlflow_tracking.get('tracking_uri', '?')}") - emit(f"Experiment : {mlflow_tracking.get('experiment_name', '?')}") - emit() - - emit("COMMAND STATES") - emit("-" * 78) - if state_counts: - emit(" | ".join(f"{state}: {count}" for state, count in sorted(state_counts.items()))) - else: - emit("No command_log.csv yet.") - emit() - - latest = latest_command_states(rows) - failed_rows = [row for row in latest.values() if row.get("state") == "failed"] - if failed_rows: - emit("LATEST FAILED COMMANDS") - emit("-" * 78) - for row in sorted(failed_rows, key=lambda item: item.get("captured_at_utc", ""))[-8:]: - emit( - f"- {command_subphase(row.get('name', 'unknown'))}: " - f"{classify_failure_reason(row, artifact_root)}" - ) - emit() - - emit("ACTIVE SUBPHASES") - emit("-" * 78) - if not active: - emit("No active commands in command_log.csv.") - for row in active[:10]: - phase = row.get("phase", "") - target = PHASE_TRIAL_TARGETS.get(phase) - summary = optuna_summary(optuna_db_from_checkpoint(row.get("checkpoint", ""))) - counts = summary.get("counts", Counter()) - complete = counts.get("COMPLETE", 0) - running_trials = counts.get("RUNNING", 0) - failed = counts.get("FAIL", 0) + counts.get("FAILED", 0) - trial_text = "no Optuna DB yet" - if counts: - trial_text = ( - "Optuna trials complete/running/historical_failed: " - f"{complete}/{running_trials}/{failed}" - ) - if target is not None: - trial_text += f" of target {target}" - if failed: - trial_text += " (failed trials are kept for resume/accounting)" - emit(f"- {command_subphase(row.get('name', 'unknown'))}") - emit(f" {trial_text}") - if summary.get("best_value") is not None: - emit(f" best validation AUC: {float(summary['best_value']):.6f}") - if summary.get("study_name"): - emit(f" study: {summary['study_name']}") - emit() - - emit("SELECTION FILES") - emit("-" * 78) - for phase in ("pd-smoke", "pd-broad", "pd-refine"): - path = artifact_root / "pd" / "_selection" / f"{phase}_selection.json" - status = "ready" if path.exists() else "pending" - emit(f"{phase:9s}: {status} {path if path.exists() else ''}") - emit() - - emit("ACTIVE COMMAND STDERR") - emit("-" * 78) - active_log = row_stderr_path(active[0], artifact_root) if active else None - lines = tail_lines_from_last_start(active_log, 10) if active_log is not None else [] - if active_log is not None: - emit(str(active_log)) - if not lines: - lines = tail_lines(artifact_root / "orchestrator.err.log", 12) - for line in lines: - emit(line[:160]) - emit() - emit("Ctrl+C to stop monitor. The sandbox keeps running in background.") - - -def _round_or_unknown(value: Any) -> str: - try: - return f"{float(value):.2f}" - except (TypeError, ValueError): - return "?" - - -def main(argv: list[str] | None = None) -> int: - """Run the monitor loop.""" - parser = argparse.ArgumentParser() - parser.add_argument("--artifact-root", default=str(DEFAULT_ARTIFACT_ROOT)) - parser.add_argument("--interval", type=int, default=60) - parser.add_argument("--once", action="store_true") - args = parser.parse_args(argv) - - artifact_root = Path(args.artifact_root).expanduser().resolve() - interval = max(5, int(args.interval)) - while True: - clear_screen() - render(artifact_root) - if args.once: - return 0 - time.sleep(interval) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/archive/search/resume_conformal_reopen_closure.py b/scripts/archive/search/resume_conformal_reopen_closure.py deleted file mode 100644 index 23db70d..0000000 --- a/scripts/archive/search/resume_conformal_reopen_closure.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Derived resume entrypoint for conformal reopen closure.""" - -from __future__ import annotations - -import argparse -import sys -from datetime import UTC, datetime -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.search.run_conformal_reopen_search import main as reopen_main # noqa: E402 - - -def _default_derived_tag(source_run_tag: str) -> str: - stamp = datetime.now(UTC).strftime("%Y-%m-%d-%H%M") - return f"{str(source_run_tag).strip()}__resume__{stamp}" - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--source-run-tag", required=True) - parser.add_argument("--derived-run-tag", default=None) - parser.add_argument("--upstream-canonical-run-tag", required=True) - parser.add_argument("--pipeline-profile", default="search_conformal_reopen_exhaustive") - parser.add_argument("--top-k-finalists", type=int, default=3) - parser.add_argument( - "--phase2-enabled", - default="true", - choices=["true", "false"], - ) - args = parser.parse_args(argv) - - derived_run_tag = ( - str(args.derived_run_tag).strip() - if args.derived_run_tag - else _default_derived_tag(args.source_run_tag) - ) - reopen_args = [ - "--run-tag", - derived_run_tag, - "--pipeline-profile", - str(args.pipeline_profile), - "--upstream-canonical-run-tag", - str(args.upstream_canonical_run_tag), - "--resume-from-run-tag", - str(args.source_run_tag), - ] - # top-k is governed by the profile validation config; this script exposes it - # for parity with the recovery protocol even though the current profile default is 3. - if int(args.top_k_finalists) != 3: - raise ValueError("top-k-finalists is fixed to 3 in the current recovery protocol.") - if str(args.phase2_enabled).strip().lower() == "false": - reopen_args.append("--phase1-only") - return reopen_main(reopen_args) - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/archive/search/sweep_alpha_gamma_bound_finalists.py b/scripts/archive/search/sweep_alpha_gamma_bound_finalists.py deleted file mode 100644 index 4f81c81..0000000 --- a/scripts/archive/search/sweep_alpha_gamma_bound_finalists.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Run alpha-gamma bound validation across multiple conformal finalists.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.validate_alpha_gamma_bound import main as validate_bound_main # noqa: E402 - - -def _coerce_csv(raw: str | None) -> list[str]: - if not raw: - return [] - return [token.strip() for token in str(raw).split(",") if token.strip()] - - -def _default_label(path: str) -> str: - return (Path(path).stem or "finalist").replace("/", "_") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--interval-paths", required=True) - parser.add_argument("--labels", default=None) - parser.add_argument("--portfolio-policy-path", default="models/champion_portfolio_policy.json") - parser.add_argument("--alpha-grid", default=None) - parser.add_argument("--max-candidates", type=int, default=5000) - parser.add_argument("--random-state", type=int, default=42) - parser.add_argument( - "--output-parquet", - default="data/processed/alpha_gamma_bound_finalist_sweep.parquet", - ) - parser.add_argument( - "--output-json", - default="data/processed/alpha_gamma_bound_finalist_sweep.json", - ) - args = parser.parse_args(argv) - - interval_paths = _coerce_csv(args.interval_paths) - labels = _coerce_csv(args.labels) - if labels and len(labels) != len(interval_paths): - raise ValueError("labels and interval-paths must have the same number of entries") - if not labels: - labels = [_default_label(path) for path in interval_paths] - - rows: list[dict[str, object]] = [] - for label, interval_path in zip(labels, interval_paths, strict=True): - output_json = ( - Path("data/processed/alpha_gamma_bound") - / f"{label}_alpha_gamma_bound_validation_exact.json" - ) - validate_bound_main( - [ - "--conformal-intervals-path", - interval_path, - "--portfolio-policy-path", - args.portfolio_policy_path, - "--allocator-mode", - "exact", - "--output-json", - str(output_json), - "--alpha-grid", - str(args.alpha_grid or ""), - "--max-candidates", - str(int(args.max_candidates)), - "--random-state", - str(int(args.random_state)), - "--figure-prefix", - str(Path("reports/crpto/figures") / f"{label}_alpha_gamma_bound"), - "--comparison-output", - str(Path("data/processed/alpha_gamma_bound") / f"{label}_proxy_vs_exact.parquet"), - ] - ) - payload = json.loads(output_json.read_text(encoding="utf-8")) - for result in payload.get("results", []): - rows.append( - { - "label": label, - "conformal_intervals_path": interval_path, - **result, - } - ) - - sweep = pd.DataFrame(rows) - output_parquet = Path(args.output_parquet) - output_json_path = Path(args.output_json) - output_parquet.parent.mkdir(parents=True, exist_ok=True) - output_json_path.parent.mkdir(parents=True, exist_ok=True) - sweep.to_parquet(output_parquet, index=False) - output_json_path.write_text( - json.dumps( - { - "n_rows": len(sweep), - "rows": sweep.to_dict(orient="records"), - }, - indent=2, - ensure_ascii=False, - ) - + "\n", - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/benchmark_conformal_variants.py b/scripts/benchmark_conformal_variants.py deleted file mode 100644 index 65741c3..0000000 --- a/scripts/benchmark_conformal_variants.py +++ /dev/null @@ -1,1072 +0,0 @@ -"""Benchmark conformal variants for coverage/efficiency trade-offs. - -Usage: - uv run python scripts/benchmark_conformal_variants.py -""" - -from __future__ import annotations - -import argparse -import json -import pickle -from dataclasses import dataclass, field -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from loguru import logger - -from scripts.generate_conformal_intervals import ( - _build_feature_matrix, - _load_calibrator, - _load_model, - _resolve_features, - _subset_calibration_frame, -) -from src.evaluation.backtesting import winkler_interval_score -from src.models.conformal import ( - apply_probability_calibrator, - build_mondrian_partition_labels, - conditional_coverage_by_group, - create_cross_conformal_score_intervals, - create_pd_intervals, - create_pd_intervals_mondrian_from_predictions, - validate_coverage, -) -from src.utils.io_utils import read_with_fallback - -TARGET_COL = "default_flag" -GROUP_COL = "grade" -DEFAULT_POLICY_CONFIG = "configs/crpto_conformal_policy.yaml" - - -@dataclass(frozen=True) -class BenchmarkData: - model: Any - calibrator: Any | None - cal_df: pd.DataFrame - test_df: pd.DataFrame - features: list[str] - categorical: list[str] - X_cal: pd.DataFrame - y_cal: pd.Series - X_test: pd.DataFrame - y_test: np.ndarray - group_cal: pd.Series - group_test: pd.Series - issue_test: pd.Series - y_prob_cal_raw: np.ndarray - y_prob_test_raw: np.ndarray - y_prob_calibrated: np.ndarray - y_prob_test_calibrated: np.ndarray - prob_cal_lookup: dict[str, np.ndarray] - prob_test_lookup: dict[str, np.ndarray] - - -@dataclass(frozen=True) -class SearchSpace: - partition_candidates: tuple[str, ...] - partition_probability_sources: tuple[str, ...] - n_score_bins_candidates: tuple[int, ...] - fallback_modes: tuple[str, ...] - score_scale_families: tuple[str, ...] - min_group_sizes: tuple[int, ...] - calibration_size_fractions: tuple[float, ...] - - -@dataclass -class VariantResults: - rows: list[dict[str, Any]] = field(default_factory=list) - by_group_rows: list[pd.DataFrame] = field(default_factory=list) - temporal_rows: list[pd.DataFrame] = field(default_factory=list) - local_rows: list[pd.DataFrame] = field(default_factory=list) - - -def _summarize_variant( - name: str, - y_true: np.ndarray, - y_intervals: np.ndarray, - groups: pd.Series, - alpha: float, -) -> tuple[dict[str, Any], pd.DataFrame]: - metrics = validate_coverage(y_true, y_intervals, alpha=alpha, log_summary=False) - by_group = conditional_coverage_by_group(y_true, y_intervals, groups) - widths = y_intervals[:, 1] - y_intervals[:, 0] - winkler_90 = float( - np.mean(winkler_interval_score(y_true, y_intervals[:, 0], y_intervals[:, 1], alpha=alpha)) - ) - row = { - "variant": name, - "alpha": float(alpha), - "target_coverage": float(1.0 - alpha), - "coverage": float(metrics["empirical_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_width": float(metrics["avg_interval_width"]), - "median_width": float(metrics["median_interval_width"]), - "winkler_90": winkler_90, - "p90_width": float(np.quantile(widths, 0.90)), - "p95_width": float(np.quantile(widths, 0.95)), - "min_group_coverage": float(by_group["coverage"].min()), - "max_group_coverage": float(by_group["coverage"].max()), - "std_group_coverage": float(by_group["coverage"].std(ddof=0)), - } - by_group = by_group.copy() - by_group["variant"] = name - by_group["alpha"] = float(alpha) - return row, by_group - - -def _summarize_temporal_stability( - name: str, - y_true: np.ndarray, - y_intervals: np.ndarray, - issue_dates: pd.Series, -) -> tuple[dict[str, float], pd.DataFrame]: - dates = pd.to_datetime(issue_dates, errors="coerce") - frame = pd.DataFrame( - { - "variant": name, - "month": dates.dt.to_period("M").dt.to_timestamp(), - "y_true": np.asarray(y_true, dtype=float), - "low": np.asarray(y_intervals[:, 0], dtype=float), - "high": np.asarray(y_intervals[:, 1], dtype=float), - } - ).dropna(subset=["month"]) - if frame.empty: - return ( - { - "min_monthly_coverage": float("nan"), - "last_monthly_coverage": float("nan"), - "max_monthly_gap": float("nan"), - "stability_over_time": float("inf"), - }, - pd.DataFrame( - { - "variant": pd.Series(dtype="object"), - "month": pd.Series(dtype="object"), - "n": pd.Series(dtype="int64"), - "coverage_90": pd.Series(dtype="float64"), - "avg_width_90": pd.Series(dtype="float64"), - "coverage_gap_90": pd.Series(dtype="float64"), - } - ), - ) - - frame["covered"] = ( - (frame["y_true"] >= frame["low"]) & (frame["y_true"] <= frame["high"]) - ).astype(float) - frame["width"] = frame["high"] - frame["low"] - monthly = ( - frame.groupby("month", observed=True) - .agg( - n=("covered", "size"), - coverage_90=("covered", "mean"), - avg_width_90=("width", "mean"), - ) - .reset_index() - .sort_values("month") - ) - monthly["coverage_gap_90"] = (monthly["coverage_90"] - 0.90).abs() - stability = { - "min_monthly_coverage": float(monthly["coverage_90"].min()), - "last_monthly_coverage": float(monthly["coverage_90"].iloc[-1]), - "max_monthly_gap": float(monthly["coverage_gap_90"].max()), - "stability_over_time": float(monthly["coverage_gap_90"].mean()), - } - monthly.insert(0, "variant", name) - return stability, monthly - - -def _load_policy_config(path: str = DEFAULT_POLICY_CONFIG) -> dict[str, Any]: - with open(path, encoding="utf-8") as f: - return yaml.safe_load(f) or {} - - -def _promotion_pass(row: pd.Series, policy: dict[str, Any]) -> bool: - return bool( - float(row.get("coverage", 0.0)) >= float(policy.get("target_coverage_90_min", 0.90)) - and float(row.get("min_group_coverage", 0.0)) - >= float(policy.get("min_group_coverage_90_min", 0.88)) - and float(row.get("winkler_90", float("inf"))) <= float(policy.get("max_winkler_90", 1.20)) - and float(row.get("avg_width", float("inf"))) <= float(policy.get("max_avg_width_90", 0.80)) - ) - - -def _build_output_paths(namespace: str | None = None) -> dict[str, Path]: - if namespace: - ns = str(namespace).strip().replace("/", "_") - data_dir = Path("data/processed/conformal_gap") / ns - models_dir = Path("models/conformal_gap") / ns - else: - data_dir = Path("data/processed") - models_dir = Path("models") - data_dir.mkdir(parents=True, exist_ok=True) - models_dir.mkdir(parents=True, exist_ok=True) - return { - "data_dir": data_dir, - "models_dir": models_dir, - "benchmark": data_dir / "conformal_variant_benchmark.parquet", - "benchmark_by_group": data_dir / "conformal_variant_benchmark_by_group.parquet", - "selection_report": data_dir / "conformal_variant_selection_report.parquet", - "temporal_diagnostics": data_dir / "conformal_temporal_diagnostics.parquet", - "local_diagnostics": data_dir / "conformal_local_diagnostics.parquet", - "selection_status": models_dir / "conformal_variant_selection_status.json", - "selected_intervals": data_dir / "conformal_intervals_mondrian.parquet", - } - - -def _coerce_csv_tuple(raw: str | None, *, cast=str) -> tuple[Any, ...]: - if raw is None: - return () - values = [] - for token in str(raw).split(","): - token = token.strip() - if not token: - continue - values.append(cast(token)) - return tuple(values) - - -def _variant_name( - *, - partition: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - score_scale_family: str, - min_group_size: int, - calibration_fraction: float | None = None, -) -> str: - parts = [ - str(partition), - f"prob={partition_probability_source}", - f"bins={int(n_score_bins)}", - f"fallback={fallback_mode}", - f"scale={score_scale_family}", - f"mgs={int(min_group_size)}", - ] - if calibration_fraction is not None: - parts.append(f"calfrac={float(calibration_fraction):.2f}") - return "::".join(parts) - - -def _load_benchmark_data(calibrator_override_path: str | None) -> BenchmarkData: - model, _ = _load_model() - calibrator = _load_calibrator(calibrator_override_path) - cal_df = read_with_fallback( - "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" - ) - test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df[TARGET_COL].astype(float) - X_test = _build_feature_matrix(test_df, features, categorical) - y_test = test_df[TARGET_COL].astype(float).to_numpy(dtype=float) - group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - issue_test = test_df.get("issue_d", pd.Series([pd.NaT] * len(test_df))).reset_index(drop=True) - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - y_prob_test_raw = model.predict_proba(X_test)[:, 1] - y_prob_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_raw) - if calibrator is not None - else np.asarray(y_prob_cal_raw, dtype=float) - ) - y_prob_test_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_test_raw) - if calibrator is not None - else np.asarray(y_prob_test_raw, dtype=float) - ) - return BenchmarkData( - model=model, - calibrator=calibrator, - cal_df=cal_df, - test_df=test_df, - features=features, - categorical=categorical, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - y_test=y_test, - group_cal=group_cal, - group_test=group_test, - issue_test=issue_test, - y_prob_cal_raw=y_prob_cal_raw, - y_prob_test_raw=y_prob_test_raw, - y_prob_calibrated=y_prob_calibrated, - y_prob_test_calibrated=y_prob_test_calibrated, - prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, - prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, - ) - - -def _unique_clean_strings(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: - cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) - return cleaned or fallback - - -def _unique_clean_lower_strings( - values: tuple[str, ...], - fallback: tuple[str, ...], -) -> tuple[str, ...]: - cleaned = tuple(dict.fromkeys(str(x).strip().lower() for x in values if str(x).strip())) - return cleaned or fallback - - -def _positive_int_tuple(values: tuple[int, ...], fallback: tuple[int, ...]) -> tuple[int, ...]: - cleaned = tuple(int(x) for x in values if int(x) > 0) - return cleaned or fallback - - -def _min_group_size_tuple( - values: tuple[int, ...] | None, - default: int, -) -> tuple[int, ...]: - return _positive_int_tuple(values or (default,), (int(default),)) - - -def _valid_fraction_tuple(values: tuple[float, ...]) -> tuple[float, ...]: - return tuple(float(x) for x in values if 0 < float(x) <= 1) - - -def _normalize_search_space( - *, - calibration_size_fractions: tuple[float, ...], - partition_candidates: tuple[str, ...], - partition_probability_sources: tuple[str, ...], - n_score_bins_candidates: tuple[int, ...], - fallback_modes: tuple[str, ...], - score_scale_families: tuple[str, ...], - min_group_sizes: tuple[int, ...] | None, - min_group_size_default: int, -) -> SearchSpace: - return SearchSpace( - partition_candidates=_unique_clean_strings(partition_candidates, ("grade",)), - partition_probability_sources=_unique_clean_lower_strings( - partition_probability_sources, ("raw",) - ), - n_score_bins_candidates=_positive_int_tuple(n_score_bins_candidates, (10,)), - fallback_modes=_unique_clean_lower_strings(fallback_modes, ("grade_then_global",)), - score_scale_families=_unique_clean_lower_strings(score_scale_families, ("none",)), - min_group_sizes=_min_group_size_tuple(min_group_sizes, min_group_size_default), - calibration_size_fractions=_valid_fraction_tuple(calibration_size_fractions), - ) - - -def _append_global_variant( - *, - data: BenchmarkData, - results: VariantResults, - alpha: float, -) -> None: - _y_pred_global, y_int_global = create_pd_intervals( - classifier=data.model, - X_cal=data.X_cal, - y_cal=data.y_cal, - X_test=data.X_test, - alpha=alpha, - calibrator=data.calibrator, - ) - row, by_group = _summarize_variant( - "global_split", data.y_test, y_int_global, data.group_test, alpha - ) - temporal_meta, temporal_monthly = _summarize_temporal_stability( - "global_split", data.y_test, y_int_global, data.issue_test - ) - row.update(temporal_meta) - results.rows.append(row) - results.by_group_rows.append(by_group) - results.temporal_rows.append(temporal_monthly) - - -def _append_mondrian_variant( - *, - data: BenchmarkData, - results: VariantResults, - partition_cache: dict[ - tuple[str, str, int, str, int, float, int], - tuple[pd.Series, pd.Series, dict[str, Any]], - ], - name: str, - partition: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - score_scale_family: str, - alpha: float, - alpha_used: float, - min_group_size: int, - collect_local_diagnostics: bool, - y_cal_variant: pd.Series | None = None, - y_prob_cal_variant: np.ndarray | None = None, - base_groups_cal_variant: pd.Series | None = None, - calibration_fraction: float | None = None, -) -> None: - y_cal_use = data.y_cal if y_cal_variant is None else y_cal_variant - y_prob_cal_use = ( - data.prob_cal_lookup[partition_probability_source] - if y_prob_cal_variant is None - else y_prob_cal_variant - ) - y_interval_cal_pred = ( - data.y_prob_calibrated if y_prob_cal_variant is None else y_prob_cal_variant - ) - base_groups_cal_use = ( - data.group_cal if base_groups_cal_variant is None else base_groups_cal_variant - ) - cache_key = ( - str(partition), - str(partition_probability_source), - int(n_score_bins), - str(fallback_mode), - int(min_group_size), - float(calibration_fraction or 1.0), - int(len(y_cal_use)), - ) - if cache_key in partition_cache: - group_cal_part, group_test_part, partition_meta = partition_cache[cache_key] - else: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_use, - y_prob_eval=data.prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=base_groups_cal_use.iloc[: len(y_cal_use)].reset_index(drop=True), - base_groups_eval=data.group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=fallback_mode, - ) - partition_cache[cache_key] = (group_cal_part, group_test_part, partition_meta) - _y_pred, y_int, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_interval_cal_pred, - y_test_pred=data.y_prob_test_calibrated, - y_cal=y_cal_use, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha_used, - min_group_size=min_group_size, - score_scale_family=score_scale_family, - log_summary=False, - ) - row, by_group = _summarize_variant(name, data.y_test, y_int, group_test_part, alpha) - temporal_meta, temporal_monthly = _summarize_temporal_stability( - name, data.y_test, y_int, data.issue_test - ) - row.update(temporal_meta) - row["partition"] = partition_meta.get("partition", partition) - row["partition_probability_source"] = partition_probability_source - row["n_score_bins"] = int(n_score_bins) - row["fallback_mode"] = str(partition_meta.get("fallback_mode", fallback_mode)) - row["scaled_scores"] = bool(score_scale_family != "none") - row["score_scale_family"] = score_scale_family - row["min_group_size"] = int(min_group_size) - row["selected_alpha_used"] = float(alpha_used) - row["fallback_groups_n"] = len(partition_meta.get("fallback_groups", [])) - row["calibration_fraction"] = float(calibration_fraction or 1.0) - results.rows.append(row) - results.by_group_rows.append(by_group) - results.temporal_rows.append(temporal_monthly) - - if collect_local_diagnostics: - local_diag = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": name, - "partition": row["partition"], - "group": pd.Series(group_test_part).astype(str), - "y_true": data.y_test, - "low": y_int[:, 0], - "high": y_int[:, 1], - } - ) - local_diag["covered"] = ( - (local_diag["y_true"] >= local_diag["low"]) - & (local_diag["y_true"] <= local_diag["high"]) - ).astype(float) - local_diag["width"] = local_diag["high"] - local_diag["low"] - results.local_rows.append(local_diag) - - -def _append_search_space_variants( - *, - data: BenchmarkData, - results: VariantResults, - partition_cache: dict[ - tuple[str, str, int, str, int, float, int], - tuple[pd.Series, pd.Series, dict[str, Any]], - ], - space: SearchSpace, - alpha: float, - collect_local_diagnostics: bool, -) -> None: - for partition in space.partition_candidates: - for partition_probability_source in space.partition_probability_sources: - for n_score_bins in space.n_score_bins_candidates: - for fallback_mode in space.fallback_modes: - for score_scale_family in space.score_scale_families: - for min_group_size in space.min_group_sizes: - _append_mondrian_variant( - data=data, - results=results, - partition_cache=partition_cache, - name=_variant_name( - partition=partition, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - score_scale_family=score_scale_family, - min_group_size=min_group_size, - ), - partition=partition, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - score_scale_family=score_scale_family, - alpha=alpha, - alpha_used=alpha, - min_group_size=min_group_size, - collect_local_diagnostics=collect_local_diagnostics, - ) - - -def _sample_indices(n_rows: int, sample_size: int, rng: np.random.RandomState) -> np.ndarray: - return ( - rng.choice(n_rows, size=min(int(sample_size), n_rows), replace=False) - if n_rows > int(sample_size) - else np.arange(n_rows) - ) - - -def _append_cross_conformal_variant( - *, - data: BenchmarkData, - results: VariantResults, - alpha: float, - cross_cal_sample_size: int, - cross_test_sample_size: int, -) -> None: - rng = np.random.RandomState(42) - cal_idx = _sample_indices(len(data.y_cal), cross_cal_sample_size, rng) - test_idx = _sample_indices(len(data.y_test), cross_test_sample_size, rng) - _y_pred_cross, y_int_cross = create_cross_conformal_score_intervals( - y_cal=data.y_cal.iloc[cal_idx].reset_index(drop=True), - y_prob_cal=data.y_prob_calibrated[cal_idx], - y_prob_test=data.y_prob_test_calibrated[test_idx], - alpha=alpha, - method="plus", - cv=5, - ) - row, by_group = _summarize_variant( - "cross_conformal_score_space", - data.y_test[test_idx], - y_int_cross, - data.group_test.iloc[test_idx].reset_index(drop=True), - alpha, - ) - temporal_meta, temporal_monthly = _summarize_temporal_stability( - "cross_conformal_score_space", - data.y_test[test_idx], - y_int_cross, - data.issue_test.iloc[test_idx].reset_index(drop=True), - ) - row.update(temporal_meta) - row["implementation_note"] = ( - "Cross conformal executed on calibrated score space with a lightweight linear regressor." - ) - row["evaluation_sample_n_cal"] = len(cal_idx) - row["evaluation_sample_n_test"] = len(test_idx) - results.rows.append(row) - results.by_group_rows.append(by_group) - results.temporal_rows.append(temporal_monthly) - - -def _append_selected_config_variant( - *, - data: BenchmarkData, - results: VariantResults, - partition_cache: dict[ - tuple[str, str, int, str, int, float, int], - tuple[pd.Series, pd.Series, dict[str, Any]], - ], - selected_config_path: str, - alpha: float, - min_group_size_default: int, - collect_local_diagnostics: bool, -) -> None: - cfg_path = Path(selected_config_path) - if not cfg_path.exists(): - return - with open(cfg_path, "rb") as f: - payload = pickle.load(f) - best = payload.get("tuning_90_best", {}) if isinstance(payload, dict) else {} - _append_mondrian_variant( - data=data, - results=results, - partition_cache=partition_cache, - name="mondrian_selected_cfg", - partition=str(best.get("partition", "grade")), - partition_probability_source=str(best.get("partition_probability_source", "raw")), - n_score_bins=int(best.get("n_score_bins", 10)), - fallback_mode=str(best.get("fallback_mode", "grade_then_global")), - score_scale_family=str(best.get("score_scale_family", "none")), - alpha=alpha, - alpha_used=float(best.get("alpha_used_90", alpha)), - min_group_size=int(best.get("min_group_size", min_group_size_default)), - collect_local_diagnostics=collect_local_diagnostics, - ) - - -def _calibration_sensitivity_rows( - *, - data: BenchmarkData, - space: SearchSpace, - alpha: float, -) -> list[dict[str, Any]]: - sensitivity_rows: list[dict[str, Any]] = [] - for frac in space.calibration_size_fractions: - cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=float(frac)) - X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) - y_cal_sub = cal_df_sub[TARGET_COL].astype(float).reset_index(drop=True) - group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] - y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) - if data.calibrator is not None - else np.asarray(y_prob_cal_sub_raw, dtype=float) - ) - prob_cal_sub_lookup = {"raw": y_prob_cal_sub_raw, "calibrated": y_prob_cal_sub_calibrated} - for partition in ("score_decile_mondrian", "grade_x_scoreband_mondrian"): - for partition_probability_source in space.partition_probability_sources: - sensitivity_rows.append( - _calibration_sensitivity_row( - data=data, - space=space, - alpha=alpha, - partition=partition, - partition_probability_source=partition_probability_source, - calibration_fraction=float(frac), - X_cal_sub=X_cal_sub, - y_cal_sub=y_cal_sub, - group_cal_sub=group_cal_sub, - y_prob_cal_sub_calibrated=y_prob_cal_sub_calibrated, - y_prob_cal_sub=prob_cal_sub_lookup[partition_probability_source], - ) - ) - return sensitivity_rows - - -def _calibration_sensitivity_row( - *, - data: BenchmarkData, - space: SearchSpace, - alpha: float, - partition: str, - partition_probability_source: str, - calibration_fraction: float, - X_cal_sub: pd.DataFrame, - y_cal_sub: pd.Series, - group_cal_sub: pd.Series, - y_prob_cal_sub_calibrated: np.ndarray, - y_prob_cal_sub: np.ndarray, -) -> dict[str, Any]: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_sub, - y_prob_eval=data.prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=group_cal_sub, - base_groups_eval=data.group_test, - n_score_bins=space.n_score_bins_candidates[0], - min_group_size=space.min_group_sizes[0], - fallback_mode=space.fallback_modes[0], - ) - _y_pred_sub, y_int_sub, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_prob_cal_sub_calibrated, - y_test_pred=data.y_prob_test_calibrated, - y_cal=y_cal_sub, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - min_group_size=space.min_group_sizes[0], - score_scale_family=space.score_scale_families[0], - log_summary=False, - ) - metrics = validate_coverage(data.y_test, y_int_sub, alpha=alpha, log_summary=False) - by_group_sub = conditional_coverage_by_group(data.y_test, y_int_sub, group_test_part) - stability_sub, _ = _summarize_temporal_stability( - partition, data.y_test, y_int_sub, data.issue_test - ) - return { - "record_type": "calibration_size_sensitivity", - "variant": _variant_name( - partition=partition, - partition_probability_source=partition_probability_source, - n_score_bins=space.n_score_bins_candidates[0], - fallback_mode=space.fallback_modes[0], - score_scale_family=space.score_scale_families[0], - min_group_size=space.min_group_sizes[0], - calibration_fraction=calibration_fraction, - ), - "partition": partition_meta.get("partition", partition), - "partition_probability_source": partition_probability_source, - "calibration_fraction": calibration_fraction, - "n_calibration_rows": len(X_cal_sub), - "coverage": float(metrics["empirical_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_width": float(metrics["avg_interval_width"]), - "min_group_coverage": float(by_group_sub["coverage"].min()), - "winkler_90": float( - np.mean( - winkler_interval_score(data.y_test, y_int_sub[:, 0], y_int_sub[:, 1], alpha=alpha) - ) - ), - "stability_over_time": float(stability_sub["stability_over_time"]), - } - - -def _final_benchmark_frames( - *, - results: VariantResults, - sensitivity_rows: list[dict[str, Any]], - policy: dict[str, Any], -) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: - bench = pd.DataFrame(results.rows) - bench["promotion_pass"] = bench.apply(lambda row: _promotion_pass(row, policy), axis=1) - bench = bench.sort_values( - [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time", - ], - ascending=[False, True, False, True, True, True], - ).reset_index(drop=True) - bench["selection_rank"] = np.arange(1, len(bench) + 1, dtype=int) - bench_by_group = ( - pd.concat(results.by_group_rows, ignore_index=True) - .sort_values(["variant", "group"]) - .reset_index(drop=True) - ) - temporal_diagnostics = ( - pd.concat(results.temporal_rows, ignore_index=True) - .sort_values(["variant", "month"]) - .reset_index(drop=True) - ) - local_diagnostics = ( - pd.concat(results.local_rows, ignore_index=True) if results.local_rows else pd.DataFrame() - ) - if sensitivity_rows: - local_diagnostics = pd.concat( - [local_diagnostics, pd.DataFrame(sensitivity_rows)], - ignore_index=True, - sort=False, - ) - return bench, bench_by_group, temporal_diagnostics, local_diagnostics - - -def _selected_config_local_diagnostics( - *, - selected_config_path: str, - selected_intervals_path: Path, -) -> pd.DataFrame: - selected_cfg_path = Path(selected_config_path) - if not selected_cfg_path.exists() or not selected_intervals_path.exists(): - return pd.DataFrame() - with open(selected_cfg_path, "rb") as f: - selected_payload = pickle.load(f) - selected_intervals = pd.read_parquet(selected_intervals_path) - if not {"y_true", "pd_low_90", "pd_high_90", GROUP_COL}.issubset(selected_intervals.columns): - return pd.DataFrame() - selected_local = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": "mondrian_selected_cfg", - "partition": str(selected_payload.get("partition", "grade")), - "group": selected_intervals[GROUP_COL].fillna("UNKNOWN").astype(str), - "y_true": pd.to_numeric(selected_intervals["y_true"], errors="coerce"), - "low": pd.to_numeric(selected_intervals["pd_low_90"], errors="coerce"), - "high": pd.to_numeric(selected_intervals["pd_high_90"], errors="coerce"), - } - ) - selected_local["covered"] = ( - (selected_local["y_true"] >= selected_local["low"]) - & (selected_local["y_true"] <= selected_local["high"]) - ).astype(float) - selected_local["width"] = selected_local["high"] - selected_local["low"] - return selected_local - - -def _append_selected_local_diagnostics( - *, - local_diagnostics: pd.DataFrame, - selected_config_path: str, - selected_intervals_path: Path, -) -> pd.DataFrame: - selected_local = _selected_config_local_diagnostics( - selected_config_path=selected_config_path, - selected_intervals_path=selected_intervals_path, - ) - if selected_local.empty: - return local_diagnostics - return pd.concat([local_diagnostics, selected_local], ignore_index=True, sort=False) - - -def _selection_status_payload( - *, - bench: pd.DataFrame, - output_paths: dict[str, Path], - artifact_namespace: str | None, - calibrator_override_path: str | None, - policy_config_path: str, - collect_local_diagnostics: bool, - space: SearchSpace, -) -> dict[str, Any]: - selected = bench.iloc[0].to_dict() - return { - "schema_version": "2026-04-03.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "artifact_namespace": artifact_namespace or "", - "calibrator_override_path": str(calibrator_override_path or ""), - "policy_config_path": str(policy_config_path), - "selected_variant": str(selected.get("variant", "")), - "selection_rank": int(selected.get("selection_rank", 1)), - "promotion_pass": bool(selected.get("promotion_pass", False)), - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time", - ], - "retired_backtest_role": ( - "Kupiec/Christoffersen are research diagnostics outside the IJDS " - "promotion gate; validate_conformal_policy.py promotes on material " - "coverage, group coverage, width, alert, and Winkler checks." - ), - "local_diagnostics_mode": ( - "all_variants" if collect_local_diagnostics else "selected_config_plus_sensitivity" - ), - "variants_tested": bench["variant"].astype(str).tolist(), - "report_path": str(output_paths["selection_report"]), - "summary_path": str(output_paths["benchmark"]), - "temporal_diagnostics_path": str(output_paths["temporal_diagnostics"]), - "local_diagnostics_path": str(output_paths["local_diagnostics"]), - "selected_metrics": { - "coverage": float(selected.get("coverage", 0.0)), - "coverage_gap": float(selected.get("coverage_gap", 0.0)), - "avg_width": float(selected.get("avg_width", 0.0)), - "min_group_coverage": float(selected.get("min_group_coverage", 0.0)), - "winkler_90": float(selected.get("winkler_90", 0.0)), - "stability_over_time": float(selected.get("stability_over_time", 0.0)), - }, - "search_space": { - "partition_candidates": list(space.partition_candidates), - "partition_probability_sources": list(space.partition_probability_sources), - "n_score_bins_candidates": [int(x) for x in space.n_score_bins_candidates], - "fallback_modes": list(space.fallback_modes), - "score_scale_families": list(space.score_scale_families), - "min_group_sizes": [int(x) for x in space.min_group_sizes], - "calibration_size_fractions": [float(x) for x in space.calibration_size_fractions], - }, - "top_variants": bench.head(5).to_dict(orient="records"), - } - - -def _write_benchmark_outputs( - *, - output_paths: dict[str, Path], - bench: pd.DataFrame, - bench_by_group: pd.DataFrame, - temporal_diagnostics: pd.DataFrame, - local_diagnostics: pd.DataFrame, - status_payload: dict[str, Any], -) -> None: - bench_path = output_paths["benchmark"] - bench_group_path = output_paths["benchmark_by_group"] - selection_path = output_paths["selection_report"] - temporal_path = output_paths["temporal_diagnostics"] - local_path = output_paths["local_diagnostics"] - bench.to_parquet(bench_path, index=False) - bench_by_group.to_parquet(bench_group_path, index=False) - bench.to_parquet(selection_path, index=False) - temporal_diagnostics.to_parquet(temporal_path, index=False) - if not local_diagnostics.empty: - local_diagnostics.to_parquet(local_path, index=False) - - status_path = output_paths["selection_status"] - status_path.parent.mkdir(parents=True, exist_ok=True) - status_path.write_text(json.dumps(status_payload, indent=2, default=str), encoding="utf-8") - logger.info("Saved conformal benchmark summary: {} ({})", bench_path, bench.shape) - logger.info( - "Saved conformal benchmark by-group: {} ({})", bench_group_path, bench_by_group.shape - ) - logger.info( - "Saved conformal temporal diagnostics: {} ({})", temporal_path, temporal_diagnostics.shape - ) - if not local_diagnostics.empty: - logger.info( - "Saved conformal local diagnostics: {} ({})", local_path, local_diagnostics.shape - ) - logger.info("Saved conformal variant selection report: {}", selection_path) - logger.info("Saved conformal variant selection status: {}", status_path) - - -def main( - alpha: float = 0.10, - selected_config_path: str = "models/conformal_results_mondrian.pkl", - min_group_size_default: int = 500, - cross_cal_sample_size: int = 5000, - cross_test_sample_size: int = 5000, - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - partition_candidates: tuple[str, ...] = ( - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian", - ), - partition_probability_sources: tuple[str, ...] = ("raw",), - n_score_bins_candidates: tuple[int, ...] = (10,), - fallback_modes: tuple[str, ...] = ("grade_then_global",), - score_scale_families: tuple[str, ...] = ("none", "bernoulli_sqrt"), - min_group_sizes: tuple[int, ...] | None = None, - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, - policy_config_path: str = DEFAULT_POLICY_CONFIG, - collect_local_diagnostics: bool = False, -) -> None: - policy = _load_policy_config(policy_config_path).get("policy", {}) or {} - data = _load_benchmark_data(calibrator_override_path) - space = _normalize_search_space( - calibration_size_fractions=calibration_size_fractions, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - min_group_sizes=min_group_sizes, - min_group_size_default=min_group_size_default, - ) - results = VariantResults() - partition_cache: dict[ - tuple[str, str, int, str, int, float, int], - tuple[pd.Series, pd.Series, dict[str, Any]], - ] = {} - - _append_global_variant(data=data, results=results, alpha=alpha) - _append_search_space_variants( - data=data, - results=results, - partition_cache=partition_cache, - space=space, - alpha=alpha, - collect_local_diagnostics=collect_local_diagnostics, - ) - _append_cross_conformal_variant( - data=data, - results=results, - alpha=alpha, - cross_cal_sample_size=cross_cal_sample_size, - cross_test_sample_size=cross_test_sample_size, - ) - _append_selected_config_variant( - data=data, - results=results, - partition_cache=partition_cache, - selected_config_path=selected_config_path, - alpha=alpha, - min_group_size_default=min_group_size_default, - collect_local_diagnostics=collect_local_diagnostics, - ) - sensitivity_rows = _calibration_sensitivity_rows(data=data, space=space, alpha=alpha) - bench, bench_by_group, temporal_diagnostics, local_diagnostics = _final_benchmark_frames( - results=results, - sensitivity_rows=sensitivity_rows, - policy=policy, - ) - - output_paths = _build_output_paths(artifact_namespace) - local_diagnostics = _append_selected_local_diagnostics( - local_diagnostics=local_diagnostics, - selected_config_path=selected_config_path, - selected_intervals_path=output_paths["selected_intervals"], - ) - status_payload = _selection_status_payload( - bench=bench, - output_paths=output_paths, - artifact_namespace=artifact_namespace, - calibrator_override_path=calibrator_override_path, - policy_config_path=policy_config_path, - collect_local_diagnostics=collect_local_diagnostics, - space=space, - ) - _write_benchmark_outputs( - output_paths=output_paths, - bench=bench, - bench_by_group=bench_by_group, - temporal_diagnostics=temporal_diagnostics, - local_diagnostics=local_diagnostics, - status_payload=status_payload, - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--alpha", type=float, default=0.10) - parser.add_argument("--selected_config_path", default="models/conformal_results_mondrian.pkl") - parser.add_argument("--min_group_size_default", type=int, default=500) - parser.add_argument("--cross_cal_sample_size", type=int, default=5000) - parser.add_argument("--cross_test_sample_size", type=int, default=5000) - parser.add_argument("--calibration_size_fractions", default="0.25,0.50,0.75,1.0") - parser.add_argument( - "--partition_candidates", - default="grade,score_decile_mondrian,grade_x_scoreband_mondrian", - ) - parser.add_argument("--partition_probability_sources", default="raw") - parser.add_argument("--n_score_bins_candidates", default="10") - parser.add_argument("--fallback_modes", default="grade_then_global") - parser.add_argument("--score_scale_families", default="none,bernoulli_sqrt") - parser.add_argument("--min_group_sizes", default=None) - parser.add_argument("--artifact_namespace", default=None) - parser.add_argument("--calibrator_override_path", default=None) - parser.add_argument("--policy_config_path", default=DEFAULT_POLICY_CONFIG) - parser.add_argument( - "--collect_local_diagnostics", - action="store_true", - help=( - "Persist row-level local diagnostics for every benchmark variant. " - "Disabled by default because exhaustive searches can create tens of " - "millions of diagnostic rows." - ), - ) - args = parser.parse_args() - calibration_size_fractions = tuple( - float(x.strip()) for x in str(args.calibration_size_fractions).split(",") if x.strip() - ) - main( - alpha=args.alpha, - selected_config_path=args.selected_config_path, - min_group_size_default=args.min_group_size_default, - cross_cal_sample_size=args.cross_cal_sample_size, - cross_test_sample_size=args.cross_test_sample_size, - calibration_size_fractions=calibration_size_fractions, - partition_candidates=_coerce_csv_tuple(args.partition_candidates, cast=str), - partition_probability_sources=_coerce_csv_tuple( - args.partition_probability_sources, cast=str - ), - n_score_bins_candidates=_coerce_csv_tuple(args.n_score_bins_candidates, cast=int), - fallback_modes=_coerce_csv_tuple(args.fallback_modes, cast=str), - score_scale_families=_coerce_csv_tuple(args.score_scale_families, cast=str), - min_group_sizes=( - _coerce_csv_tuple(args.min_group_sizes, cast=int) - if args.min_group_sizes is not None - else None - ), - artifact_namespace=args.artifact_namespace, - calibrator_override_path=args.calibrator_override_path, - policy_config_path=args.policy_config_path, - collect_local_diagnostics=bool(args.collect_local_diagnostics), - ) diff --git a/scripts/benchmark_pd_set_prediction.py b/scripts/benchmark_pd_set_prediction.py deleted file mode 100644 index c987848..0000000 --- a/scripts/benchmark_pd_set_prediction.py +++ /dev/null @@ -1,744 +0,0 @@ -"""Benchmark binary conformal prediction sets for PD ambiguity/abstention analysis.""" - -from __future__ import annotations - -import argparse -import json -import os -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -from scripts.generate_conformal_intervals import ( - _build_feature_matrix, - _load_calibrator, - _load_model, - _resolve_features, - _subset_calibration_frame, -) -from src.models.conformal import ( - apply_probability_calibrator, - build_mondrian_partition_labels, - create_classification_sets, - create_classification_sets_mondrian, - summarize_prediction_sets, -) -from src.utils.io_utils import read_with_fallback - -TARGET_COL = "default_flag" -GROUP_COL = "grade" - - -@dataclass(frozen=True) -class SetBenchmarkData: - model: Any - calibrator: Any | None - calibrator_name: str - cal_df: pd.DataFrame - test_df: pd.DataFrame - features: list[str] - categorical: list[str] - X_cal: pd.DataFrame - y_cal: pd.Series - X_test: pd.DataFrame - y_test: pd.Series - group_cal: pd.Series - group_test: pd.Series - prob_cal_lookup: dict[str, np.ndarray] - prob_test_lookup: dict[str, np.ndarray] - - -@dataclass(frozen=True) -class SetBenchmarkSettings: - alpha: float - methods: tuple[str, ...] - partitions: tuple[str, ...] - partition_probability_source: str - n_score_bins: int - min_group_size: int - requested_fallback_mode: str - effective_fallback_mode: str - calibration_size_fractions: tuple[float, ...] - - -@dataclass(frozen=True) -class VariantPrediction: - method: str - partition: str - y_pred: np.ndarray - y_sets: np.ndarray - - -def _build_output_paths(namespace: str | None = None) -> dict[str, Path]: - if namespace: - ns = str(namespace).strip().replace("/", "_") - data_dir = Path("data/processed/conformal_gap") / ns - model_dir = Path("models/conformal_gap") / ns - else: - data_dir = Path("data/processed") - model_dir = Path("models") - data_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - return { - "cases": data_dir / "pd_set_prediction_cases.parquet", - "by_slice": data_dir / "pd_set_prediction_by_slice.parquet", - "sensitivity": data_dir / "pd_set_prediction_sensitivity.parquet", - "benchmark": data_dir / "pd_set_prediction_benchmark.parquet", - "status": model_dir / "pd_set_prediction_status.json", - } - - -def _build_paths(namespace: str | None = None) -> dict[str, Path]: - return _build_output_paths(namespace) - - -def _slice_summary(frame: pd.DataFrame, column: str) -> pd.DataFrame: - work = frame.loc[frame[column].notna()].copy() - if work.empty: - return pd.DataFrame() - work[column] = work[column].astype(str) - summary = ( - work.groupby(column, observed=True) - .agg( - n_obs=("y_true", "size"), - set_coverage=("covered", "mean"), - singleton_rate=("singleton", "mean"), - ambiguity_rate=("ambiguous", "mean"), - empty_set_rate=("empty_set", "mean"), - default_rate=("y_true", "mean"), - ) - .reset_index() - .rename(columns={column: "slice_value"}) - ) - ambiguous_rates = ( - work.loc[work["ambiguous"] == 1] - .groupby(column, observed=True)["y_true"] - .mean() - .rename("default_rate_ambiguous") - .reset_index() - .rename(columns={column: "slice_value"}) - ) - summary = summary.merge(ambiguous_rates, on="slice_value", how="left") - summary.insert(0, "slice_name", column) - return summary - - -def _rank_variant(row: pd.Series) -> tuple[float, float, float, float]: - return ( - float(row.get("set_coverage", 0.0)), - float(row.get("singleton_rate", 0.0)), - -float(row.get("ambiguity_rate", 1.0)), - -float(row.get("empty_set_rate", 1.0)), - ) - - -def _coerce_csv_tuple(raw: str | None, *, cast=str) -> tuple[Any, ...]: - if raw is None: - return () - values = [] - for token in str(raw).split(","): - token = token.strip() - if not token: - continue - values.append(cast(token)) - return tuple(values) - - -def _normalize_sidecar_fallback_mode(fallback_mode: str) -> str: - mode = str(fallback_mode or "grade_then_global").strip().lower() - if mode in {"grade_then_global", "global_only"}: - return mode - if mode == "score_only": - logger.warning( - "Sidecar fallback_mode=score_only is not supported for hybrid set partitions; " - "using global_only instead." - ) - return "global_only" - logger.warning("Unknown sidecar fallback_mode={!r}; using grade_then_global.", fallback_mode) - return "grade_then_global" - - -def _unique_csv_values(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: - cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) - return cleaned or fallback - - -def _valid_calibration_fractions(values: tuple[float, ...]) -> tuple[float, ...]: - return tuple(float(x) for x in values if 0 < float(x) <= 1) - - -def _calibrator_name(calibrator: Any | None, calibrator_override_path: str | None) -> str: - if calibrator_override_path: - return Path(str(calibrator_override_path)).stem - if calibrator is not None: - return type(calibrator).__name__ - return "raw" - - -def _load_set_benchmark_data(calibrator_override_path: str | None) -> SetBenchmarkData: - model, _ = _load_model() - calibrator = _load_calibrator(calibrator_override_path) - cal_df = read_with_fallback( - "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" - ) - test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df[TARGET_COL].astype(int).reset_index(drop=True) - X_test = _build_feature_matrix(test_df, features, categorical) - y_test = test_df[TARGET_COL].astype(int).reset_index(drop=True) - group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - y_prob_test_raw = model.predict_proba(X_test)[:, 1] - y_prob_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_raw) - if calibrator is not None - else np.asarray(y_prob_cal_raw, dtype=float) - ) - y_prob_test_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_test_raw) - if calibrator is not None - else np.asarray(y_prob_test_raw, dtype=float) - ) - return SetBenchmarkData( - model=model, - calibrator=calibrator, - calibrator_name=_calibrator_name(calibrator, calibrator_override_path), - cal_df=cal_df, - test_df=test_df, - features=features, - categorical=categorical, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - y_test=y_test, - group_cal=group_cal, - group_test=group_test, - prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, - prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, - ) - - -def _set_benchmark_settings( - *, - alpha: float, - method: str, - methods: tuple[str, ...] | None, - partitions: tuple[str, ...], - partition_probability_source: str, - n_score_bins: int, - min_group_size: int, - fallback_mode: str, - calibration_size_fractions: tuple[float, ...], - prob_cal_lookup: dict[str, np.ndarray], -) -> SetBenchmarkSettings: - source = str(partition_probability_source).strip().lower() or "raw" - if source not in prob_cal_lookup: - raise ValueError(f"Unsupported partition_probability_source: {source}") - return SetBenchmarkSettings( - alpha=float(alpha), - methods=_unique_csv_values(methods or (method,), ("lac",)), - partitions=_unique_csv_values(partitions, ("global",)), - partition_probability_source=source, - n_score_bins=int(n_score_bins), - min_group_size=int(min_group_size), - requested_fallback_mode=str(fallback_mode), - effective_fallback_mode=_normalize_sidecar_fallback_mode(fallback_mode), - calibration_size_fractions=_valid_calibration_fractions(calibration_size_fractions), - ) - - -def _make_cases( - *, - y_true: pd.Series, - y_pred: np.ndarray, - y_sets: np.ndarray, - test_df: pd.DataFrame, - method: str, - partition: str, - partition_probability_source: str, - calibrator_name: str, -) -> pd.DataFrame: - cases = pd.DataFrame( - { - "y_true": y_true.to_numpy(dtype=int), - "y_pred_label": np.asarray(y_pred, dtype=int), - "set_contains_0": y_sets[:, 0].astype(int), - "set_contains_1": y_sets[:, 1].astype(int), - "set_size": y_sets.sum(axis=1).astype(int), - "ambiguous": (y_sets.sum(axis=1) > 1).astype(int), - "singleton": (y_sets.sum(axis=1) == 1).astype(int), - "empty_set": (y_sets.sum(axis=1) == 0).astype(int), - "method": str(method), - "partition": str(partition), - "partition_probability_source": str(partition_probability_source), - "calibrator": str(calibrator_name), - } - ) - cases["covered"] = y_sets[np.arange(len(y_true)), y_true.to_numpy(dtype=int)].astype(int) - for col in ("grade", "term", "home_ownership", "issue_d"): - if col in test_df.columns: - cases[col] = test_df[col].reset_index(drop=True) - if "id" in test_df.columns: - cases["id"] = test_df["id"].astype(str).reset_index(drop=True) - elif "loan_id" in test_df.columns: - cases["loan_id"] = test_df["loan_id"].astype(str).reset_index(drop=True) - if "issue_d" in cases.columns: - cases["issue_quarter"] = ( - pd.to_datetime(cases["issue_d"], errors="coerce").dt.to_period("Q").astype(str) - ) - return cases - - -def _is_global_partition(partition: str) -> bool: - return str(partition).strip().lower() == "global" - - -def _predict_variant( - *, - data: SetBenchmarkData, - settings: SetBenchmarkSettings, - method_name: str, - partition_name: str, - X_cal: pd.DataFrame, - y_cal: pd.Series, - group_cal: pd.Series, - y_prob_cal: np.ndarray, -) -> VariantPrediction: - if _is_global_partition(partition_name): - y_pred, y_sets = create_classification_sets( - classifier=data.model, - X_cal=X_cal, - y_cal=y_cal, - X_test=data.X_test, - alpha=settings.alpha, - method=method_name, - calibrator=data.calibrator, - ) - return VariantPrediction( - method=str(method_name), - partition=str(partition_name), - y_pred=y_pred, - y_sets=y_sets, - ) - - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal, - y_prob_eval=data.prob_test_lookup[settings.partition_probability_source], - partition=partition_name, - base_groups_cal=group_cal, - base_groups_eval=data.group_test, - n_score_bins=settings.n_score_bins, - min_group_size=settings.min_group_size, - fallback_mode=settings.effective_fallback_mode, - ) - y_pred, y_sets, _ = create_classification_sets_mondrian( - classifier=data.model, - X_cal=X_cal, - y_cal=y_cal, - X_test=data.X_test, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=settings.alpha, - method=method_name, - min_group_size=settings.min_group_size, - calibrator=data.calibrator, - ) - return VariantPrediction( - method=str(method_name), - partition=str(partition_meta.get("partition", partition_name)), - y_pred=y_pred, - y_sets=y_sets, - ) - - -def _benchmark_row( - *, - data: SetBenchmarkData, - settings: SetBenchmarkSettings, - prediction: VariantPrediction, -) -> dict[str, Any]: - summary = summarize_prediction_sets( - data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets - ) - return { - "method": prediction.method, - "partition": prediction.partition, - "partition_probability_source": settings.partition_probability_source, - "calibrator": data.calibrator_name, - "alpha": settings.alpha, - **{k: float(v) for k, v in summary.items()}, - } - - -def _subsample_calibration_data( - *, - data: SetBenchmarkData, - calibration_fraction: float, -) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, np.ndarray, np.ndarray]: - cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=calibration_fraction) - X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) - y_cal_sub = cal_df_sub[TARGET_COL].astype(int).reset_index(drop=True) - group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] - y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) - if data.calibrator is not None - else np.asarray(y_prob_cal_sub_raw, dtype=float) - ) - return ( - cal_df_sub, - X_cal_sub, - y_cal_sub, - group_cal_sub, - y_prob_cal_sub_raw, - y_prob_cal_sub_calibrated, - ) - - -def _sensitivity_row( - *, - data: SetBenchmarkData, - settings: SetBenchmarkSettings, - method_name: str, - partition_name: str, - calibration_fraction: float, -) -> dict[str, Any]: - _cal_df_sub, X_cal_sub, y_cal_sub, group_cal_sub, y_prob_raw, y_prob_calibrated = ( - _subsample_calibration_data(data=data, calibration_fraction=calibration_fraction) - ) - y_prob_cal = y_prob_raw if settings.partition_probability_source == "raw" else y_prob_calibrated - prediction = _predict_variant( - data=data, - settings=settings, - method_name=method_name, - partition_name=partition_name, - X_cal=X_cal_sub, - y_cal=y_cal_sub, - group_cal=group_cal_sub, - y_prob_cal=y_prob_cal, - ) - summary_sub = summarize_prediction_sets( - data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets - ) - return { - "method": str(method_name), - "partition": str(prediction.partition), - "partition_probability_source": settings.partition_probability_source, - "calibrator": data.calibrator_name, - "calibration_fraction": float(calibration_fraction), - "n_calibration_rows": int(len(X_cal_sub)), - "set_coverage": float(summary_sub["set_coverage"]), - "singleton_rate": float(summary_sub["singleton_rate"]), - "ambiguity_rate": float(summary_sub["ambiguity_rate"]), - "empty_set_rate": float(summary_sub["empty_set_rate"]), - "default_rate_ambiguous": float(summary_sub["default_rate_ambiguous"]), - } - - -def _sensitivity_rows_for_variant( - *, - data: SetBenchmarkData, - settings: SetBenchmarkSettings, - method_name: str, - partition_name: str, -) -> list[dict[str, Any]]: - return [ - _sensitivity_row( - data=data, - settings=settings, - method_name=method_name, - partition_name=partition_name, - calibration_fraction=float(frac), - ) - for frac in settings.calibration_size_fractions - ] - - -def _run_benchmark_matrix( - *, - data: SetBenchmarkData, - settings: SetBenchmarkSettings, -) -> tuple[ - pd.DataFrame, - dict[tuple[str, str], pd.DataFrame], - dict[tuple[str, str], list[dict[str, Any]]], -]: - benchmark_rows: list[dict[str, Any]] = [] - cases_by_variant: dict[tuple[str, str], pd.DataFrame] = {} - sensitivity_by_variant: dict[tuple[str, str], list[dict[str, Any]]] = {} - - for method_name in settings.methods: - for partition_name in settings.partitions: - prediction = _predict_variant( - data=data, - settings=settings, - method_name=method_name, - partition_name=partition_name, - X_cal=data.X_cal, - y_cal=data.y_cal, - group_cal=data.group_cal, - y_prob_cal=data.prob_cal_lookup[settings.partition_probability_source], - ) - benchmark_rows.append( - _benchmark_row(data=data, settings=settings, prediction=prediction) - ) - key = (prediction.method, prediction.partition) - cases_by_variant[key] = _make_cases( - y_true=data.y_test, - y_pred=prediction.y_pred, - y_sets=prediction.y_sets, - test_df=data.test_df, - method=prediction.method, - partition=prediction.partition, - partition_probability_source=settings.partition_probability_source, - calibrator_name=data.calibrator_name, - ) - sensitivity_by_variant[key] = _sensitivity_rows_for_variant( - data=data, - settings=settings, - method_name=method_name, - partition_name=prediction.partition, - ) - - benchmark_df = pd.DataFrame(benchmark_rows) - if benchmark_df.empty: - raise RuntimeError("No set-prediction variants were benchmarked.") - return ( - benchmark_df.sort_values( - by=["set_coverage", "singleton_rate", "ambiguity_rate", "empty_set_rate"], - ascending=[False, False, True, True], - ).reset_index(drop=True), - cases_by_variant, - sensitivity_by_variant, - ) - - -def _slice_reports(cases: pd.DataFrame) -> pd.DataFrame: - slice_reports = [] - for col in ("grade", "term", "issue_quarter"): - if col in cases.columns: - report = _slice_summary(cases, col) - if not report.empty: - slice_reports.append(report) - return pd.concat(slice_reports, ignore_index=True) if slice_reports else pd.DataFrame() - - -def _grade_slices(by_slice: pd.DataFrame) -> list[dict[str, Any]]: - if by_slice.empty or "slice_name" not in by_slice.columns: - return [] - return by_slice.loc[by_slice["slice_name"] == "grade"].to_dict(orient="records") - - -def _slice_records(by_slice: pd.DataFrame, slice_name: str) -> list[dict[str, Any]]: - if by_slice.empty or "slice_name" not in by_slice.columns: - return [] - return by_slice.loc[by_slice["slice_name"] == slice_name].to_dict(orient="records") - - -def _promotion_gate(summary: dict[str, Any], grade_slices: list[dict[str, Any]]) -> dict[str, Any]: - gate_coverage = float(summary.get("set_coverage", 0)) - gate_grade_a_singleton = 0.0 - gate_grades_above_40 = 0 - for gs in grade_slices: - singleton_rate = float(gs.get("singleton_rate", 0)) - if str(gs.get("slice_value", "")) == "A": - gate_grade_a_singleton = singleton_rate - if singleton_rate > 0.40: - gate_grades_above_40 += 1 - gate_pass = bool( - gate_coverage >= 0.85 and gate_grade_a_singleton >= 0.80 and gate_grades_above_40 >= 3 - ) - return { - "coverage": gate_coverage, - "min_coverage": 0.85, - "grade_a_singleton_rate": gate_grade_a_singleton, - "min_grade_a_singleton": 0.80, - "grades_with_singleton_above_40pct": gate_grades_above_40, - "min_grades_above_40pct": 3, - "pass": gate_pass, - } - - -def _selected_summary(cases: pd.DataFrame) -> dict[str, float]: - return { - k: float(v) - for k, v in summarize_prediction_sets( - cases["y_true"].to_numpy(dtype=int), - cases["y_pred_label"].to_numpy(dtype=int), - cases[["set_contains_0", "set_contains_1"]].to_numpy(dtype=int), - ).items() - } - - -def _status_payload( - *, - settings: SetBenchmarkSettings, - selected: pd.Series, - summary: dict[str, float], - gate: dict[str, Any], - by_slice: pd.DataFrame, - benchmark_df: pd.DataFrame, - outputs: dict[str, Path], - artifact_namespace: str | None, -) -> dict[str, Any]: - gate_pass = bool(gate["pass"]) - promotion_status = "promoted_guardrail" if gate_pass else "research_sidecar" - return { - "schema_version": "2026-04-03.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": os.environ.get("PIPELINE_RUN_TAG", "untracked"), - "artifact_namespace": artifact_namespace or "", - "status": promotion_status, - "promoted": gate_pass, - "method": str(selected["method"]), - "selected_method": str(selected["method"]), - "selected_partition": str(selected["partition"]), - "selected_partition_probability_source": str(selected["partition_probability_source"]), - "selected_calibrator": str(selected["calibrator"]), - "requested_fallback_mode": settings.requested_fallback_mode, - "effective_fallback_mode": settings.effective_fallback_mode, - "alpha": settings.alpha, - "confidence_level": float(1.0 - settings.alpha), - "summary": summary, - "promotion_gate": gate, - "artifact_path": str(outputs["cases"]), - "by_slice_path": str(outputs["by_slice"]), - "calibration_size_sensitivity_path": str(outputs["sensitivity"]), - "benchmark_matrix_path": str(outputs["benchmark"]), - "slice_metrics": { - "grade": _grade_slices(by_slice), - "term": _slice_records(by_slice, "term"), - "issue_quarter": _slice_records(by_slice, "issue_quarter"), - }, - "benchmark_matrix": benchmark_df.to_dict(orient="records"), - "decision_use_case": { - "probability_first": True, - "set_first": False, - "recommended_guardrail": "selective_ambiguity_defer", - }, - "promotion_rationale": ( - f"Binary conformal sets selected via {selected['method']} + {selected['partition']} " - f"with set coverage {gate['coverage']:.1%} and ambiguity {summary['ambiguity_rate']:.1%}." - ), - "promotion_note": ( - "Binary set prediction remains a sidecar triage/abstention signal; it does not replace " - "the interval-first conformal stack." - ), - } - - -def _write_outputs( - *, - outputs: dict[str, Path], - cases: pd.DataFrame, - by_slice: pd.DataFrame, - sensitivity_df: pd.DataFrame, - benchmark_df: pd.DataFrame, - status: dict[str, Any], -) -> None: - cases.to_parquet(outputs["cases"], index=False) - if not by_slice.empty: - by_slice.to_parquet(outputs["by_slice"], index=False) - sensitivity_df.to_parquet(outputs["sensitivity"], index=False) - benchmark_df.to_parquet(outputs["benchmark"], index=False) - outputs["status"].write_text( - json.dumps(status, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" - ) - logger.info("Saved PD set prediction cases: {}", outputs["cases"]) - logger.info("Saved PD set prediction status: {}", outputs["status"]) - - -def main( - alpha: float = 0.10, - method: str = "lac", - methods: tuple[str, ...] | None = None, - partitions: tuple[str, ...] = ("global",), - partition_probability_source: str = "raw", - n_score_bins: int = 10, - min_group_size: int = 500, - fallback_mode: str = "grade_then_global", - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, -) -> None: - data = _load_set_benchmark_data(calibrator_override_path) - settings = _set_benchmark_settings( - alpha=alpha, - method=method, - methods=methods, - partitions=partitions, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=fallback_mode, - calibration_size_fractions=calibration_size_fractions, - prob_cal_lookup=data.prob_cal_lookup, - ) - benchmark_df, cases_by_variant, sensitivity_by_variant = _run_benchmark_matrix( - data=data, - settings=settings, - ) - selected = benchmark_df.iloc[0] - selected_key = (str(selected["method"]), str(selected["partition"])) - cases = cases_by_variant[selected_key] - by_slice = _slice_reports(cases) - sensitivity_df = pd.DataFrame(sensitivity_by_variant[selected_key]) - summary = _selected_summary(cases) - gate = _promotion_gate(summary, _grade_slices(by_slice)) - outputs = _build_output_paths(artifact_namespace) - status = _status_payload( - settings=settings, - selected=selected, - summary=summary, - gate=gate, - by_slice=by_slice, - benchmark_df=benchmark_df, - outputs=outputs, - artifact_namespace=artifact_namespace, - ) - _write_outputs( - outputs=outputs, - cases=cases, - by_slice=by_slice, - sensitivity_df=sensitivity_df, - benchmark_df=benchmark_df, - status=status, - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--alpha", type=float, default=0.10) - parser.add_argument("--method", default="lac") - parser.add_argument("--methods", default=None) - parser.add_argument("--partitions", default="global") - parser.add_argument("--partition_probability_source", default="raw") - parser.add_argument("--n_score_bins", type=int, default=10) - parser.add_argument("--min_group_size", type=int, default=500) - parser.add_argument("--fallback_mode", default="grade_then_global") - parser.add_argument("--calibration-size-fractions", default="0.25,0.50,0.75,1.0") - parser.add_argument("--artifact_namespace", default=None) - parser.add_argument("--calibrator_override_path", default=None) - args = parser.parse_args() - main( - alpha=args.alpha, - method=args.method, - methods=_coerce_csv_tuple(args.methods, cast=str) if args.methods else None, - partitions=_coerce_csv_tuple(args.partitions, cast=str), - partition_probability_source=args.partition_probability_source, - n_score_bins=args.n_score_bins, - min_group_size=args.min_group_size, - fallback_mode=args.fallback_mode, - calibration_size_fractions=tuple( - float(x.strip()) for x in str(args.calibration_size_fractions).split(",") if x.strip() - ), - artifact_namespace=args.artifact_namespace, - calibrator_override_path=args.calibrator_override_path, - ) diff --git a/scripts/build_bound_tightening_audit.py b/scripts/build_bound_tightening_audit.py deleted file mode 100644 index 3429809..0000000 --- a/scripts/build_bound_tightening_audit.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Build an experimental CRPTO bound-tightening audit from frozen funded weights. - -The active manuscript keeps Markov as the main distribution-free bound. This -script computes the sharper concentration bounds that become available only -under extra assumptions, so the IJDS appendix can discuss tightness without -reopening the frozen champion or re-running protected DVC stages. -""" - -from __future__ import annotations - -import math -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -from src.utils.script_helpers import write_table - -ROOT = Path(__file__).resolve().parents[1] -TABLE_DIR = ROOT / "reports" / "crpto" / "tables" -OUTPUT_DIR = ROOT / "docs" / "research" / "bound_tightening_audit" -REPORT_PATH = ROOT / "docs" / "research" / "crpto_bound_tightening_experiment_2026-06-11.md" -FUNDED_LOANS_PATH = TABLE_DIR / "crpto_tableA7_funded_set_loans.csv" - -TABLE_A21C_NAME = "crpto_tableA21c_bound_comparison_experimental" -TABLE_A21D_NAME = "crpto_tableA21d_bound_assumption_audit_experimental" - -ALPHAS = (0.01, 0.03, 0.05, 0.10) -ROBUST_REGION_POLICY_COUNT = 45 - - -def _normalized_weights(funded: pd.DataFrame) -> np.ndarray: - weights = pd.to_numeric(funded["portfolio_weight"], errors="coerce").fillna(0.0) - values = weights.to_numpy(dtype=float) - if np.any(values < 0): - raise ValueError("Funded-set weights must be non-negative.") - total = float(values.sum()) - if total <= 0.0: - raise ValueError("Funded-set weights sum to zero.") - return values / total - - -def _empirical_weighted_miscoverage(funded: pd.DataFrame, weights: np.ndarray) -> float: - misses = funded["miscovered_alpha01"].astype(bool).astype(float).to_numpy(dtype=float) - return float(np.sum(weights * misses)) - - -def _bennett_threshold( - *, alpha: float, delta: float, variance_bound: float, max_weight: float -) -> float: - if variance_bound <= 0.0 or max_weight <= 0.0: - return alpha - - target = math.log(1.0 / delta) - - def h(value: float) -> float: - return (1.0 + value) * math.log1p(value) - value - - low = 0.0 - high = max(1.0 - alpha, max_weight) - for _ in range(100): - mid = (low + high) / 2.0 - lhs = (variance_bound / (max_weight**2)) * h(max_weight * mid / variance_bound) - if lhs >= target: - high = mid - else: - low = mid - return alpha + high - - -def _threshold_rows(funded: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, float]]: - weights = _normalized_weights(funded) - empirical_v = _empirical_weighted_miscoverage(funded, weights) - sum_w2 = float(np.sum(weights**2)) - max_weight = float(np.max(weights)) - n_eff = float(1.0 / sum_w2) - n_funded = float(len(weights)) - - rows: list[dict[str, Any]] = [] - for alpha in ALPHAS: - delta = math.sqrt(alpha) - log_term = math.log(1.0 / delta) - markov_threshold = alpha / delta - hoeffding_threshold = alpha + math.sqrt(sum_w2 * log_term / 2.0) - - strong_variance = alpha * sum_w2 - weak_variance = alpha * max_weight - for variance_mode, variance_bound in [ - ("strong_individual_validity", strong_variance), - ("weak_weighted_validity", weak_variance), - ]: - cantelli_threshold = alpha + math.sqrt(variance_bound * (1.0 - delta) / delta) - bernstein_threshold = alpha + ( - max_weight * log_term / 3.0 - + math.sqrt((max_weight * log_term / 3.0) ** 2 + 2.0 * variance_bound * log_term) - ) - bennett_threshold = _bennett_threshold( - alpha=alpha, - delta=delta, - variance_bound=variance_bound, - max_weight=max_weight, - ) - - for bound_name, threshold, paper_role in [ - ( - "cantelli_one_sided", - cantelli_threshold, - "conditional variance diagnostic; sharper one-sided Chebyshev", - ), - ( - "bernstein", - bernstein_threshold, - "conditional independence/variance tightening; appendix-only", - ), - ( - "freedman_martingale", - bernstein_threshold, - "martingale analogue of Bernstein; needs a sealed sequential protocol", - ), - ( - "bennett", - bennett_threshold, - "conditional independence/variance tightening; appendix-only", - ), - ]: - rows.append( - { - "alpha": alpha, - "delta": delta, - "bound": bound_name, - "variance_mode": variance_mode, - "threshold_t": threshold, - "empirical_V_alpha01": empirical_v, - "margin_vs_empirical_V": threshold - empirical_v, - "tighter_than_markov": threshold < markov_threshold, - "empirical_V_below_threshold": empirical_v <= threshold, - "paper_role": paper_role, - } - ) - - # Agnostic mode: only the theorem's own assumption E[V] <= alpha, no - # independence or correlation structure. The sharp variance bound is - # then Var(V) <= alpha(1 - alpha) (attained by V ~ Bernoulli(alpha)), - # under which one-sided Cantelli is WORSE than Markov — the cleanest - # quantitative defense of keeping Markov as the body claim. - agnostic_variance = alpha * (1.0 - alpha) - cantelli_agnostic = alpha + math.sqrt(agnostic_variance * (1.0 - delta) / delta) - - base_rows = [ - ( - "markov", - "none", - markov_threshold, - "main distribution-free claim; only first moment needed", - ), - ( - "cantelli_one_sided", - "agnostic_theorem_assumption_only", - cantelli_agnostic, - "sharp variance under E[V]<=alpha alone; worse than Markov, so no " - "second-moment tightening exists without extra assumptions", - ), - ( - "hoeffding", - "loan_independence", - hoeffding_threshold, - "conditional bounded-difference diagnostic", - ), - ] - for bound_name, variance_mode, threshold, paper_role in base_rows: - rows.append( - { - "alpha": alpha, - "delta": delta, - "bound": bound_name, - "variance_mode": variance_mode, - "threshold_t": threshold, - "empirical_V_alpha01": empirical_v, - "margin_vs_empirical_V": threshold - empirical_v, - "tighter_than_markov": threshold < markov_threshold, - "empirical_V_below_threshold": empirical_v <= threshold, - "paper_role": paper_role, - } - ) - - table = pd.DataFrame(rows).sort_values(["alpha", "threshold_t", "bound"]).reset_index(drop=True) - stats = { - "n_funded": n_funded, - "n_eff": n_eff, - "sum_w2": sum_w2, - "max_weight": max_weight, - "empirical_v": empirical_v, - } - return table, stats - - -def _cluster_assumption_rows(funded: pd.DataFrame, stats: dict[str, float]) -> pd.DataFrame: - rows: list[dict[str, Any]] = [ - { - "assumption": "nonnegative_normalized_weights", - "status": "pass", - "diagnostic_value": 1.0, - "interpretation": "Funded-set weights are non-negative and normalized.", - }, - { - "assumption": "bounded_miss_indicators", - "status": "pass", - "diagnostic_value": 1.0, - "interpretation": "miscovered_alpha01 is binary, so V is a bounded weighted sum.", - }, - { - "assumption": "effective_sample_size", - "status": "concentrated", - "diagnostic_value": stats["n_eff"], - "interpretation": ( - "n_eff is far below the funded loan count, so iid-style concentration " - "does not get to use the headline OOT sample size." - ), - }, - { - "assumption": "loan_independence", - "status": "not_verified", - "diagnostic_value": float("nan"), - "interpretation": ( - "Defaults and conformal misses can share calibration history and macro " - "period shocks; loan-level concentration bounds are appendix-only." - ), - }, - { - "assumption": "post_selection_uniformity", - "status": "not_supported_by_markov", - "diagnostic_value": ROBUST_REGION_POLICY_COUNT, - "interpretation": ( - "A naive union Markov statement over the 45 final policies is vacuous " - "at alpha01; the exact robust-region audit remains empirical evidence." - ), - }, - { - "assumption": "sequential_martingale_protocol", - "status": "not_available", - "diagnostic_value": float("nan"), - "interpretation": ( - "Azuma/Freedman need a prospective filtration or online validation " - "design; the current A24 replay is diagnostic, not a live guarantee." - ), - }, - { - "assumption": "chebyshev_two_sided", - "status": "drop_from_table", - "diagnostic_value": float("nan"), - "interpretation": ( - "Two-sided Chebyshev is dominated by Cantelli for the one-sided " - "exceedance probability used in A21." - ), - }, - { - "assumption": "azuma_hoeffding_martingale", - "status": "drop_from_table", - "diagnostic_value": float("nan"), - "interpretation": ( - "Azuma gives the same numerical threshold as Hoeffding here while " - "adding a sequential validation protocol assumption." - ), - }, - { - "assumption": "chernoff_mgf", - "status": "drop_from_table", - "diagnostic_value": float("nan"), - "interpretation": ( - "Chernoff is sharp, but it requires independent misses with each " - "individual miss probability bounded by alpha." - ), - }, - { - "assumption": "union_markov_45_policy_region", - "status": "drop_from_table", - "diagnostic_value": ROBUST_REGION_POLICY_COUNT, - "interpretation": ( - "A naive union Markov statement over the 45 final policies is " - "vacuous at the paper alphas." - ), - }, - { - "assumption": "empirical_bernstein_or_bootstrap", - "status": "diagnostic_only", - "diagnostic_value": float("nan"), - "interpretation": ( - "Empirical-Bernstein or bootstrap intervals would use observed OOT " - "labels; useful for sensitivity, not for the distribution-free theorem." - ), - }, - ] - - cluster_specs = { - "period": ["period"], - "grade": ["original_grade"], - "period_grade": ["period", "original_grade"], - } - for cluster_type, columns in cluster_specs.items(): - cluster_weights = funded.groupby(columns, dropna=False)["portfolio_weight"].sum() - cluster_weights = cluster_weights / float(cluster_weights.sum()) - sum_cluster_w2 = float(np.sum(np.square(cluster_weights.to_numpy(dtype=float)))) - max_cluster_weight = float(cluster_weights.max()) - cluster_threshold = 0.01 + math.sqrt(0.5 * sum_cluster_w2 * math.log(10.0)) - rows.append( - { - "assumption": f"cluster_independence_{cluster_type}", - "status": "conditional_loose", - "diagnostic_value": cluster_threshold, - "interpretation": ( - f"Cluster Hoeffding threshold at delta=0.10 is {cluster_threshold:.4f}; " - f"max cluster exposure is {max_cluster_weight:.4f}, so this is not " - "tighter than Markov's 0.1000 threshold." - ), - } - ) - return pd.DataFrame(rows) - - -def _write_report( - bound_table: pd.DataFrame, assumption_table: pd.DataFrame, stats: dict[str, float] -) -> None: - alpha01 = bound_table[bound_table["alpha"].eq(0.01)].copy() - selected = alpha01[ - alpha01["bound"].isin( - [ - "markov", - "hoeffding", - "bernstein", - "freedman_martingale", - "bennett", - "cantelli_one_sided", - ] - ) - ].sort_values("threshold_t") - lines: list[str] = [ - "# CRPTO Bound Tightening Experiment - 2026-06-11", - "", - "Merged into `main` (2026-06-11) and cited by Online Supplement Appendix A. " - "This audit reads frozen funded-set weights only; it does not re-run DVC " - "stages, does not search policies, and does not promote a new Lending Club " - "champion. The A21c/A21d tables live under " - "`docs/research/bound_tightening_audit/`, deliberately outside the " - "`EXTRACTION_MANIFEST` sweep area: git versioning plus " - "`tests/test_scripts/test_build_bound_tightening_audit.py` guarantee their " - "integrity by re-deriving them deterministically from the frozen A7 weights.", - "", - "## Fixed Funded-Set Diagnostics", - "", - f"- funded loans: `{int(stats['n_funded'])}`", - f"- effective sample size: `{stats['n_eff']:.1f}`", - f"- sum of squared weights: `{stats['sum_w2']:.6f}`", - f"- max loan weight: `{stats['max_weight']:.4f}`", - f"- observed `V(alpha=0.01)`: `{stats['empirical_v']:.6f}`", - "", - "## Alpha 0.01 Bound Menu", - "", - "| Bound | Mode | threshold t | margin vs V | Role |", - "|---|---|---:|---:|---|", - ] - for row in selected.itertuples(index=False): - lines.append( - f"| `{row.bound}` | `{row.variance_mode}` | `{row.threshold_t:.6f}` | " - f"`{row.margin_vs_empirical_V:.6f}` | {row.paper_role} |" - ) - - lines.extend( - ( - "", - "## Recommendation", - "", - "- Keep Markov as the body theorem: it is the only first-moment, " - "distribution-free statement compatible with the current post-selection caveat.", - "- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: " - "cluster exposure is too concentrated.", - "- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett " - "and Freedman show how much tightness is available if a reviewer accepts stronger " - "independence, variance, or martingale assumptions.", - "- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. " - "They are respectively dominated, duplicative, too strong for the current " - "individual-alpha evidence, or vacuous after policy-region correction.", - "", - "## Assumption Audit", - "", - str(assumption_table.to_markdown(index=False)), - "", - ) - ) - REPORT_PATH.parent.mkdir(parents=True, exist_ok=True) - REPORT_PATH.write_text("\n".join(lines), encoding="utf-8", newline="") - - -def build_bound_tightening_audit() -> dict[str, Any]: - funded = pd.read_csv(FUNDED_LOANS_PATH) - bound_table, stats = _threshold_rows(funded) - assumption_table = _cluster_assumption_rows(funded, stats) - - artifacts = [] - artifacts += write_table(TABLE_A21C_NAME, bound_table, table_dir=OUTPUT_DIR, root=ROOT) - artifacts += write_table(TABLE_A21D_NAME, assumption_table, table_dir=OUTPUT_DIR, root=ROOT) - _write_report(bound_table, assumption_table, stats) - artifacts.append(REPORT_PATH) - - return { - "artifacts": [path.relative_to(ROOT).as_posix() for path in artifacts], - "n_funded": int(stats["n_funded"]), - "n_eff": stats["n_eff"], - "max_weight": stats["max_weight"], - "empirical_v": stats["empirical_v"], - } - - -def main() -> int: - status = build_bound_tightening_audit() - logger.info( - "Built bound tightening audit: n_funded={} n_eff={:.1f} V={:.6f}", - status["n_funded"], - status["n_eff"], - status["empirical_v"], - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/build_champion_search_bundle.py b/scripts/build_champion_search_bundle.py deleted file mode 100644 index 6c37987..0000000 --- a/scripts/build_champion_search_bundle.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Assemble a promotion-ready search bundle from current project artifacts.""" - -from __future__ import annotations - -import json -import os -import pickle -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from src.utils.baseline_registry import resolve_official_baseline_run_tag -from src.utils.threshold_semantics import load_threshold_semantics - -ROOT = Path(__file__).resolve().parents[1] -DATA = ROOT / "data" / "processed" -MODELS = ROOT / "models" -BASELINES = ROOT / "configs" / "baselines" -SCHEMA_VERSION = "2026-03-13.2" - - -def _meaningful_run_tag(*values: object) -> str: - for value in values: - candidate = str(value or "").strip() - if candidate and candidate.lower() not in {"untracked", "unknown"}: - return candidate - return "untracked" - - -def _load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - - -def _load_pickle(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - with open(path, "rb") as f: - payload = pickle.load(f) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def _resolve_upstream_baseline() -> str | None: - return resolve_official_baseline_run_tag() - - -def _artifact_run_tags(payloads: dict[str, dict[str, Any]]) -> dict[str, str | None]: - return { - name: ( - str(payload.get("run_tag")).strip() - if str(payload.get("run_tag") or "").strip() - else None - ) - for name, payload in payloads.items() - } - - -def main() -> None: - training_record = _load_pickle(MODELS / "pd_training_record.pkl") - model_comparison = _load_json(DATA / "model_comparison.json") - conformal_status = _load_json(MODELS / "conformal_policy_status.json") - conformal_method_registry = _load_json(MODELS / "conformal_method_registry.json") - conformal_variant_status = _load_json(MODELS / "conformal_variant_selection_status.json") - fairness_status = _load_json(MODELS / "fairness_audit_status.json") - champion_policy = _load_json(MODELS / "champion_portfolio_policy.json") - governance_status = _load_json(MODELS / "governance_status.json") - pd_set_prediction = _load_json(MODELS / "pd_set_prediction_status.json") - pd_rare_event = _load_json(MODELS / "pd_rare_event_calibration_status.json") - pd_calibration_diagnostics = _load_json(MODELS / "pd_calibration_diagnostics.json") - threshold_semantics = load_threshold_semantics() - upstream_baseline = _resolve_upstream_baseline() - artifact_run_tags = _artifact_run_tags( - { - "governance": governance_status, - "portfolio": champion_policy, - "threshold_semantics": threshold_semantics, - "conformal_policy_status": conformal_status, - } - ) - - run_tag = _meaningful_run_tag( - os.environ.get("PIPELINE_RUN_TAG", ""), - governance_status.get("run_tag", ""), - champion_policy.get("run_tag", ""), - threshold_semantics.get("run_tag", ""), - upstream_baseline, - ) - payload = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(UTC).isoformat(), - "run_tag": run_tag, - "pipeline_family": str(os.environ.get("PIPELINE_FAMILY", "search_pd") or "search_pd"), - "pipeline_profile": str( - os.environ.get("PIPELINE_PROFILE", "search_pd_default") or "search_pd_default" - ), - "artifact_scope": "search", - "promotion_state": str( - os.environ.get("PIPELINE_PROMOTION_STATE", "research_open") or "research_open" - ), - "writes_canonical_artifacts": str( - os.environ.get("WRITES_CANONICAL_ARTIFACTS", "false") - ).lower() - in {"1", "true", "yes", "on"}, - "upstream_canonical_run_tag": upstream_baseline, - "artifact_run_tags": artifact_run_tags, - "mixed_run_tags_present": len( - { - value - for value in artifact_run_tags.values() - if value and value not in {"untracked", "unknown"} - } - ) - > 1, - "threshold_semantics": threshold_semantics, - "pd": { - "best_model": model_comparison.get("best_model"), - "best_calibration": model_comparison.get("best_calibration"), - "training_regime": training_record.get("training_regime", {}), - "stable_core": training_record.get("stable_core", {}), - "decision_threshold": training_record.get("decision_threshold", {}), - "decision_threshold_semantics": { - "pd_internal_selected_threshold": threshold_semantics.get( - "pd_internal_selected_threshold" - ), - "fairness_primary_threshold": threshold_semantics.get("fairness_primary_threshold"), - "decision_policy_global_threshold": threshold_semantics.get( - "decision_policy_global_threshold" - ), - }, - "set_prediction": pd_set_prediction, - "rare_event_calibration": pd_rare_event, - "calibration_diagnostics": pd_calibration_diagnostics, - }, - "conformal": { - "policy_status": conformal_status, - "method_registry": conformal_method_registry, - "variant_selection": conformal_variant_status, - }, - "fairness": fairness_status, - "portfolio": champion_policy, - "governance": governance_status, - } - out_path = MODELS / "champion_search_bundle.json" - out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - print(f"[champion_search_bundle] saved {out_path}") - - -if __name__ == "__main__": - main() diff --git a/scripts/build_crpto_journal_package.py b/scripts/build_crpto_journal_package.py index 02e5e86..b04456f 100644 --- a/scripts/build_crpto_journal_package.py +++ b/scripts/build_crpto_journal_package.py @@ -810,7 +810,10 @@ def _plot_robust_region_heatmap(shortlist: pd.DataFrame, promotion: dict[str, An for j in range(pivot.shape[1]): value = pivot.iloc[i, j] if pd.notna(value): - rgba = cmap(im.norm(float(value))) + rgba = np.asarray( + cmap(im.norm(np.asarray([float(value)], dtype=float))), + dtype=float, + ).reshape(-1, 4)[0] luminance = (0.2126 * rgba[0]) + (0.7152 * rgba[1]) + (0.0722 * rgba[2]) text_color = "white" if luminance < 0.45 else "#111111" ax.text( diff --git a/scripts/build_distribution_robustness_diagnostics.py b/scripts/build_distribution_robustness_diagnostics.py deleted file mode 100644 index 15917d3..0000000 --- a/scripts/build_distribution_robustness_diagnostics.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Build the P2 distribution-robustness diagnostics (Tables A23 and A24). - -These close the two remaining P2 roadmap items that do NOT require reopening the -frozen champion search, using only the frozen Mondrian conformal intervals: - -* A23 -- Multi-distribution coverage robustness (MDCP / group-weighted spirit, - [@yang2026multidistribution; @bhattacharyya2026groupweighted]). Reports the - worst-case one-/two-sided coverage across grade groups and grade x vintage - cells, i.e. whether the 90% guarantee survives when an unknown test-time group - or source distribution dominates. - -* A24 -- Online conformal stability over the OOT vintage sequence (ACI, - [@gibbs2021aci; @angelopoulos2025gradient; @liu2026portfolio]). Reports - per-vintage coverage, the cumulative ("streaming") coverage, and the adaptive - conformal inference (ACI) target trajectory ``alpha_t`` that an online - controller would follow. The OOT dataset is static, so this is a diagnostic of - how hard an online controller would have to work, not a streaming validation. - -The script reads only ``data/processed/conformal_intervals_mondrian.parquet`` -(a frozen artifact) and writes journal-only tables. It does not touch the -champion or any optimization artifact. - -Usage:: - - uv run python scripts/build_distribution_robustness_diagnostics.py -""" - -from __future__ import annotations - -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -from src.utils.script_helpers import write_json, write_table - -ROOT = Path(__file__).resolve().parents[1] -TABLE_DIR = ROOT / "reports" / "crpto" / "tables" -MODEL_DIR = ROOT / "models" -INTERVALS_PATH = ROOT / "data" / "processed" / "conformal_intervals_mondrian.parquet" -STATUS_PATH = MODEL_DIR / "crpto_distribution_robustness_status.json" - -TABLE_A23_NAME = "crpto_tableA23_multidistribution_robustness" -TABLE_A24_NAME = "crpto_tableA24_online_conformal_stability" - -TARGET_COVERAGE = 0.90 -TARGET_ALPHA = 1.0 - TARGET_COVERAGE -ACI_STEP = 0.05 # Gibbs-Candes learning rate gamma. -EPS = 1e-9 - - -def _load_intervals() -> pd.DataFrame: - df = pd.read_parquet(INTERVALS_PATH) - df = df.copy() - df["period"] = df["temporal_segment"].astype(str).str.split("vintage=").str[-1] - df["y_true"] = pd.to_numeric(df["y_true"], errors="coerce").astype(float) - df["covered_90"] = ( - (df["pd_low_90"] - EPS <= df["y_true"]) & (df["y_true"] <= df["pd_high_90"] + EPS) - ).astype(float) - df["width_90"] = (df["pd_high_90"] - df["pd_low_90"]).astype(float) - logger.info( - "Loaded {} conformal intervals ({} periods, {} grades)", - len(df), - df["period"].nunique(), - df["grade"].nunique(), - ) - return df - - -def _slice_coverage(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame: - grouped = df.groupby(columns, dropna=False) - out = grouped.agg( - n=("y_true", "size"), - coverage_90=("covered_90", "mean"), - default_rate=("y_true", "mean"), - mean_width_90=("width_90", "mean"), - ).reset_index() - return out - - -def _build_a23(df: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: - marginal = float(df["covered_90"].mean()) - by_grade = _slice_coverage(df, ["grade"]).sort_values("grade") - by_period = _slice_coverage(df, ["period"]) - by_cell = _slice_coverage(df, ["grade", "period"]) - - rows: list[dict[str, Any]] = [ - { - "slice_type": "marginal", - "slice": "all", - "n": int(len(df)), - "coverage_90": marginal, - "default_rate": float(df["y_true"].mean()), - "mean_width_90": float(df["width_90"].mean()), - "robust_gap_vs_target": max(0.0, TARGET_COVERAGE - marginal), - "meets_target_90": bool(marginal >= TARGET_COVERAGE), - } - ] - for _, r in by_grade.iterrows(): - rows.append( - { - "slice_type": "grade", - "slice": str(r["grade"]), - "n": int(r["n"]), - "coverage_90": float(r["coverage_90"]), - "default_rate": float(r["default_rate"]), - "mean_width_90": float(r["mean_width_90"]), - "robust_gap_vs_target": max(0.0, TARGET_COVERAGE - float(r["coverage_90"])), - "meets_target_90": bool(float(r["coverage_90"]) >= TARGET_COVERAGE), - } - ) - # Worst grade x vintage cell with non-trivial support (unknown-group stress). - by_cell_supported = by_cell.loc[by_cell["n"] >= 200] - worst_cell = by_cell_supported.sort_values("coverage_90").iloc[0] - rows.append( - { - "slice_type": "worst_grade_x_period_cell", - "slice": f"{worst_cell['grade']}|{worst_cell['period']}", - "n": int(worst_cell["n"]), - "coverage_90": float(worst_cell["coverage_90"]), - "default_rate": float(worst_cell["default_rate"]), - "mean_width_90": float(worst_cell["mean_width_90"]), - "robust_gap_vs_target": max(0.0, TARGET_COVERAGE - float(worst_cell["coverage_90"])), - "meets_target_90": bool(float(worst_cell["coverage_90"]) >= TARGET_COVERAGE), - } - ) - frame = pd.DataFrame(rows) - - worst_grade = by_grade.sort_values("coverage_90").iloc[0] - worst_period = by_period.sort_values("coverage_90").iloc[0] - summary = { - "marginal_coverage_90": marginal, - "min_grade_coverage_90": float(worst_grade["coverage_90"]), - "worst_grade": str(worst_grade["grade"]), - "min_period_coverage_90": float(worst_period["coverage_90"]), - "worst_period": str(worst_period["period"]), - "min_grade_period_cell_coverage_90": float(worst_cell["coverage_90"]), - "worst_grade_period_cell": f"{worst_cell['grade']}|{worst_cell['period']}", - "robust_coverage_gap": max(0.0, TARGET_COVERAGE - float(worst_cell["coverage_90"])), - "all_grades_meet_target": bool((by_grade["coverage_90"] >= TARGET_COVERAGE).all()), - "all_supported_cells_meet_target": bool( - (by_cell_supported["coverage_90"] >= TARGET_COVERAGE).all() - ), - "n_supported_cells": int(len(by_cell_supported)), - } - return frame, summary - - -def _build_a24(df: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: - by_period = _slice_coverage(df, ["period"]).sort_values("period").reset_index(drop=True) - rows: list[dict[str, Any]] = [] - cum_n = 0.0 - cum_cov = 0.0 - alpha_t = TARGET_ALPHA # ACI controller target before the first vintage. - for _, r in by_period.iterrows(): - n = int(r["n"]) - cov = float(r["coverage_90"]) - miscov = 1.0 - cov - cum_n += n - cum_cov += cov * n - rows.append( - { - "period": str(r["period"]), - "n": n, - "default_rate": float(r["default_rate"]), - "coverage_90": cov, - "miscoverage_90": miscov, - "mean_width_90": float(r["mean_width_90"]), - "cumulative_coverage_90": cum_cov / max(cum_n, 1.0), - "aci_alpha_target_before": alpha_t, - } - ) - # Gibbs-Candes ACI update on the realized vintage miscoverage. - alpha_t = float(np.clip(alpha_t + ACI_STEP * (TARGET_ALPHA - miscov), 0.0, 1.0)) - frame = pd.DataFrame(rows) - alpha_traj = frame["aci_alpha_target_before"].to_numpy(dtype=float) - summary = { - "n_periods": int(len(frame)), - "first_period": str(frame["period"].iloc[0]), - "last_period": str(frame["period"].iloc[-1]), - "min_period_coverage_90": float(frame["coverage_90"].min()), - "max_period_coverage_90": float(frame["coverage_90"].max()), - "final_cumulative_coverage_90": float(frame["cumulative_coverage_90"].iloc[-1]), - "all_periods_meet_target": bool((frame["coverage_90"] >= TARGET_COVERAGE).all()), - "aci_step_gamma": ACI_STEP, - "aci_alpha_target_max_abs_deviation": float(np.max(np.abs(alpha_traj - TARGET_ALPHA))), - "default_rate_first": float(frame["default_rate"].iloc[0]), - "default_rate_last": float(frame["default_rate"].iloc[-1]), - } - return frame, summary - - -def build_distribution_robustness_diagnostics() -> dict[str, Any]: - start = datetime.now(tz=UTC) - df = _load_intervals() - a23, a23_summary = _build_a23(df) - a24, a24_summary = _build_a24(df) - artifacts = write_table( - TABLE_A23_NAME, a23, table_dir=TABLE_DIR, root=ROOT, float_precision=4 - ) + write_table(TABLE_A24_NAME, a24, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - status = { - "schema_version": "2026-05-28.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "elapsed_sec": (datetime.now(tz=UTC) - start).total_seconds(), - "target_coverage_90": TARGET_COVERAGE, - "source_intervals": str(INTERVALS_PATH.relative_to(ROOT)).replace("\\", "/"), - "generated_artifacts": [str(p.relative_to(ROOT)).replace("\\", "/") for p in artifacts], - "multidistribution_robustness": a23_summary, - "online_conformal_stability": a24_summary, - "champion_promotion_changed": False, - "notes": [ - "Read-only diagnostics on frozen Mondrian conformal intervals.", - "A23 stresses coverage across unknown groups (grade, grade x vintage).", - "A24 is a static-OOT online-control diagnostic, not streaming validation.", - ], - } - write_json(STATUS_PATH, status) - logger.info("Wrote {}", STATUS_PATH.relative_to(ROOT)) - logger.info( - "A23 worst grade={} cov={:.4f}; worst cell={} cov={:.4f}", - a23_summary["worst_grade"], - a23_summary["min_grade_coverage_90"], - a23_summary["worst_grade_period_cell"], - a23_summary["min_grade_period_cell_coverage_90"], - ) - logger.info( - "A24 periods={} min_cov={:.4f} final_cum_cov={:.4f} aci_alpha_max_dev={:.4f}", - a24_summary["n_periods"], - a24_summary["min_period_coverage_90"], - a24_summary["final_cumulative_coverage_90"], - a24_summary["aci_alpha_target_max_abs_deviation"], - ) - return status - - -def main() -> int: - build_distribution_robustness_diagnostics() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/build_ijds_binary_geometry_frontier_v4_evidence.py b/scripts/build_ijds_binary_geometry_frontier_v4_evidence.py new file mode 100644 index 0000000..2044d4e --- /dev/null +++ b/scripts/build_ijds_binary_geometry_frontier_v4_evidence.py @@ -0,0 +1,2264 @@ +"""Build the single paper-facing IJDS V4 evidence package.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, cast + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from loguru import logger + +from src.ijds_audit.claim_ledger import materialize_claim_ledger +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.grid_contracts import ( + require_exact_grid, + require_finite, + require_unique_row, + require_unique_value, +) +from src.ijds_audit.publication_generation import ( + promote_publication_generation, + publication_implementation_descriptors, + staged_artifact_descriptor, + staged_output_path, +) +from src.ijds_audit.publication_sources import load_verified_source_registry +from src.ijds_audit.robustness_evidence import ( + allocation_granularity_publication_table, + fit_label_completion_publication_table, + load_allocation_granularity_evidence, + load_fit_label_completion_evidence, +) +from src.ijds_audit.sensitivity_evidence import ( + endpoint_publication_table, + load_endpoint_sensitivity_evidence, +) +from src.ijds_audit.structural_evidence import ( + load_structural_sensitivity_evidence, + structural_publication_table, +) +from src.utils.artifact_descriptor import relative_artifact_descriptor +from src.utils.pipeline_runtime import atomic_write_strict_json, atomic_write_text + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_REGISTRY_PATH = ROOT / "configs/ijds_active_evidence_sources.yaml" +CLAIM_LEDGER_PATH = ROOT / "configs/ijds_claim_ledger.yaml" +EVIDENCE_PATH = ROOT / "reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json" +TABLE_DIR = ROOT / "reports/crpto/tables" +FIGURE_DIR = ROOT / "reports/crpto/figures" + +TABLE_TARGETS = { + "coverage": TABLE_DIR / "crpto_ijds_v4_table1_coverage_windows.csv", + "phase_transition": TABLE_DIR / "crpto_ijds_v4_table2_phase_transition.csv", + "development_envelopes": TABLE_DIR / "crpto_ijds_v4_table3_development_envelopes.csv", + "direction_summary": TABLE_DIR / "crpto_ijds_v4_table4_direction_summary.csv", + "two_ruler_tracks": TABLE_DIR / "crpto_ijds_v4_table5_two_ruler_tracks.csv", + "named_comparators": TABLE_DIR / "crpto_ijds_v4_tableS1_named_comparators.csv", + "credit_controls": TABLE_DIR / "crpto_ijds_v4_table6_credit_controls.csv", + "credit_prediction_metrics": TABLE_DIR / "crpto_ijds_v4_tableS2_credit_prediction_metrics.csv", + "woe_iv_psi": TABLE_DIR / "crpto_ijds_v4_tableS3_woe_iv_psi.csv", + "score_psi": TABLE_DIR / "crpto_ijds_v4_tableS4_score_psi.csv", + "label_lag_sensitivity": TABLE_DIR / "crpto_ijds_v4_tableS5_label_lag_sensitivity.csv", + "endpoint_availability_sensitivity": ( + TABLE_DIR / "crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv" + ), + "portfolio_structure_sensitivity": ( + TABLE_DIR / "crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv" + ), + "endpoint_resolution": TABLE_DIR / "crpto_ijds_v4_tableS8_endpoint_resolution.csv", + "missingness_encoding": ( + TABLE_DIR / "crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv" + ), + "rolling_origin": TABLE_DIR / "crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv", + "fit_label_completion": (TABLE_DIR / "crpto_ijds_v4_tableS11_fit_label_completion.csv"), + "allocation_granularity": (TABLE_DIR / "crpto_ijds_v4_tableS12_allocation_granularity.csv"), +} +FIGURE_STEMS = { + "coverage": "crpto_ijds_v4_fig1_coverage", + "phase_transition": "crpto_ijds_v4_fig2_phase_transition", + "development_envelopes": "crpto_ijds_v4_fig3_envelopes", +} + +CREDIT_LEARNER_ORDER = ( + "catboost_platt", + "numeric_logistic_platt", + "catboost_monotonic_platt", + "woe_scorecard_platform_platt", + "woe_scorecard_borrower_platt", +) +CREDIT_LEARNER_LABELS = { + "catboost_platt": "CatBoost", + "numeric_logistic_platt": "Numeric logistic", + "catboost_monotonic_platt": "Monotonic CatBoost", + "woe_scorecard_platform_platt": "Platform-signal WOE scorecard", + "woe_scorecard_borrower_platt": "Pricing-excluded application WOE scorecard", +} +WINDOW_IDS = ( + "w01_2012m01_m06", + "w02_2012m02_m07", + "w03_2012m03_m08", + "w04_2012m04_m09", + "w05_2012m05_m10", + "w06_2012m06_m11", + "w07_2012m07_m12", + "w08_2012m08_2013m01", +) +ROLLING_WINDOW_IDS = ( + "w01_2013m01_m06", + "w02_2013m02_m07", + "w03_2013m03_m08", + "w04_2013m04_m09", + "w05_2013m05_m10", + "w06_2013m06_m11", + "w07_2013m07_m12", + "w08_2013m08_2014m01", +) +PREDICTION_ROLES = ( + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", +) +SCORE_PSI_ROLES = PREDICTION_ROLES[1:] +RULERS = ("objective_matched", "normalized_score") +COORDINATES = (0.25, 0.50, 0.75) +TWO_RULER_METRICS = ( + "standardized_payoff", + "funded_default", + "funded_binary_miscoverage", +) +EXPECTED_TWO_RULER_COUNTS = { + "evaluated_portfolios": 6240, + "joined_funded_rows": 622455, + "window_endpoint_contrasts": 48, + "monthly_endpoint_contrasts": 720, + "metric_direction_cells": 144, + "outcome_audit_rows": 8, +} +POLICY_IDS = tuple(f"linear-{index:03d}" for index in range(1, 10)) +PRIMARY_PERIODS = tuple(str(period) for period in pd.period_range("2016-04", "2017-06", freq="M")) +SUPPORT_SCOPES = ( + "named_c0_c1_c2", + "development_admissible_exact_frontier", + "broad_stress_exact_frontier", +) +SUPPORT_METRICS = ("standardized_payoff", "terminal_default", "funded_miscoverage") + +BLUE = "#2F6690" +ORANGE = "#D97706" +GOLD = "#C8A951" +INK = "#20262E" +MID = "#6B7280" +LIGHT = "#E5E7EB" + + +def _verified_path(descriptor: Mapping[str, Any]) -> Path: + path = (ROOT / str(descriptor["path"])).resolve() + actual = relative_artifact_descriptor(path, repo_root=ROOT) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Paper-facing artifact mismatch for {path}: {field}.") + return path + + +def _require_identity( + actual: Mapping[str, Any], + expected: Mapping[str, Any], + *, + label: str, +) -> None: + """Fail when a registered run identity differs from its frozen artifact.""" + fields = ("run_tag", "protocol_tag", "protocol_commit") + mismatches = [field for field in fields if actual.get(field) != expected.get(field)] + if mismatches: + raise RuntimeError(f"{label} identity changed: {', '.join(mismatches)}.") + + +def _require_machine_tolerance_recovery( + recovery: Mapping[str, Any] | None, + *, + label: str, +) -> dict[str, float]: + """Validate a V5 reason-only recovery and return its observed drift maxima.""" + if not isinstance(recovery, Mapping): + raise RuntimeError(f"{label} omits the endpoint-recovery audit.") + if recovery.get("status") != "reference_column_equivalence_verified_with_float_tolerance": + raise RuntimeError(f"{label} endpoint-recovery status changed.") + equivalence = recovery.get("equivalence") + if ( + not isinstance(equivalence, Mapping) + or equivalence.get("non_float_columns_exact") is not True + ): + raise RuntimeError(f"{label} does not retain exact non-floating equivalence.") + float_atol = float(equivalence.get("float_atol", -1.0)) + float_rtol = float(equivalence.get("float_rtol", -1.0)) + if not (0.0 <= float_atol <= 1.0e-12 and 0.0 <= float_rtol <= 1.0e-12): + raise RuntimeError(f"{label} float tolerances exceed the publication ceiling.") + frames = recovery.get("frames") + if not isinstance(frames, Mapping) or not frames: + raise RuntimeError(f"{label} endpoint recovery has no reconciled frames.") + maximum_absolute = 0.0 + maximum_relative = 0.0 + for frame_name, raw_frame in frames.items(): + if not isinstance(raw_frame, Mapping): + raise TypeError(f"{label} recovery frame {frame_name!r} must be a mapping.") + drift = raw_frame.get("float_drift") + if not isinstance(drift, Mapping): + raise RuntimeError(f"{label} recovery frame {frame_name!r} omits float drift.") + for column, raw_values in drift.items(): + if not isinstance(raw_values, Mapping): + raise TypeError( + f"{label} drift record {frame_name!r}/{column!r} must be a mapping." + ) + absolute = float(raw_values.get("maximum_absolute", float("nan"))) + relative = float(raw_values.get("maximum_relative", float("nan"))) + if not np.isfinite(absolute) or not np.isfinite(relative): + raise RuntimeError(f"{label} endpoint-recovery drift is not finite.") + if absolute < 0.0 or relative < 0.0 or absolute > 1.0e-12 or relative > 1.0e-12: + raise RuntimeError(f"{label} endpoint-recovery drift exceeds machine scale.") + maximum_absolute = max(maximum_absolute, absolute) + maximum_relative = max(maximum_relative, relative) + return { + "float_atol": float_atol, + "float_rtol": float_rtol, + "maximum_absolute_drift": maximum_absolute, + "maximum_relative_drift": maximum_relative, + } + + +def _read_json(path: Path, *, label: str) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError(f"{label} must contain a JSON object.") + return payload + + +def _verified_artifact_paths( + descriptors: Mapping[str, Mapping[str, Any]], +) -> dict[str, Path]: + return {name: _verified_path(descriptor) for name, descriptor in descriptors.items()} + + +def _require_clean_execution(payload: Mapping[str, Any], *, label: str) -> None: + if ( + payload.get("protected_stages_run") != [] + or payload.get("protected_artifacts_written") != [] + ): + raise RuntimeError(f"{label} reports a protected-stage side effect.") + + +@dataclass(frozen=True) +class V4Inputs: + config_path: Path + summary_path: Path + receipt_path: Path + config: dict[str, Any] + summary: dict[str, Any] + recovery: dict[str, float] + artifacts: dict[str, Path] + freeze_path: Path + source_freeze_path: Path + source_artifacts: dict[str, Path] + + +def _load_v4_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> V4Inputs: + config_path = registered["v4_config"] + summary_path = registered["v4_summary"] + receipt_path = registered["v4_receipt"] + config = load_v4_config(config_path) + summary = _read_json(summary_path, label="V4 summary") + if summary.get("status") != "complete_retrospective_binary_geometry_frontier_audit": + raise RuntimeError("V4 deterministic summary is incomplete.") + _require_identity(summary, lineage["evaluation"], label="V4 evaluation") + recovery = _require_machine_tolerance_recovery( + summary.get("endpoint_reason_recovery"), + label="V4 evaluation", + ) + receipt = _read_json(receipt_path, label="V4 execution receipt") + if receipt.get("protocol_commit") != lineage["evaluation"]["protocol_commit"]: + raise RuntimeError("V4 receipt protocol commit changed.") + if receipt.get("summary") != relative_artifact_descriptor(summary_path, repo_root=ROOT): + raise RuntimeError("V4 receipt no longer binds the registered summary.") + artifacts = _verified_artifact_paths(summary["artifacts"]) + freeze_path = _verified_path(summary["outcome_free_freeze"]) + freeze = _read_json(freeze_path, label="V4 outcome-free freeze") + _require_identity( + freeze["outcome_free_lineage"], + lineage["outcome_free"], + label="V4 outcome-free freeze", + ) + source_freeze = freeze["outcome_free_lineage"]["source_protocol_freeze"] + source_freeze_path = _verified_path(source_freeze) + if source_freeze["sha256"] != lineage["outcome_free"]["freeze_sha256"]: + raise RuntimeError("V4 outcome-free freeze hash changed.") + return V4Inputs( + config_path=config_path, + summary_path=summary_path, + receipt_path=receipt_path, + config=config, + summary=summary, + recovery=recovery, + artifacts=artifacts, + freeze_path=freeze_path, + source_freeze_path=source_freeze_path, + source_artifacts=_verified_artifact_paths(freeze["outcome_free_artifacts"]), + ) + + +@dataclass(frozen=True) +class TwoRulerInputs: + manifest_path: Path + freeze_path: Path + summary_path: Path + receipt_path: Path + source_artifacts: dict[str, Path] + evaluation_artifacts: dict[str, Path] + summary: dict[str, Any] + recovery: dict[str, float] + + +def _load_two_ruler_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> TwoRulerInputs: + manifest_path = registered["two_ruler_manifest"] + manifest = _read_json(manifest_path, label="Two-ruler manifest") + if manifest.get("status") != "verified_post_freeze_outcome_evaluation_complete": + raise RuntimeError("The verified two-ruler evaluation is incomplete.") + _require_identity(manifest, lineage["evaluation"], label="Two-ruler evaluation") + freeze_path = _verified_path(manifest["source_frontier_freeze"]) + freeze = _read_json(freeze_path, label="Two-ruler outcome-free freeze") + _require_identity(freeze, lineage["outcome_free"], label="Two-ruler outcome-free freeze") + if manifest["source_frontier_freeze"]["sha256"] != lineage["outcome_free"]["freeze_sha256"]: + raise RuntimeError("Two-ruler outcome-free freeze hash changed.") + if any(value is not None for value in manifest["selection"].values()): + raise RuntimeError("The active manifest reports a selected two-ruler result.") + _require_clean_execution(manifest, label="The active two-ruler manifest") + summary_path = _verified_path(manifest["summary"]) + receipt_path = _verified_path(manifest["execution_receipt"]) + summary = _read_json(summary_path, label="Two-ruler summary") + if summary.get("counts") != EXPECTED_TWO_RULER_COUNTS: + raise RuntimeError("The active two-ruler evaluation census changed.") + recovery = _require_machine_tolerance_recovery( + summary.get("endpoint_reason_recovery"), + label="Two-ruler evaluation", + ) + return TwoRulerInputs( + manifest_path=manifest_path, + freeze_path=freeze_path, + summary_path=summary_path, + receipt_path=receipt_path, + source_artifacts=_verified_artifact_paths(manifest["source_artifacts"]), + evaluation_artifacts=_verified_artifact_paths(manifest["evaluation_artifacts"]), + summary=summary, + recovery=recovery, + ) + + +def _direction_pattern(directions: pd.DataFrame, metric: str) -> str: + counts = directions.loc[directions["metric"].eq(metric), "direction"].value_counts() + order = ("gamma_1_higher", "gamma_1_lower", "crosses_zero", "exact_zero") + return ";".join(f"{name}:{int(counts[name])}" for name in order if name in counts) + + +def _two_ruler_track_table( + window_contrasts: pd.DataFrame, + directions: pd.DataFrame, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + ruler_order = {"objective_matched": 0, "normalized_score": 1} + for group_key, frame in window_contrasts.groupby( + ["ruler", "coordinate"], observed=True, sort=True + ): + if not isinstance(group_key, tuple) or len(group_key) != 2: + raise RuntimeError("Unexpected two-ruler group key.") + ruler, coordinate = group_key + coordinate_value = float(cast(Any, coordinate)) + scoped = directions.loc[ + directions["ruler"].eq(ruler) & directions["coordinate"].eq(coordinate) + ] + rows.append( + { + "ruler": str(ruler), + "coordinate": coordinate_value, + "ruler_semantics": ( + "common_plugin_objective_floor" + if str(ruler) == "objective_matched" + else "common_relative_score_relaxation" + ), + "window_cells": int(len(frame)), + "active_months_per_window_min": int(frame["nonidentical_months"].min()), + "active_months_per_window_max": int(frame["nonidentical_months"].max()), + "expected_objective_difference_usd_min": float( + frame["expected_objective_difference"].min() + ), + "expected_objective_difference_usd_max": float( + frame["expected_objective_difference"].max() + ), + "payoff_bound_usd_lower_min": float( + frame["realized_payoff_difference_lower"].min() + ), + "payoff_bound_usd_upper_max": float( + frame["realized_payoff_difference_upper"].max() + ), + "payoff_identification_width_usd_min": float( + frame["realized_payoff_identification_width"].min() + ), + "payoff_identification_width_usd_max": float( + frame["realized_payoff_identification_width"].max() + ), + "default_bound_pp_lower_min": float( + 100.0 * frame["weighted_default_difference_lower"].min() + ), + "default_bound_pp_upper_max": float( + 100.0 * frame["weighted_default_difference_upper"].max() + ), + "default_identification_width_pp_min": float( + 100.0 * frame["weighted_default_identification_width"].min() + ), + "default_identification_width_pp_max": float( + 100.0 * frame["weighted_default_identification_width"].max() + ), + "miscoverage_bound_pp_lower_min": float( + 100.0 * frame["weighted_miscoverage_difference_lower"].min() + ), + "miscoverage_bound_pp_upper_max": float( + 100.0 * frame["weighted_miscoverage_difference_upper"].max() + ), + "miscoverage_identification_width_pp_min": float( + 100.0 * frame["weighted_miscoverage_identification_width"].min() + ), + "miscoverage_identification_width_pp_max": float( + 100.0 * frame["weighted_miscoverage_identification_width"].max() + ), + "payoff_direction_pattern": _direction_pattern(scoped, "standardized_payoff"), + "default_direction_pattern": _direction_pattern(scoped, "funded_default"), + "miscoverage_direction_pattern": _direction_pattern( + scoped, "funded_binary_miscoverage" + ), + } + ) + table = pd.DataFrame(rows) + table["_ruler_order"] = table["ruler"].map(ruler_order) + return table.sort_values(["_ruler_order", "coordinate"]).drop(columns="_ruler_order") + + +def _objective_quarter_repetition(joined: pd.DataFrame) -> dict[str, Any]: + labels = ("objective_matched_g100_c025", "objective_matched_g000_c025") + scoped = joined.loc[joined["role"].eq("primary_oot") & joined["policy_label"].isin(labels)] + audits: list[dict[str, Any]] = [] + reference: pd.DataFrame | None = None + identical_to_cents = True + for window_id, frame in scoped.groupby("window_id", observed=True, sort=True): + exposures = ( + frame.pivot(index=["period", "id"], columns="policy_label", values="exposure") + .fillna(0.0) + .sort_index() + ) + delta = exposures[labels[0]] - exposures[labels[1]] + rounded = exposures[list(labels)].round(2) + if reference is None: + reference = rounded + else: + identical_to_cents = bool(identical_to_cents and rounded.equals(reference)) + audits.append( + { + "window_id": str(window_id), + "changed_loan_month_positions": int(delta.abs().gt(1.0e-8).sum()), + "one_way_turnover_usd": float(delta.abs().sum() / 2.0), + } + ) + audit = pd.DataFrame(audits) + return { + "allocations_identical_across_windows_to_cents": identical_to_cents, + "changed_loan_month_positions_min": int(audit["changed_loan_month_positions"].min()), + "changed_loan_month_positions_max": int(audit["changed_loan_month_positions"].max()), + "one_way_turnover_usd_min": float(audit["one_way_turnover_usd"].min()), + "one_way_turnover_usd_max": float(audit["one_way_turnover_usd"].max()), + } + + +def _write_csv(frame: pd.DataFrame, path: Path) -> Path: + return atomic_write_text(path, frame.to_csv(index=False, lineterminator="\n")) + + +def _credit_control_tables( + prediction_metrics: pd.DataFrame, + temporal_coverage: pd.DataFrame, + woe_summary: pd.DataFrame, + feature_psi: pd.DataFrame, + score_psi: pd.DataFrame, +) -> dict[str, pd.DataFrame]: + metrics = prediction_metrics.copy() + require_exact_grid( + metrics, + domains={"learner": CREDIT_LEARNER_ORDER, "role": PREDICTION_ROLES}, + label="five-model prediction metrics", + ) + require_finite( + metrics, + ( + "candidate_rows", + "resolved_rows", + "unresolved_rows", + "default_rate", + "roc_auc", + "gini", + "ks", + "average_precision", + "brier", + "log_loss", + "ece_10", + "calibration_in_the_large", + "calibration_intercept", + "calibration_slope", + ), + label="five-model prediction metrics", + ) + if not metrics["calibration_optimizer_success"].all(): + raise RuntimeError("A declared calibration diagnostic did not converge.") + + canonical = temporal_coverage.loc[ + temporal_coverage["taxonomy_groups"].eq(5) + & temporal_coverage["role"].eq("primary_oot") + & temporal_coverage["conformal_group"].eq(-1) + ].copy() + require_exact_grid( + canonical, + domains={"learner": CREDIT_LEARNER_ORDER, "window_id": WINDOW_IDS}, + label="five-model canonical coverage", + ) + require_finite( + canonical, + ("candidate_rows", "resolved_rows", "unresolved_rows", "coverage_lower", "coverage_upper"), + label="five-model canonical coverage", + ) + + primary_rows: list[dict[str, Any]] = [] + for learner in CREDIT_LEARNER_ORDER: + metric = metrics.loc[metrics["learner"].eq(learner) & metrics["role"].eq("primary_oot")] + coverage = canonical.loc[canonical["learner"].eq(learner)] + row = require_unique_row( + metric, + key={"learner": learner, "role": "primary_oot"}, + label="primary OOT prediction metrics", + ) + require_exact_grid( + coverage, + domains={"learner": (learner,), "window_id": WINDOW_IDS}, + label=f"primary OOT coverage for {learner}", + ) + primary_rows.append( + { + "learner": learner, + "learner_label": CREDIT_LEARNER_LABELS[learner], + "candidate_rows": int(row["candidate_rows"]), + "resolved_rows": int(row["resolved_rows"]), + "unresolved_rows": int(row["unresolved_rows"]), + "default_rate": float(row["default_rate"]), + "roc_auc": float(row["roc_auc"]), + "gini": float(row["gini"]), + "ks": float(row["ks"]), + "average_precision": float(row["average_precision"]), + "brier": float(row["brier"]), + "log_loss": float(row["log_loss"]), + "ece_10": float(row["ece_10"]), + "mean_calibration_error": float(row["calibration_in_the_large"]), + "calibration_intercept": float(row["calibration_intercept"]), + "calibration_slope": float(row["calibration_slope"]), + "coverage_lower_min": float(coverage["coverage_lower"].min()), + "coverage_upper_max": float(coverage["coverage_upper"].max()), + "windows_upper_below_0_90": int(coverage["coverage_upper"].lt(0.90).sum()), + } + ) + primary = pd.DataFrame(primary_rows) + + role_order = { + role: index + for index, role in enumerate( + ( + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ) + ) + } + learner_order = {learner: index for index, learner in enumerate(CREDIT_LEARNER_ORDER)} + metrics.insert(1, "learner_label", metrics["learner"].map(CREDIT_LEARNER_LABELS)) + metrics["_learner_order"] = metrics["learner"].map(learner_order) + metrics["_role_order"] = metrics["role"].map(role_order) + metrics = metrics.sort_values(["_learner_order", "_role_order"]).drop( + columns=["_learner_order", "_role_order"] + ) + metrics = metrics.rename(columns={"calibration_in_the_large": "mean_calibration_error"}) + + primary_feature_psi = feature_psi.loc[ + feature_psi["comparison_role"].eq("primary_oot"), + ["learner", "feature", "psi"], + ].rename(columns={"psi": "primary_oot_psi"}) + woe = woe_summary.rename(columns={"name": "feature"}).merge( + primary_feature_psi, + on=["learner", "feature"], + how="left", + validate="one_to_one", + ) + if len(woe) != 45 or woe["primary_oot_psi"].isna().any(): + raise RuntimeError("The WOE/IV and primary OOT PSI census changed.") + woe["_learner_order"] = woe["learner"].map(learner_order) + woe = woe.sort_values(["_learner_order", "iv"], ascending=[True, False]).drop( + columns="_learner_order" + ) + + score = score_psi.copy() + require_exact_grid( + score, + domains={"learner": CREDIT_LEARNER_ORDER, "comparison_role": SCORE_PSI_ROLES}, + label="five-model score PSI", + ) + require_finite(score, ("psi",), label="five-model score PSI") + score.insert(1, "learner_label", score["learner"].map(CREDIT_LEARNER_LABELS)) + score["_learner_order"] = score["learner"].map(learner_order) + score["_role_order"] = score["comparison_role"].map(role_order) + score = score.sort_values(["_learner_order", "_role_order"]).drop( + columns=["_learner_order", "_role_order"] + ) + return { + "credit_controls": primary, + "credit_prediction_metrics": metrics, + "woe_iv_psi": woe, + "score_psi": score, + } + + +@dataclass(frozen=True) +class CreditInputs: + summary_path: Path + receipt_path: Path + summary: dict[str, Any] + recovery: dict[str, float] + freeze_path: Path + freeze: dict[str, Any] + evaluation_artifacts: dict[str, Path] + outcome_free_artifacts: dict[str, Path] + model_artifacts: dict[str, Path] + prediction_metrics: pd.DataFrame + temporal_coverage: pd.DataFrame + woe_summary: pd.DataFrame + feature_psi: pd.DataFrame + score_psi: pd.DataFrame + feature_variation: pd.DataFrame + tables: dict[str, pd.DataFrame] + + +def _load_credit_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> CreditInputs: + summary_path = registered["credit_summary"] + receipt_path = registered["credit_receipt"] + summary = _read_json(summary_path, label="Credit-control summary") + if summary.get("status") != "complete_no_model_selection_credit_risk_control_evaluation": + raise RuntimeError("The verified credit-control evaluation is incomplete.") + receipt = _read_json(receipt_path, label="Credit-control execution receipt") + _require_identity(receipt, lineage["evaluation"], label="Credit-control evaluation") + expected_interpretation = { + "model_or_feature_selected_from_oot": False, + "portfolio_claim_authorized": False, + "scorecard_superiority_claim_authorized": False, + "universal_transport_claim_authorized": False, + } + if summary.get("interpretation") != expected_interpretation: + raise RuntimeError("The active credit-control claim boundary changed.") + _require_clean_execution(summary, label="The active credit controls") + if summary.get("coverage_recovery") is not None: + raise RuntimeError("Credit controls unexpectedly report a coverage-recovery block.") + recovery = _require_machine_tolerance_recovery( + summary.get("endpoint_reason_recovery"), + label="Credit-control evaluation", + ) + freeze_path = _verified_path(summary["source_freeze"]) + freeze = _read_json(freeze_path, label="Credit-control outcome-free freeze") + if freeze.get("status") != "credit_control_scores_frozen_before_primary_oot_outcome_join": + raise RuntimeError("The V1b credit-control freeze is incomplete.") + _require_identity(freeze, lineage["outcome_free"], label="Credit-control outcome-free freeze") + if summary["source_freeze"]["sha256"] != lineage["outcome_free"]["freeze_sha256"]: + raise RuntimeError("Credit-control outcome-free freeze hash changed.") + if freeze.get("co_primary_learners") != list(CREDIT_LEARNER_ORDER): + raise RuntimeError("The frozen five-model specification changed.") + if ( + freeze.get("model_selection") != "none_all_five_reported" + or freeze.get("window_selection") != "none_all_eight_reported" + or freeze.get("portfolio_optimization") is not False + or freeze.get("sampling") != "none_all_eligible_rows" + or freeze.get("primary_oot_outcome_columns_in_frozen_scores") != [] + ): + raise RuntimeError("The frozen credit-control selection boundary changed.") + evaluation_artifacts = _verified_artifact_paths(summary["evaluation_artifacts"]) + outcome_free_artifacts = _verified_artifact_paths(freeze["outcome_free_artifacts"]) + model_artifacts = _verified_artifact_paths(freeze["model_artifacts"]) + prediction_metrics = pd.read_parquet(evaluation_artifacts["prediction_metrics"]) + temporal_coverage = pd.read_parquet(evaluation_artifacts["temporal_coverage"]) + woe_summary = pd.read_parquet(outcome_free_artifacts["woe_summary"]) + feature_psi = pd.read_parquet(outcome_free_artifacts["scorecard_feature_psi"]) + score_psi = pd.read_parquet(outcome_free_artifacts["score_psi"]) + feature_variation = pd.read_parquet(outcome_free_artifacts["feature_variation"]) + return CreditInputs( + summary_path=summary_path, + receipt_path=receipt_path, + summary=summary, + recovery=recovery, + freeze_path=freeze_path, + freeze=freeze, + evaluation_artifacts=evaluation_artifacts, + outcome_free_artifacts=outcome_free_artifacts, + model_artifacts=model_artifacts, + prediction_metrics=prediction_metrics, + temporal_coverage=temporal_coverage, + woe_summary=woe_summary, + feature_psi=feature_psi, + score_psi=score_psi, + feature_variation=feature_variation, + tables=_credit_control_tables( + prediction_metrics, + temporal_coverage, + woe_summary, + feature_psi, + score_psi, + ), + ) + + +def _direction(lower: pd.Series, upper: pd.Series) -> pd.Series: + return pd.Series( + np.where( + lower > 0.0, + "guardrail_higher", + np.where(upper < 0.0, "guardrail_lower", "crosses_zero"), + ), + index=lower.index, + dtype="string", + ) + + +def _style() -> None: + plt.rcParams.update( + { + "font.family": "DejaVu Sans", + "font.size": 9, + "axes.titlesize": 10, + "axes.labelsize": 9, + "axes.edgecolor": MID, + "axes.labelcolor": INK, + "xtick.color": INK, + "ytick.color": INK, + "text.color": INK, + "figure.facecolor": "white", + "axes.facecolor": "white", + "axes.grid": True, + "grid.color": LIGHT, + "grid.linewidth": 0.7, + "grid.alpha": 0.8, + "legend.frameon": False, + } + ) + + +def _save_figure(figure: plt.Figure, stem: str, *, output_dir: Path) -> dict[str, Path]: + output_dir.mkdir(parents=True, exist_ok=True) + png = output_dir / f"{stem}.png" + pdf = output_dir / f"{stem}.pdf" + figure.savefig(png, dpi=300, bbox_inches="tight", facecolor="white") + figure.savefig( + pdf, + bbox_inches="tight", + facecolor="white", + metadata={"CreationDate": None, "ModDate": None}, + ) + plt.close(figure) + return {"png": png, "pdf": pdf} + + +def _coverage_figure(coverage: pd.DataFrame, *, output_dir: Path) -> dict[str, Path]: + _style() + figure, axis = plt.subplots(figsize=(7.2, 3.7)) + labels = [f"W{index}" for index in range(1, 9)] + x = np.arange(8, dtype=float) + specifications = ( + ("catboost_platt", "CatBoost + Platt", BLUE, "o", -0.08), + ("numeric_logistic_platt", "Logistic + Platt", ORANGE, "s", 0.08), + ) + for learner, label, color, marker, offset in specifications: + frame = coverage.loc[coverage["learner"].eq(learner)].sort_values("window_id") + center = frame["coverage_resolved"].to_numpy(dtype=float) + lower = frame["coverage_lower"].to_numpy(dtype=float) + upper = frame["coverage_upper"].to_numpy(dtype=float) + axis.errorbar( + x + offset, + center, + yerr=np.vstack([center - lower, upper - center]), + color=color, + marker=marker, + markersize=4.5, + linewidth=1.4, + capsize=2.5, + label=label, + ) + axis.axhline(0.90, color=INK, linestyle="--", linewidth=1.1, label="Nominal 0.90") + axis.set_xticks(x, labels) + axis.set_ylim(0.83, 0.905) + axis.set_ylabel("Coverage") + axis.set_xlabel("Six-month residual window (W1 = Jan-Jun 2012; W8 = Aug 2012-Jan 2013)") + axis.set_title("Primary OOT binary-outcome coverage across the complete window specification") + axis.legend(ncol=3, loc="lower left") + axis.spines[["top", "right"]].set_visible(False) + figure.tight_layout() + return _save_figure(figure, FIGURE_STEMS["coverage"], output_dir=output_dir) + + +def _phase_figure(phase: pd.DataFrame, *, output_dir: Path) -> dict[str, Path]: + _style() + frame = phase.sort_values("window_id") + require_exact_grid( + frame, + domains={"window_id": WINDOW_IDS}, + label="phase-transition figure", + ) + w7 = require_unique_row( + frame, + key={"window_id": "w07_2012m07_m12"}, + label="phase-transition W7", + ) + w8 = require_unique_row( + frame, + key={"window_id": "w08_2012m08_2013m01"}, + label="phase-transition W8", + ) + x = np.arange(len(frame), dtype=float) + labels = [f"W{index}" for index in range(1, 9)] + figure, axes = plt.subplots(1, 2, figsize=(7.2, 3.35), sharex=True) + axes[0].plot(x, frame["fit_prevalence"], color=BLUE, marker="o", linewidth=1.5) + axes[0].axhline(0.10, color=INK, linestyle="--", linewidth=1.1, label=r"$\alpha=0.10$") + axes[0].set_ylabel("Fit default prevalence") + axes[0].set_title("Stratum-2 prevalence") + axes[0].legend(loc="lower left") + axes[1].plot( + x, + frame["fit_residual_quantile"], + color=GOLD, + marker="s", + linewidth=1.5, + ) + axes[1].set_ylabel("Residual quantile") + axes[1].set_title("Applied conformal quantile") + for axis in axes: + axis.set_xticks(x, labels) + axis.set_xlabel("Residual window") + axis.spines[["top", "right"]].set_visible(False) + axes[0].annotate( + "W7: 0.1017", + xy=(6, float(w7["fit_prevalence"])), + xytext=(4.6, 0.111), + arrowprops={"arrowstyle": "-", "color": MID}, + fontsize=8, + ) + axes[0].annotate( + "W8: 0.0971", + xy=(7, float(w8["fit_prevalence"])), + xytext=(5.5, 0.0975), + arrowprops={"arrowstyle": "-", "color": MID}, + fontsize=8, + ) + axes[1].annotate( + "0.8884 to 0.1118", + xy=(7, float(w8["fit_residual_quantile"])), + xytext=(3.8, 0.35), + arrowprops={"arrowstyle": "->", "color": MID}, + fontsize=8, + ) + figure.suptitle("Binary residual geometry changes discontinuously at the prevalence threshold") + figure.tight_layout() + return _save_figure(figure, FIGURE_STEMS["phase_transition"], output_dir=output_dir) + + +def _envelope_figure(envelopes: pd.DataFrame, *, output_dir: Path) -> dict[str, Path]: + _style() + metrics = ("standardized_payoff", "funded_miscoverage") + direction_code = {"guardrail_lower": -1, "crosses_zero": 0, "guardrail_higher": 1} + colors = [BLUE, "#F3F4F6", ORANGE] + from matplotlib.colors import BoundaryNorm, ListedColormap + + cmap = ListedColormap(colors) + norm = BoundaryNorm([-1.5, -0.5, 0.5, 1.5], cmap.N) + figure, axes = plt.subplots(2, 1, figsize=(7.2, 5.2), sharex=True) + for axis, metric in zip(axes, metrics, strict=True): + frame = envelopes.loc[envelopes["metric"].eq(metric)].copy() + matrix = ( + frame.assign(code=frame["direction"].map(direction_code)) + .pivot(index="paired_policy_id", columns="window_id", values="code") + .sort_index() + ) + axis.imshow(matrix.to_numpy(dtype=float), cmap=cmap, norm=norm, aspect="auto") + axis.set_yticks(np.arange(9), [f"P{index}" for index in range(1, 10)]) + axis.set_ylabel("Policy") + axis.set_title( + "Standardized payoff" if metric == "standardized_payoff" else "Funded miscoverage" + ) + axis.grid(False) + for row in range(matrix.shape[0]): + for column in range(matrix.shape[1]): + value = int(matrix.iloc[row, column]) + axis.text( + column, + row, + {1: "+", 0: "0", -1: "-"}[value], + ha="center", + va="center", + color=INK if value == 0 else "white", + fontsize=8, + fontweight="bold", + ) + axes[-1].set_xticks(np.arange(8), [f"W{index}" for index in range(1, 9)]) + axes[-1].set_xlabel("Residual window") + figure.suptitle("Guardrail-minus-point envelopes over the development-admissible cap frontier") + figure.text( + 0.5, + 0.015, + "- guardrail lower; 0 envelope crosses zero; + guardrail higher. Default crosses zero in every cell.", + ha="center", + fontsize=8, + color=MID, + ) + figure.tight_layout(rect=(0, 0.04, 1, 0.96)) + return _save_figure(figure, FIGURE_STEMS["development_envelopes"], output_dir=output_dir) + + +@dataclass(frozen=True) +class DiagnosticInputs: + raw_audit_path: Path + raw_audit: dict[str, Any] + raw_artifacts: dict[str, Path] + raw_coverage_exceptions: pd.DataFrame + lag_evidence_path: Path + lag_evidence: dict[str, Any] + lag_table_path: Path + lag_table: pd.DataFrame + admissible_lag_table: pd.DataFrame + nonadmissible_lag_table: pd.DataFrame + lag_w7_w8: pd.DataFrame + tie_evidence_path: Path + tie_census: dict[str, Any] + tie_order: dict[str, Any] + + +def _load_diagnostic_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> DiagnosticInputs: + raw_audit_path = registered["raw_data_audit"] + raw_audit = _read_json(raw_audit_path, label="Raw-data audit") + if raw_audit.get("status") != "complete_full_archive_data_contract_audit": + raise RuntimeError("The full-archive data audit is incomplete.") + if raw_audit.get("run_tag") != lineage["raw_data_audit"]["run_tag"]: + raise RuntimeError("The raw-data audit identity changed.") + _require_clean_execution(raw_audit, label="The raw-data audit") + raw_artifacts = _verified_artifact_paths(raw_audit["artifacts"]) + raw_feature_contract = pd.read_csv(raw_artifacts["feature_contract"]) + raw_coverage_exceptions = raw_feature_contract.loc[ + raw_feature_contract["coverage_exception"].notna() + & raw_feature_contract["requires_sensitivity"].eq(True) + ].copy() + if len(raw_coverage_exceptions) != 2: + raise RuntimeError("The declared raw-feature coverage exceptions changed.") + + lag_evidence_path = registered["label_lag_sensitivity"] + lag_evidence = _read_json(lag_evidence_path, label="Label-lag sensitivity") + if lag_evidence.get("status") != "complete_frozen_score_label_lag_sensitivity": + raise RuntimeError("The label-lag sensitivity is incomplete.") + _require_identity( + lag_evidence, + lineage["label_lag_sensitivity"], + label="Label-lag sensitivity", + ) + lag_table_path = _verified_path(lag_evidence["artifact"]) + lag_table = pd.read_csv(lag_table_path) + require_exact_grid( + lag_table, + domains={"charged_off_lag_months": (0, 3, 6, 8, 12), "window_id": WINDOW_IDS}, + label="fit-label lag sensitivity", + ) + require_finite( + lag_table, + ("minimum_monthly_retention", "phase_prevalence", "phase_residual_quantile"), + label="fit-label lag sensitivity", + ) + admissible_lag_table = lag_table.loc[lag_table["passes_locked_retention"]].copy() + nonadmissible_lag_table = lag_table.loc[~lag_table["passes_locked_retention"]].copy() + if set(admissible_lag_table["charged_off_lag_months"]) != {0, 3, 6}: + raise RuntimeError("The admissible label-lag set changed.") + lag_w7_w8 = lag_table.loc[ + lag_table["window_id"].isin(("w07_2012m07_m12", "w08_2012m08_2013m01")) + ].copy() + + tie_evidence_path = registered["solver_tie_audit"] + tie_evidence = _read_json(tie_evidence_path, label="Solver-tie audit") + if tie_evidence.get("status") != "complete_prefreeze_structural_evidence": + raise RuntimeError("The solver-tie audit is incomplete.") + _require_identity(tie_evidence, lineage["solver_tie_audit"], label="Solver-tie audit") + tie_census = tie_evidence["results"]["point_cap_census"] + tie_order = tie_evidence["results"]["order_sensitivity"] + if int(tie_census["near_zero_bases"]) != 0 or int(tie_order["tie_sensitive_rows"]) != 0: + raise RuntimeError("The evaluated point-cap census contains an unresolved solver tie.") + return DiagnosticInputs( + raw_audit_path=raw_audit_path, + raw_audit=raw_audit, + raw_artifacts=raw_artifacts, + raw_coverage_exceptions=raw_coverage_exceptions, + lag_evidence_path=lag_evidence_path, + lag_evidence=lag_evidence, + lag_table_path=lag_table_path, + lag_table=lag_table, + admissible_lag_table=admissible_lag_table, + nonadmissible_lag_table=nonadmissible_lag_table, + lag_w7_w8=lag_w7_w8, + tie_evidence_path=tie_evidence_path, + tie_census=tie_census, + tie_order=tie_order, + ) + + +@dataclass(frozen=True) +class RollingInputs: + summary_path: Path + receipt_path: Path + summary: dict[str, Any] + artifacts: dict[str, Path] + coverage: pd.DataFrame + + +def _load_rolling_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> RollingInputs: + summary_path = registered["rolling_origin_summary"] + receipt_path = registered["rolling_origin_receipt"] + summary = _read_json(summary_path, label="Rolling-origin summary") + if summary.get("status") != "complete_retrospective_binary_geometry_frontier_audit": + raise RuntimeError("The rolling-origin evaluation is incomplete.") + _require_identity(summary, lineage, label="Rolling-origin evaluation") + receipt = _read_json(receipt_path, label="Rolling-origin execution receipt") + if receipt.get("protocol_commit") != lineage["protocol_commit"]: + raise RuntimeError("The rolling-origin receipt protocol commit changed.") + if receipt.get("summary") != relative_artifact_descriptor(summary_path, repo_root=ROOT): + raise RuntimeError("The rolling-origin receipt no longer binds its summary.") + _require_clean_execution(summary, label="The rolling-origin run") + if summary.get("endpoint_reason_recovery") is not None: + raise RuntimeError("The rolling-origin run violates its fresh-run boundary.") + artifacts = _verified_artifact_paths(summary["artifacts"]) + coverage_all = pd.read_parquet(artifacts["temporal_coverage"]) + coverage = coverage_all.loc[ + coverage_all["learner"].eq("catboost_platt") + & coverage_all["taxonomy_groups"].eq(5) + & coverage_all["role"].eq("primary_oot") + & coverage_all["conformal_group"].eq(-1) + ].sort_values("window_id") + require_exact_grid( + coverage, + domains={"window_id": ROLLING_WINDOW_IDS}, + label="rolling-origin primary coverage", + ) + require_finite( + coverage, + ("candidate_rows", "resolved_rows", "unresolved_rows", "coverage_lower", "coverage_upper"), + label="rolling-origin primary coverage", + ) + return RollingInputs( + summary_path=summary_path, + receipt_path=receipt_path, + summary=summary, + artifacts=artifacts, + coverage=coverage, + ) + + +@dataclass(frozen=True) +class MissingnessInputs: + summary_path: Path + receipt_path: Path + summary: dict[str, Any] + freeze_path: Path + artifacts: dict[str, Path] + freeze_artifacts: dict[str, Path] + model_artifacts: dict[str, Path] + publication_table: pd.DataFrame + + +@dataclass(frozen=True) +class StagedPublicationGeneration: + """One complete set of staged paper-facing tables and figures.""" + + table_paths: dict[str, Path] + figures: dict[str, dict[str, Path]] + figure_targets: dict[str, dict[str, Path]] + outputs: dict[Path, Path] + + +def _load_missingness_inputs( + registered: Mapping[str, Path], + lineage: Mapping[str, Any], +) -> MissingnessInputs: + summary_path = registered["missingness_summary"] + receipt_path = registered["missingness_receipt"] + summary = _read_json(summary_path, label="Missingness summary") + if summary.get("status") != "complete_no_selection_missingness_sensitivity": + raise RuntimeError("The missingness-encoding sensitivity is incomplete.") + _require_identity(summary, lineage, label="Missingness sensitivity") + receipt = _read_json(receipt_path, label="Missingness execution receipt") + _require_identity(receipt, lineage, label="Missingness receipt") + if receipt.get("summary") != relative_artifact_descriptor(summary_path, repo_root=ROOT): + raise RuntimeError("The missingness receipt no longer binds its summary.") + expected_interpretation = { + "model_or_encoding_selected": False, + "portfolio_claim_authorized": False, + "missing_at_random_claim_authorized": False, + "robustness_scope": "three_declared_feature_semantics_preserving_missingness_encodings_only", + } + if summary.get("interpretation") != expected_interpretation: + raise RuntimeError("The missingness sensitivity claim boundary changed.") + _require_clean_execution(summary, label="The missingness sensitivity") + freeze_path = _verified_path(summary["source_freeze"]) + freeze = _read_json(freeze_path, label="Missingness outcome-free freeze") + if ( + freeze.get("status") != "missingness_scores_frozen_before_primary_oot_outcome_join" + or freeze.get("primary_oot_outcome_columns_in_frozen_scores") != [] + or any(value is not None for value in freeze.get("selection", {}).values()) + ): + raise RuntimeError("The missingness outcome-free freeze boundary changed.") + artifacts = _verified_artifact_paths(summary["evaluation_artifacts"]) + freeze_artifacts = _verified_artifact_paths(freeze["outcome_free_artifacts"]) + model_artifacts = _verified_artifact_paths(freeze["model_artifacts"]) + coverage_all = pd.read_parquet(artifacts["temporal_coverage"]) + coverage = coverage_all.loc[ + coverage_all["taxonomy_groups"].eq(5) + & coverage_all["role"].eq("primary_oot") + & coverage_all["conformal_group"].eq(-1) + ].sort_values(["learner", "window_id"]) + learners = tuple(str(item["id"]) for item in summary["specifications"]) + require_exact_grid( + coverage, + domains={"learner": learners, "window_id": WINDOW_IDS}, + label="missingness-encoding coverage", + ) + prediction = pd.read_parquet(artifacts["prediction_metrics"]) + if set(prediction["learner"].astype(str)) != set(learners): + raise RuntimeError("The missingness prediction-metric family changed.") + publication_table = pd.DataFrame(summary["coverage"]).merge( + prediction[["learner", "roc_auc", "brier", "log_loss", "ece_10", "calibration_slope"]], + on="learner", + how="left", + validate="one_to_one", + ) + return MissingnessInputs( + summary_path=summary_path, + receipt_path=receipt_path, + summary=summary, + freeze_path=freeze_path, + artifacts=artifacts, + freeze_artifacts=freeze_artifacts, + model_artifacts=model_artifacts, + publication_table=publication_table, + ) + + +def _stage_publication_generation( + staging_root: Path, + *, + tables: Mapping[str, pd.DataFrame], + coverage: pd.DataFrame, + phase: pd.DataFrame, + development_envelopes: pd.DataFrame, +) -> StagedPublicationGeneration: + """Write one complete staged generation and validate its exact surface.""" + if set(tables) != set(TABLE_TARGETS): + missing = sorted(set(TABLE_TARGETS).difference(tables)) + unexpected = sorted(set(tables).difference(TABLE_TARGETS)) + raise RuntimeError( + f"The publication table family changed: missing={missing}, unexpected={unexpected}." + ) + staged_table_targets = { + name: staged_output_path(staging_root, target, repo_root=ROOT) + for name, target in TABLE_TARGETS.items() + } + table_paths = { + name: _write_csv(tables[name], staged_table_targets[name]) for name in TABLE_TARGETS + } + staged_figure_dir = staging_root / "outputs" / FIGURE_DIR.relative_to(ROOT) + figures = { + "coverage": _coverage_figure(coverage, output_dir=staged_figure_dir), + "phase_transition": _phase_figure(phase, output_dir=staged_figure_dir), + "development_envelopes": _envelope_figure( + development_envelopes, + output_dir=staged_figure_dir, + ), + } + figure_targets = { + name: {kind: FIGURE_DIR / f"{FIGURE_STEMS[name]}.{kind}" for kind in ("png", "pdf")} + for name in FIGURE_STEMS + } + outputs = { + **{TABLE_TARGETS[name]: path for name, path in table_paths.items()}, + **{ + figure_targets[name][kind]: path + for name, paths in figures.items() + for kind, path in paths.items() + }, + } + expected_targets = { + *TABLE_TARGETS.values(), + *(target for targets in figure_targets.values() for target in targets.values()), + } + if len(outputs) != 24 or set(outputs) != expected_targets: + raise RuntimeError( + "The staged publication generation is not exactly 18 CSVs and 6 figures." + ) + return StagedPublicationGeneration( + table_paths=table_paths, + figures=figures, + figure_targets=figure_targets, + outputs=outputs, + ) + + +def _without_simulation_artifacts(artifacts: Mapping[str, Path]) -> dict[str, Path]: + """Exclude the historical synthetic mechanism outputs from paper evidence.""" + return {name: path for name, path in artifacts.items() if not name.startswith("simulation_")} + + +def _publication_source_descriptors( + *, + direct_paths: Mapping[str, Path], + artifact_groups: Mapping[str, Mapping[str, Path]], +) -> dict[str, dict[str, Any]]: + """Describe every implementation and scientific source exactly once.""" + descriptors = dict(publication_implementation_descriptors(ROOT)) + + def add(name: str, path: Path) -> None: + if name in descriptors: + raise RuntimeError(f"Duplicate publication source descriptor: {name!r}.") + descriptors[name] = relative_artifact_descriptor(path, repo_root=ROOT) + + for name, path in direct_paths.items(): + add(name, path) + for prefix, paths in artifact_groups.items(): + for name, path in paths.items(): + add(f"{prefix}/{name}", path) + return descriptors + + +def _paper_artifact_descriptors( + generation: StagedPublicationGeneration, +) -> dict[str, dict[str, Any]]: + """Bind staged outputs to their final paper-facing paths.""" + descriptors = { + f"table/{name}": staged_artifact_descriptor( + path, + TABLE_TARGETS[name], + repo_root=ROOT, + ) + for name, path in generation.table_paths.items() + } + descriptors.update( + { + f"figure/{name}/{kind}": staged_artifact_descriptor( + path, + generation.figure_targets[name][kind], + repo_root=ROOT, + ) + for name, paths in generation.figures.items() + for kind, path in paths.items() + } + ) + return descriptors + + +def _build_evidence(staging_root: Path) -> Path: + registry, registered = load_verified_source_registry( + SOURCE_REGISTRY_PATH, + repo_root=ROOT, + ) + lineages = cast(dict[str, Any], registry["lineages"]) + v4_lineage = cast(dict[str, Any], lineages["binary_geometry"]) + two_ruler_lineage = cast(dict[str, Any], lineages["two_ruler"]) + credit_lineage = cast(dict[str, Any], lineages["credit_controls"]) + diagnostic_lineage = cast(dict[str, Any], lineages["diagnostics"]) + sensitivities = cast(dict[str, Any], registry["sensitivities"]) + endpoint_lineage = cast(dict[str, Any], sensitivities["endpoint_availability"]) + structural_lineage = cast(dict[str, Any], sensitivities["portfolio_structure"]) + rolling_lineage = cast(dict[str, Any], sensitivities["rolling_origin"]) + missingness_lineage = cast(dict[str, Any], sensitivities["missingness_encoding"]) + fit_label_lineage = cast(dict[str, Any], sensitivities["fit_label_completion"]) + granularity_lineage = cast(dict[str, Any], sensitivities["allocation_granularity"]) + v4 = _load_v4_inputs(registered, v4_lineage) + config_path = v4.config_path + summary_path = v4.summary_path + v4_receipt_path = v4.receipt_path + config = v4.config + summary = v4.summary + v4_recovery = v4.recovery + artifacts = v4.artifacts + freeze_path = v4.freeze_path + v4_source_freeze_path = v4.source_freeze_path + source_artifacts = v4.source_artifacts + + two_ruler = _load_two_ruler_inputs(registered, two_ruler_lineage) + two_ruler_manifest_path = two_ruler.manifest_path + two_ruler_freeze_path = two_ruler.freeze_path + two_ruler_summary_path = two_ruler.summary_path + two_ruler_receipt_path = two_ruler.receipt_path + two_ruler_evaluation_artifacts = two_ruler.evaluation_artifacts + two_ruler_source_artifacts = two_ruler.source_artifacts + two_ruler_summary = two_ruler.summary + two_ruler_recovery = two_ruler.recovery + expected_two_ruler_counts = EXPECTED_TWO_RULER_COUNTS + + credit = _load_credit_inputs(registered, credit_lineage) + credit_summary_path = credit.summary_path + credit_receipt_path = credit.receipt_path + credit_summary = credit.summary + credit_recovery = credit.recovery + credit_freeze_path = credit.freeze_path + credit_freeze = credit.freeze + credit_evaluation_artifacts = credit.evaluation_artifacts + credit_outcome_free_artifacts = credit.outcome_free_artifacts + credit_model_artifacts = credit.model_artifacts + diagnostics = _load_diagnostic_inputs(registered, diagnostic_lineage) + raw_audit_path = diagnostics.raw_audit_path + raw_audit = diagnostics.raw_audit + raw_audit_artifacts = diagnostics.raw_artifacts + raw_coverage_exceptions = diagnostics.raw_coverage_exceptions + lag_evidence_path = diagnostics.lag_evidence_path + lag_evidence = diagnostics.lag_evidence + lag_table_path = diagnostics.lag_table_path + lag_table = diagnostics.lag_table + admissible_lag_table = diagnostics.admissible_lag_table + nonadmissible_lag_table = diagnostics.nonadmissible_lag_table + lag_w7_w8 = diagnostics.lag_w7_w8 + tie_evidence_path = diagnostics.tie_evidence_path + tie_census = diagnostics.tie_census + tie_order = diagnostics.tie_order + + credit_prediction_metrics = credit.prediction_metrics + credit_temporal_coverage = credit.temporal_coverage + credit_woe_summary = credit.woe_summary + credit_feature_psi = credit.feature_psi + credit_score_psi = credit.score_psi + credit_feature_variation = credit.feature_variation + credit_tables = credit.tables + + two_ruler_windows = pd.read_parquet(two_ruler_evaluation_artifacts["window_endpoint_contrasts"]) + two_ruler_monthly = pd.read_parquet( + two_ruler_evaluation_artifacts["monthly_endpoint_contrasts"] + ) + two_ruler_directions = pd.read_parquet( + two_ruler_evaluation_artifacts["metric_direction_census"] + ) + two_ruler_joined = pd.read_parquet(two_ruler_evaluation_artifacts["joined_funded_allocations"]) + structural_config_path = registered["structural_sensitivity_config"] + structural_freeze_path = registered["structural_sensitivity_freeze"] + structural_summary_path = registered["structural_sensitivity_summary"] + structural_evidence = load_structural_sensitivity_evidence( + structural_summary_path, + freeze_path=structural_freeze_path, + config_path=structural_config_path, + identity=structural_lineage, + repo_root=ROOT, + reference_two_ruler=two_ruler_windows, + ) + structural_table = structural_publication_table(structural_evidence) + + rolling = _load_rolling_inputs(registered, rolling_lineage) + rolling_summary_path = rolling.summary_path + rolling_receipt_path = rolling.receipt_path + rolling_summary = rolling.summary + rolling_artifacts = rolling.artifacts + rolling_coverage = rolling.coverage + + missingness = _load_missingness_inputs(registered, missingness_lineage) + missingness_summary_path = missingness.summary_path + missingness_receipt_path = missingness.receipt_path + missingness_summary = missingness.summary + missingness_freeze_path = missingness.freeze_path + missingness_artifacts = missingness.artifacts + missingness_freeze_artifacts = missingness.freeze_artifacts + missingness_model_artifacts = missingness.model_artifacts + missingness_table = missingness.publication_table + fit_label_freeze_path = registered["fit_label_completion_freeze"] + fit_label_summary_path = registered["fit_label_completion_summary"] + fit_label_evidence = load_fit_label_completion_evidence( + fit_label_summary_path, + freeze_path=fit_label_freeze_path, + identity=fit_label_lineage, + repo_root=ROOT, + ) + fit_label_table = fit_label_completion_publication_table(fit_label_evidence) + granularity_freeze_path = registered["allocation_granularity_freeze"] + granularity_summary_path = registered["allocation_granularity_summary"] + granularity_evidence = load_allocation_granularity_evidence( + granularity_summary_path, + freeze_path=granularity_freeze_path, + identity=granularity_lineage, + repo_root=ROOT, + ) + granularity_table = allocation_granularity_publication_table(granularity_evidence) + require_exact_grid( + two_ruler_windows, + domains={"window_id": WINDOW_IDS, "ruler": RULERS, "coordinate": COORDINATES}, + label="two-ruler window contrasts", + ) + require_exact_grid( + two_ruler_monthly, + domains={ + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + "period": PRIMARY_PERIODS, + }, + label="two-ruler monthly contrasts", + ) + require_exact_grid( + two_ruler_directions, + domains={ + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + "metric": TWO_RULER_METRICS, + }, + label="two-ruler metric directions", + ) + two_ruler_table = _two_ruler_track_table(two_ruler_windows, two_ruler_directions) + require_exact_grid( + two_ruler_table, + domains={"ruler": RULERS, "coordinate": COORDINATES}, + label="paper-facing two-ruler tracks", + ) + objective_quarter = _objective_quarter_repetition(two_ruler_joined) + + coverage_all = pd.read_parquet(artifacts["temporal_coverage"]) + coverage = coverage_all.loc[ + coverage_all["taxonomy_groups"].eq(5) + & coverage_all["role"].eq("primary_oot") + & coverage_all["conformal_group"].eq(-1) + ].sort_values(["learner", "window_id"]) + require_exact_grid( + coverage, + domains={ + "learner": ("catboost_platt", "numeric_logistic_platt"), + "window_id": WINDOW_IDS, + }, + label="detailed V4 canonical coverage", + ) + require_finite( + coverage, + ("candidate_rows", "resolved_rows", "unresolved_rows", "coverage_lower", "coverage_upper"), + label="detailed V4 canonical coverage", + ) + phase = coverage_all.loc[ + coverage_all["learner"].eq("catboost_platt") + & coverage_all["taxonomy_groups"].eq(5) + & coverage_all["role"].eq("primary_oot") + & coverage_all["conformal_group"].eq(2) + ].sort_values("window_id") + require_exact_grid( + phase, + domains={"window_id": WINDOW_IDS}, + label="binary phase transition", + ) + require_finite( + phase, + ( + "fit_prevalence", + "fit_residual_quantile", + "mean_width", + "coverage_lower", + "coverage_upper", + ), + label="binary phase transition", + ) + contrasts = pd.read_parquet(artifacts["paired_contrasts"]) + envelopes = pd.read_parquet(artifacts["comparator_envelopes"]) + require_exact_grid( + envelopes, + domains={ + "window_id": WINDOW_IDS, + "paired_policy_id": POLICY_IDS, + "scope": SUPPORT_SCOPES, + "metric": SUPPORT_METRICS, + }, + label="exact comparator envelopes", + ) + require_finite(envelopes, ("lower", "upper"), label="exact comparator envelopes") + if not envelopes["lower"].le(envelopes["upper"]).all(): + raise RuntimeError("An exact comparator envelope has reversed bounds.") + endpoint_summary_path = registered["endpoint_sensitivity_summary"] + endpoint_evidence = load_endpoint_sensitivity_evidence( + endpoint_summary_path, + identity=endpoint_lineage, + repo_root=ROOT, + reference_coverage=credit_temporal_coverage, + reference_two_ruler=two_ruler_windows, + reference_envelopes=envelopes, + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + endpoint_table = endpoint_publication_table(endpoint_evidence) + require_exact_grid( + endpoint_table, + domains={"charged_off_lag_months": (0, 3, 6, 8, 12)}, + label="paper-facing endpoint availability sensitivity", + ) + endpoint_sensitivity_artifacts = { + name: _verified_path(descriptor) + for name, descriptor in endpoint_evidence.summary["artifacts"].items() + } + development_envelopes = envelopes.loc[ + envelopes["scope"].eq("development_admissible_exact_frontier") + ].copy() + fit_audit = pd.read_parquet(source_artifacts["fit_audit"]) + solve_records = pd.read_parquet(source_artifacts["solve_records"]) + support = pd.read_parquet(source_artifacts["comparator_support"]) + require_exact_grid( + support, + domains={"window_id": WINDOW_IDS, "paired_policy_id": POLICY_IDS}, + label="development comparator support", + ) + require_finite( + support, + ("development_months", "c1_cap", "support_lower", "support_upper"), + label="development comparator support", + ) + if not support["support_lower"].le(support["support_upper"]).all(): + raise RuntimeError("Development comparator support has reversed bounds.") + + endpoint_resolution_table = pd.DataFrame(summary["endpoint_resolution_audit"]) + endpoint_resolution_table = endpoint_resolution_table.loc[ + endpoint_resolution_table["role"].eq("primary_oot") + ].sort_values("snapshot_resolution") + expected_endpoint_reasons = { + "charged_off_by_reconstructed_cutoff", + "fully_paid_by_reconstructed_cutoff", + "nonterminal_or_unresolved_status", + "terminal_after_reconstructed_cutoff", + "terminal_availability_date_missing", + } + if ( + set(endpoint_resolution_table["snapshot_resolution"].astype(str)) + != expected_endpoint_reasons + or int(endpoint_resolution_table["candidate_rows"].sum()) != 376890 + or int(endpoint_resolution_table["resolved_rows"].sum()) != 364814 + or int(endpoint_resolution_table["unresolved_rows"].sum()) != 12076 + ): + raise RuntimeError("The primary endpoint-reason census changed.") + + primary_origin = coverage.loc[coverage["learner"].eq("catboost_platt")].copy() + primary_origin.insert(0, "origin", "primary_2016") + later_origin = rolling_coverage.copy() + later_origin.insert(0, "origin", "rolling_2017") + rolling_table_columns = [ + "origin", + "window_id", + "candidate_rows", + "resolved_rows", + "unresolved_rows", + "coverage_resolved", + "coverage_lower", + "coverage_upper", + "mean_width", + ] + rolling_table = pd.concat( + [primary_origin[rolling_table_columns], later_origin[rolling_table_columns]], + ignore_index=True, + ) + if len(rolling_table) != 16 or not rolling_table["coverage_upper"].lt(0.90).all(): + raise RuntimeError("The two-origin retrospective recurrence contract changed.") + + fit_coverage = ( + fit_audit.loc[fit_audit["taxonomy_groups"].eq(5)] + .groupby(["learner", "window_id"], observed=True)["covered"] + .mean() + .rename("fit_coverage") + .reset_index() + ) + coverage_table = coverage.merge(fit_coverage, on=["learner", "window_id"], how="left") + phase_table = phase[ + [ + "window_id", + "fit_rows", + "fit_prevalence", + "fit_residual_quantile", + "coverage_lower", + "coverage_upper", + "mean_width", + "set_empty_share", + "set_zero_only_share", + "set_both_share", + ] + ].copy() + direction_table = ( + development_envelopes.groupby(["metric", "direction"], observed=True) + .size() + .rename("cells") + .reset_index() + ) + named = contrasts.loc[~contrasts["comparator_rule"].eq("point_cap_frontier")].copy() + named["payoff_direction"] = _direction( + named["realized_payoff_difference_lower"], named["realized_payoff_difference_upper"] + ) + named["default_direction"] = _direction( + named["weighted_default_difference_lower"], named["weighted_default_difference_upper"] + ) + named["miscoverage_direction"] = _direction( + named["weighted_miscoverage_difference_lower"], + named["weighted_miscoverage_difference_upper"], + ) + named_counts: list[dict[str, Any]] = [] + for rule, frame in named.groupby("comparator_rule", observed=True, sort=True): + for metric in ("payoff", "default", "miscoverage"): + counts = frame[f"{metric}_direction"].value_counts() + named_counts.append( + { + "comparator_rule": str(rule), + "metric": metric, + "guardrail_lower": int(counts.get("guardrail_lower", 0)), + "crosses_zero": int(counts.get("crosses_zero", 0)), + "guardrail_higher": int(counts.get("guardrail_higher", 0)), + } + ) + named_table = pd.DataFrame(named_counts) + + publication_generation = _stage_publication_generation( + staging_root, + tables={ + "coverage": coverage_table, + "phase_transition": phase_table, + "development_envelopes": development_envelopes, + "direction_summary": direction_table, + "two_ruler_tracks": two_ruler_table, + "named_comparators": named_table, + "credit_controls": credit_tables["credit_controls"], + "credit_prediction_metrics": credit_tables["credit_prediction_metrics"], + "woe_iv_psi": credit_tables["woe_iv_psi"], + "score_psi": credit_tables["score_psi"], + "label_lag_sensitivity": lag_table.sort_values(["charged_off_lag_months", "window_id"]), + "endpoint_availability_sensitivity": endpoint_table, + "portfolio_structure_sensitivity": structural_table, + "endpoint_resolution": endpoint_resolution_table, + "missingness_encoding": missingness_table, + "rolling_origin": rolling_table, + "fit_label_completion": fit_label_table, + "allocation_granularity": granularity_table, + }, + coverage=coverage, + phase=phase, + development_envelopes=development_envelopes, + ) + + c2 = solve_records.loc[solve_records["comparator_rule"].eq("c2_contemporaneous")] + broad = envelopes.loc[envelopes["scope"].eq("broad_stress_exact_frontier")] + w8_development = development_envelopes.loc[ + development_envelopes["window_id"].eq("w08_2012m08_2013m01") + ] + credit_primary = credit_tables["credit_controls"] + primary_score_psi = credit_score_psi.loc[ + credit_score_psi["comparison_role"].eq("primary_oot") + ].set_index("learner")["psi"] + primary_feature_psi = credit_feature_psi.loc[ + credit_feature_psi["comparison_role"].eq("primary_oot") + ].sort_values("psi", ascending=False) + top_platform_iv = ( + credit_woe_summary.loc[credit_woe_summary["learner"].eq("woe_scorecard_platform_platt")] + .sort_values("iv", ascending=False) + .head(5)[["name", "iv"]] + .to_dict(orient="records") + ) + top_borrower_iv = ( + credit_woe_summary.loc[credit_woe_summary["learner"].eq("woe_scorecard_borrower_platt")] + .sort_values("iv", ascending=False) + .head(5)[["name", "iv"]] + .to_dict(orient="records") + ) + recent_chargeoff_variation = credit_feature_variation.loc[ + credit_feature_variation["feature"].eq("recent_chargeoff") + & credit_feature_variation["role"].isin(["pd_development", "probability_calibration"]) + ][["role", "rows", "unique_observed", "constant_observed"]] + primary_oot_candidates = int( + require_unique_value(coverage, "candidate_rows", label="detailed V4 canonical coverage") + ) + primary_oot_resolved = int( + require_unique_value(coverage, "resolved_rows", label="detailed V4 canonical coverage") + ) + primary_oot_unresolved = int( + require_unique_value(coverage, "unresolved_rows", label="detailed V4 canonical coverage") + ) + phase_w7 = require_unique_row( + phase, + key={"window_id": "w07_2012m07_m12"}, + label="binary phase transition W7", + ) + phase_w8 = require_unique_row( + phase, + key={"window_id": "w08_2012m08_2013m01"}, + label="binary phase transition W8", + ) + endpoint_by_reason = endpoint_resolution_table.set_index("snapshot_resolution") + structural_artifacts = _verified_artifact_paths( + cast(Mapping[str, Mapping[str, Any]], structural_evidence.summary["artifacts"]) + ) + source_artifact_descriptors = _publication_source_descriptors( + direct_paths={ + "config": config_path, + "outcome_free/source_protocol_freeze": v4_source_freeze_path, + "freeze": freeze_path, + "summary": summary_path, + "execution_receipt": v4_receipt_path, + "two_ruler/outcome_free/freeze": two_ruler_freeze_path, + "two_ruler/manifest": two_ruler_manifest_path, + "two_ruler/summary": two_ruler_summary_path, + "two_ruler/execution_receipt": two_ruler_receipt_path, + "credit_controls/summary": credit_summary_path, + "credit_controls/execution_receipt": credit_receipt_path, + "credit_controls/freeze": credit_freeze_path, + "raw_data_audit/manifest": raw_audit_path, + "label_lag_sensitivity/manifest": lag_evidence_path, + "label_lag_sensitivity/table": lag_table_path, + "endpoint_availability_sensitivity/summary": endpoint_summary_path, + "portfolio_structure_sensitivity/config": structural_config_path, + "portfolio_structure_sensitivity/freeze": structural_freeze_path, + "portfolio_structure_sensitivity/summary": structural_summary_path, + "rolling_origin/summary": rolling_summary_path, + "rolling_origin/execution_receipt": rolling_receipt_path, + "missingness_encoding/summary": missingness_summary_path, + "missingness_encoding/execution_receipt": missingness_receipt_path, + "missingness_encoding/freeze": missingness_freeze_path, + "fit_label_completion/freeze": fit_label_freeze_path, + "fit_label_completion/summary": fit_label_summary_path, + "allocation_granularity/freeze": granularity_freeze_path, + "allocation_granularity/summary": granularity_summary_path, + "solver_tie_audit/manifest": tie_evidence_path, + }, + artifact_groups={ + "outcome_free": source_artifacts, + "evaluation": _without_simulation_artifacts(artifacts), + "two_ruler/outcome_free": two_ruler_source_artifacts, + "two_ruler/evaluation": two_ruler_evaluation_artifacts, + "credit_controls/outcome_free": credit_outcome_free_artifacts, + "credit_controls/models": credit_model_artifacts, + "credit_controls/evaluation": credit_evaluation_artifacts, + "raw_data_audit": raw_audit_artifacts, + "endpoint_availability_sensitivity": endpoint_sensitivity_artifacts, + "portfolio_structure_sensitivity": structural_artifacts, + "rolling_origin": _without_simulation_artifacts(rolling_artifacts), + "missingness_encoding/evaluation": missingness_artifacts, + "missingness_encoding/outcome_free": missingness_freeze_artifacts, + "missingness_encoding/models": missingness_model_artifacts, + "fit_label_completion/outcome_free": fit_label_evidence.outcome_free_artifacts, + "fit_label_completion/evaluation": fit_label_evidence.evaluation_artifacts, + "allocation_granularity/outcome_free": (granularity_evidence.outcome_free_artifacts), + "allocation_granularity/evaluation": granularity_evidence.evaluation_artifacts, + }, + ) + paper_artifact_descriptors = _paper_artifact_descriptors(publication_generation) + evidence = { + "schema_version": "2026-07-15.4", + "status": "active_ijds_v5_endpoint_reason_audited_paper_facing_evidence", + "source_registry": { + "schema_version": str(registry["schema_version"]), + "status": str(registry["status"]), + "sources": sorted(registered), + }, + "lineages": lineages, + "sensitivities": sensitivities, + "replay_dependencies": dict(registry.get("replay_dependencies", {})), + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": str(summary["protocol_commit"]), + "claim_boundary": dict(summary["claim_boundary"]), + "design": { + "primary_oot_candidates": primary_oot_candidates, + "primary_oot_resolved": primary_oot_resolved, + "primary_oot_unresolved": primary_oot_unresolved, + "residual_windows": 8, + "learners": 5, + "v4_detailed_coverage_learners": 2, + "credit_control_learners": 5, + "portfolio_learners": 1, + "taxonomy_diagnostics": [1, 2, 5, 10], + "policies": 9, + "v4_policies_are_supporting_not_closed_family": True, + "oot_months": 15, + "development_months": 11, + "two_ruler_gamma_grid": [0.0, 0.25, 0.5, 0.75, 1.0], + "two_ruler_primary_contrast": "gamma_1_minus_gamma_0", + "two_ruler_interior_coordinates": [0.25, 0.5, 0.75], + "two_ruler_tracks": 6, + "frontier_caps": int( + contrasts.loc[ + contrasts["comparator_rule"].eq("point_cap_frontier"), "frontier_cap" + ].nunique() + ), + "development_support_lower": float(support["support_lower"].min()), + "development_support_upper": float(support["support_upper"].max()), + "evaluation_endpoint": str(config["design"]["endpoint"]), + "archive_is_verified_point_in_time_snapshot": bool( + config["target"]["evaluation_outcome_contract"][ + "archive_is_verified_point_in_time_snapshot" + ] + ), + }, + "coverage": { + "catboost_all_eight_upper_below_nominal": bool( + ( + coverage.loc[coverage["learner"].eq("catboost_platt"), "coverage_upper"] < 0.90 + ).all() + ), + "logistic_all_eight_upper_below_nominal": bool( + ( + coverage.loc[coverage["learner"].eq("numeric_logistic_platt"), "coverage_upper"] + < 0.90 + ).all() + ), + "catboost_bound_min": float( + coverage.loc[coverage["learner"].eq("catboost_platt"), "coverage_lower"].min() + ), + "catboost_bound_max": float( + coverage.loc[coverage["learner"].eq("catboost_platt"), "coverage_upper"].max() + ), + "logistic_bound_min": float( + coverage.loc[ + coverage["learner"].eq("numeric_logistic_platt"), "coverage_lower" + ].min() + ), + "logistic_bound_max": float( + coverage.loc[ + coverage["learner"].eq("numeric_logistic_platt"), "coverage_upper" + ].max() + ), + "rows": coverage_table.to_dict(orient="records"), + }, + "evaluation_endpoint": { + **dict(config["target"]["evaluation_outcome_contract"]), + "role": str(config["source"]["snapshot_date_role"]), + "terminal_statuses_after_cutoff_reclassified_unresolved": True, + "primary_oot_candidates": primary_oot_candidates, + "primary_oot_resolved": primary_oot_resolved, + "primary_oot_unresolved": primary_oot_unresolved, + "reason_census": endpoint_resolution_table.to_dict(orient="records"), + "reason_census_partitions_primary_candidates": bool( + endpoint_resolution_table["candidate_rows"].sum() == primary_oot_candidates + and endpoint_resolution_table["resolved_rows"].sum() == primary_oot_resolved + and endpoint_resolution_table["unresolved_rows"].sum() == primary_oot_unresolved + ), + "primary_oot_nonterminal_or_unresolved_status": int( + endpoint_by_reason.loc["nonterminal_or_unresolved_status", "candidate_rows"] + ), + "primary_oot_terminal_after_cutoff": int( + endpoint_by_reason.loc["terminal_after_reconstructed_cutoff", "candidate_rows"] + ), + "primary_oot_terminal_availability_date_missing": int( + endpoint_by_reason.loc["terminal_availability_date_missing", "candidate_rows"] + ), + "missingness_mechanism_identified": False, + "operational_event_dates_identified": False, + "recovery_audit": v4_recovery, + "last_payment_date_max": str(raw_audit["results"]["last_payment_date_max"]), + "last_credit_pull_date_max": str(raw_audit["results"]["last_credit_pull_date_max"]), + "last_payment_rows_after_cutoff": int( + raw_audit["results"]["last_payment_rows_after_cutoff"] + ), + "last_credit_pull_rows_after_cutoff": int( + raw_audit["results"]["last_credit_pull_rows_after_cutoff"] + ), + }, + "sensitivity": { + "evaluation_endpoint_availability": { + "scope": "complete_nonselective_retrospective_lag_grid", + "run_tag": str(endpoint_evidence.summary["run_tag"]), + "protocol_tag": str(endpoint_evidence.summary["protocol_tag"]), + "protocol_commit": str(endpoint_evidence.summary["protocol_commit"]), + "charged_off_lags_months": list(endpoint_evidence.summary["lags"]), + "endpoint_or_result_selected": False, + "allocation_refit": False, + "six_month_endpoint_reconciles_to_active_evaluation": True, + "reconciliation": dict(endpoint_evidence.reconciliation), + "fit_label_lag_crossed_factorially": False, + "estimand_boundary": ( + "This family changes evaluation-outcome availability while holding " + "scores, fitted residual recipes, supports, and allocations fixed. " + "The separate label-lag family changes conformal-fit labels and was " + "not crossed factorially with endpoint availability." + ), + "rows": endpoint_table.to_dict(orient="records"), + }, + "portfolio_structure": { + "scope": "complete_nonselective_budget_by_purpose_cap_by_lgd_grid", + "run_tag": str(structural_evidence.summary["run_tag"]), + "protocol_tag": str(structural_evidence.summary["protocol_tag"]), + "protocol_commit": str(structural_evidence.summary["protocol_commit"]), + "scenario_or_result_selected": False, + "baseline_reconciles_to_active_evaluation": True, + **dict(structural_evidence.findings), + "estimand_boundary": ( + "This complete retrospective assumption sensitivity changes budget, " + "purpose concentration, and LGD without selecting a scenario. Direction " + "remains conditional on ruler, coordinate, window, metric, and scenario." + ), + "rows": structural_table.to_dict(orient="records"), + }, + "rolling_origin": { + "scope": "two_origin_retrospective_recurrence_not_replication", + "run_tag": str(rolling_summary["run_tag"]), + "protocol_tag": str(rolling_summary["protocol_tag"]), + "protocol_commit": str(rolling_summary["protocol_commit"]), + "origins": ["primary_2016", "rolling_2017"], + "origin_count": 2, + "window_cells": int(len(rolling_table)), + "all_sixteen_upper_below_nominal": bool( + rolling_table["coverage_upper"].lt(0.90).all() + ), + "primary_2016_upper_max": float(primary_origin["coverage_upper"].max()), + "rolling_2017_upper_max": float(later_origin["coverage_upper"].max()), + "model_or_origin_selected": False, + "independent_replication_claim_authorized": False, + "rows": rolling_table.to_dict(orient="records"), + }, + "missingness_encoding": { + "scope": ("three_declared_feature_semantics_preserving_catboost_encodings"), + "run_tag": str(missingness_summary["run_tag"]), + "protocol_tag": str(missingness_summary["protocol_tag"]), + "protocol_commit": str(missingness_summary["protocol_commit"]), + "specifications": list(missingness_summary["specifications"]), + "all_three_all_eight_upper_below_nominal": bool( + missingness_table["all_windows_upper_below_nominal"].all() + and missingness_table["windows_with_upper_below_nominal"].eq(8).all() + ), + "model_or_encoding_selected": False, + "missingness_mechanism_identified": False, + "portfolio_claim_authorized": False, + "rows": missingness_table.to_dict(orient="records"), + }, + "fit_label_completion": { + "scope": "observed_only_plus_three_declared_fit_label_stress_rules", + "run_tag": str(fit_label_evidence.summary["run_tag"]), + "protocol_tag": str(fit_label_evidence.summary["protocol_tag"]), + "protocol_commit": str(fit_label_evidence.summary["protocol_commit"]), + **dict(fit_label_evidence.findings), + "scenario_or_result_selected": False, + "evaluation_outcomes_passed_to_fitting": False, + "scenarios_are_sharp_bounds_over_all_label_assignments": False, + "observed_only_active_replay": dict( + fit_label_evidence.summary["results"]["observed_only_active_replay"] + ), + "estimand_boundary": ( + "The observed-only fit and three declared stress rules vary 215 fit " + "labels that were unavailable at their information cutoffs. Every " + "scenario retains " + "all eight overall coverage upper bounds below 0.90, but the W7--W8 " + "stratum-2 crossing fails under the all-default scenario. Nonlinear " + "refitting means these scenarios are not sharp bounds over all 2^215 " + "label assignments." + ), + "rows": fit_label_table.to_dict(orient="records"), + }, + "allocation_granularity": { + "scope": "deterministic_usd25_floor_with_residual_cash", + "run_tag": str(granularity_evidence.summary["run_tag"]), + "protocol_tag": str(granularity_evidence.summary["protocol_tag"]), + "protocol_commit": str(granularity_evidence.summary["protocol_commit"]), + **dict(granularity_evidence.findings), + "scenario_or_result_selected": False, + "outcomes_passed_to_rounding": False, + "integer_policy_or_reoptimization_claim_authorized": False, + "estimand_boundary": ( + "This deterministic diagnostic floors each continuous exposure to a " + "USD 25 lot and holds the residual as cash. It supports numerical " + "adequacy of the continuous relaxation for this archive, not optimality " + "of an integer policy or robustness to other lot rules." + ), + "rows": granularity_table.to_dict(orient="records"), + }, + }, + "data_contract": { + "raw_rows": int(raw_audit["results"]["raw_rows"]), + "valid_loan_rows": int(raw_audit["results"]["valid_loan_rows"]), + "raw_schema_columns": int(raw_audit["results"]["raw_schema_columns"]), + "term36_rows_all_dates": int(raw_audit["results"]["term36_rows_all_dates"]), + "term60_rows_all_dates": int(raw_audit["results"]["term60_rows_all_dates"]), + "active_design_rows": int(raw_audit["results"]["term36_active_design_rows"]), + "eligible_raw_features": int(raw_audit["results"]["eligible_raw_features"]), + "late_schema_features": int(raw_audit["results"]["late_schema_features"]), + "declared_coverage_exceptions": int( + raw_audit["results"]["declared_coverage_exceptions"] + ), + "coverage_exceptions_requiring_sensitivity": int( + raw_audit["results"]["coverage_exceptions_requiring_sensitivity"] + ), + "coverage_exception_rows": raw_coverage_exceptions[ + [ + "feature", + "minimum_fitting_coverage", + "primary_oot_coverage", + "coverage_exception", + "missingness_semantics", + "requires_sensitivity", + ] + ].to_dict(orient="records"), + "primary_oot_funded_ratio": float(raw_audit["results"]["primary_oot_funded_ratio"]), + "primary_oot_requested_minus_funded_usd": float( + raw_audit["results"]["primary_oot_total_requested_minus_funded"] + ), + "sampling": "none_all_eligible_rows_within_each_declared_temporal_role", + "population_boundary": ( + "The active 640,543-row design is the exhaustive eligible 36-month " + "population for the declared dates, horizon, schema, and observability " + "rules; it is not a sample from the raw archive." + ), + "excluded_scope": ( + "Sixty-month contracts, immature issue dates, and late-schema fields " + "define different horizons, censoring regimes, or temporal feature support." + ), + "manifest": relative_artifact_descriptor(raw_audit_path, repo_root=ROOT), + }, + "credit_risk_controls": { + "scope": "coverage_only_five_model_temporal_transport_robustness", + "outcome_free_run_tag": str(credit_freeze["run_tag"]), + "verified_evaluation_run_tag": str(credit_summary["run_tag"]), + "all_five_all_eight_upper_below_nominal": bool( + credit_primary["windows_upper_below_0_90"].eq(8).all() + and credit_primary["coverage_upper_max"].lt(0.90).all() + ), + "learners_reported": list(CREDIT_LEARNER_ORDER), + "portfolio_learner": "catboost_platt", + "controls_enter_portfolio_optimization": False, + "model_or_feature_selected_from_oot": False, + "scorecard_superiority_claim_authorized": False, + "rows": credit_primary.to_dict(orient="records"), + "declared_descriptive_differences": dict(credit_summary["declared_diagnostics"]), + "endpoint_recovery_audit": credit_recovery, + "calibration": { + "all_primary_oot_mean_calibration_error_negative": bool( + credit_primary["mean_calibration_error"].lt(0.0).all() + ), + "all_primary_oot_slopes_below_one": bool( + credit_primary["calibration_slope"].lt(1.0).all() + ), + "optimizer_success_rows": int( + credit_prediction_metrics["calibration_optimizer_success"].sum() + ), + "optimizer_total_rows": int(len(credit_prediction_metrics)), + }, + "woe_iv": { + "optbinning_problems": int(len(credit_woe_summary)), + "all_optimal": bool(credit_woe_summary["status"].eq("OPTIMAL").all()), + "platform_features": int( + credit_woe_summary["learner"].eq("woe_scorecard_platform_platt").sum() + ), + "pricing_excluded_application_features": int( + credit_woe_summary["learner"].eq("woe_scorecard_borrower_platt").sum() + ), + "top_platform_iv": top_platform_iv, + "top_pricing_excluded_application_iv": top_borrower_iv, + }, + "temporal_shift": { + "primary_oot_score_psi": { + learner: float(primary_score_psi.loc[learner]) + for learner in CREDIT_LEARNER_ORDER + }, + "top_primary_oot_feature_psi": primary_feature_psi.head(5)[ + ["learner", "feature", "psi"] + ].to_dict(orient="records"), + "recent_chargeoff_early_role_variation": recent_chargeoff_variation.to_dict( + orient="records" + ), + }, + "interpretation": ( + "WOE/IV, a pricing-excluded application scorecard, and domain-safe " + "monotonic constraints are predeclared coverage-only specification " + "controls. They strengthen model-class robustness but do not define the " + "paper's novelty, select a learner, or authorize a portfolio policy." + ), + }, + "binary_phase_transition": { + "stratum": 2, + "w7_fit_prevalence": float(phase_w7["fit_prevalence"]), + "w8_fit_prevalence": float(phase_w8["fit_prevalence"]), + "w7_residual_quantile": float(phase_w7["fit_residual_quantile"]), + "w8_residual_quantile": float(phase_w8["fit_residual_quantile"]), + "w7_mean_width": float(phase_w7["mean_width"]), + "w8_mean_width": float(phase_w8["mean_width"]), + "w8_oot_coverage_bound": [ + float(phase_w8["coverage_lower"]), + float(phase_w8["coverage_upper"]), + ], + "label_lag_sensitivity": { + "admissible_lags_months": sorted( + int(value) for value in admissible_lag_table["charged_off_lag_months"].unique() + ), + "nonadmissible_lags_months": sorted( + int(value) + for value in nonadmissible_lag_table["charged_off_lag_months"].unique() + ), + "minimum_monthly_retention_by_lag": dict( + lag_evidence["results"]["minimum_monthly_retention_by_lag"] + ), + "locked_retention_threshold": 0.99, + "w7_to_w8_threshold_crossing_at_all_admissible_lags": bool( + lag_w7_w8.loc[ + lag_w7_w8["passes_locked_retention"] + & lag_w7_w8["window_id"].eq("w07_2012m07_m12"), + "phase_residual_quantile", + ] + .gt(0.5) + .all() + and lag_w7_w8.loc[ + lag_w7_w8["passes_locked_retention"] + & lag_w7_w8["window_id"].eq("w08_2012m08_2013m01"), + "phase_residual_quantile", + ] + .lt(0.5) + .all() + ), + "crossing_disappears_outside_locked_retention_scope": bool( + lag_w7_w8.loc[~lag_w7_w8["passes_locked_retention"], "phase_residual_quantile"] + .lt(0.5) + .all() + ), + "causal_interpretation_authorized": False, + "rows": lag_w7_w8.to_dict(orient="records"), + }, + "rows": phase_table.to_dict(orient="records"), + }, + "portfolio": { + "c2_cells": int(len(c2)), + "c2_match_residual_abs_max": float(c2["c2_match_residual"].abs().max()), + "c2_point_minus_guardrail_objective_min": float( + c2["point_minus_guardrail_objective"].min() + ), + "broad_stress_all_envelopes_cross_zero": bool( + broad["direction"].eq("crosses_zero").all() + ), + "broad_stress_cells": int(len(broad)), + "development_direction_counts": direction_table.to_dict(orient="records"), + "w8_development_all_envelopes_cross_zero": bool( + w8_development["direction"].eq("crosses_zero").all() + ), + "named_direction_counts": named_table.to_dict(orient="records"), + "evaluated_point_cap_solver_stability": { + "scope": "evaluated_point_caps_only_not_continuous_uniqueness", + "point_cap_rows": int(tie_census["rows"]), + "named_unique_cap_months": int(tie_census["named_unique_cap_months"]), + "minimum_absolute_nonbasic_reduced_cost": float( + tie_census["minimum_absolute_nonbasic_reduced_cost"] + ), + "near_zero_bases": int(tie_census["near_zero_bases"]), + "primal_degenerate_bases": int(tie_census["primal_degenerate_bases"]), + "reversed_order_reruns": int(tie_order["triggered_rows"]), + "tie_sensitive_rows": int(tie_order["tie_sensitive_rows"]), + "maximum_allocation_distance": float(tie_order["maximum_allocation_distance"]), + "maximum_absolute_objective_difference": float( + tie_order["maximum_absolute_objective_difference"] + ), + "continuous_frontier_uniqueness_claim": False, + }, + }, + "decision_challenger": { + "scope": "finite_two_ruler_three_interior_coordinate_diagnostic", + "continuous_frontier_claim": False, + "tracks_are_independent_replications": False, + "primary_ruler": "objective_matched", + "secondary_ruler": "normalized_score", + "endpoint_contrast": "gamma_1_minus_gamma_0", + "run_tag": two_ruler_lineage["evaluation"]["run_tag"], + "protocol_tag": two_ruler_lineage["evaluation"]["protocol_tag"], + "protocol_commit": two_ruler_lineage["evaluation"]["protocol_commit"], + "manifest": relative_artifact_descriptor(two_ruler_manifest_path, repo_root=ROOT), + "counts": dict(expected_two_ruler_counts), + "endpoint_recovery_audit": two_ruler_recovery, + "primary_oot_unresolved": int( + two_ruler_summary["outcomes"]["candidate_unresolved_by_role"]["primary_oot"] + ), + "metric_directions": dict(two_ruler_summary["metric_directions"]), + "objective_matched_coordinate_025_repetition": objective_quarter, + "rows": two_ruler_table.to_dict(orient="records"), + "interpretation": { + "coordinate_one_is_structural_null": True, + "objective_matched_equalizes_plugin_objective_floor": True, + "normalized_score_equalizes_relative_score_relaxation": True, + "normalized_score_equalizes_opportunity_cost": False, + "objective_matched_coordinate_025_is_one_repeated_allocation_contrast": True, + "preferred_gamma": None, + "preferred_ruler": None, + "preferred_coordinate": None, + "policy_winner": None, + "permitted_conclusion": ( + "Within the predeclared finite grid, the gamma endpoint allocation " + "contrast is not invariant to the outcome-free ruler or interior " + "coordinate." + ), + }, + }, + "audit_thesis": ( + "Binary absolute-residual conformal coverage does not transport to the later " + "archive under five declared credit-risk model specifications, recurs in the " + "only additional feasible origin, persists under three missing-value encodings, " + "and remains below nominal under four declared fit-label scenarios. A " + "prevalence-threshold crossing explains one observed geometry change but is " + "not invariant to every fit-label scenario. Portfolio direction is not identified " + "without outcome-free comparator support and is not invariant to the declared " + "ruler or interior coordinate; USD 25 floor rounding produces only negligible " + "rate perturbations in the evaluated archive." + ), + "source_artifacts": source_artifact_descriptors, + "paper_artifacts": paper_artifact_descriptors, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + evidence["claim_ledger"] = materialize_claim_ledger( + CLAIM_LEDGER_PATH, + evidence=evidence, + repo_root=ROOT, + ) + staged_manifest = staged_output_path(staging_root, EVIDENCE_PATH, repo_root=ROOT) + atomic_write_strict_json(staged_manifest, evidence) + promote_publication_generation( + publication_generation.outputs, + staged_manifest=staged_manifest, + manifest_target=EVIDENCE_PATH, + repo_root=ROOT, + transaction_root=staging_root, + ) + logger.info("Built one transactional active IJDS evidence generation: {}", EVIDENCE_PATH) + return EVIDENCE_PATH + + +def build_evidence() -> Path: + """Build and atomically promote one complete paper-facing generation.""" + staging_parent = ROOT / "reports/crpto" + staging_parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=".ijds-v4-generation-", dir=staging_parent) as staging: + return _build_evidence(Path(staging)) + + +if __name__ == "__main__": + build_evidence() diff --git a/scripts/build_ijds_calibration_selected_evidence.py b/scripts/build_ijds_calibration_selected_evidence.py deleted file mode 100644 index 685ef0e..0000000 --- a/scripts/build_ijds_calibration_selected_evidence.py +++ /dev/null @@ -1,532 +0,0 @@ -"""Build paper-facing evidence for the calibration-selected IJDS policy.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) - -from src.optimization.policy_selection import policy_eligibility_mask # noqa: E402 -from src.utils.script_helpers import load_json, write_json, write_table # noqa: E402 - -RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" -EXACT_ALPHA_TAG = "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" -MODEL_DIR = ROOT / "models/experiments/champion_reopen" / RUN_TAG / "portfolio" -DATA_DIR = ROOT / "data/processed/experiments/champion_reopen" / RUN_TAG / "portfolio" -EXACT_MODEL_DIR = ROOT / "models/experiments/champion_reopen" / EXACT_ALPHA_TAG / "conformal" -TABLE_DIR = ROOT / "reports/crpto/tables" - -TABLE_NAMES = { - "alpha": "crpto_tableA35_exact_alpha_grid", - "selector": "crpto_tableA36_calibration_policy_selector", - "temporal": "crpto_tableA37_calibration_selected_temporal_evaluation", - "grade": "crpto_tableA38_calibration_selected_grade_audit", - "bootstrap": "crpto_tableA39_calibration_selected_bootstrap", - "baseline": "crpto_tableA40_calibration_selected_point_baseline", -} -GOVERNANCE_NAME = "ijds_policy_governance.json" - - -def _weighted_average(frame: pd.DataFrame, column: str) -> float: - weights = frame["funded_weight"].to_numpy(dtype=float) - values = frame[column].to_numpy(dtype=float) - return float(np.sum(weights * values) / max(weights.sum(), 1e-12)) - - -def build_alpha_table(summary: dict[str, Any]) -> pd.DataFrame: - rows = pd.DataFrame(summary["alpha_summaries"]).copy() - rows.insert(0, "selected_for_policy", np.isclose(rows["target_alpha"], 0.10)) - return rows[ - [ - "selected_for_policy", - "target_alpha", - "used_alpha", - "target_coverage", - "empirical_coverage", - "coverage_gap", - "avg_width", - "min_partition_coverage", - "min_grade_coverage", - "high_endpoint_at_one_rate", - ] - ] - - -def build_selector_table( - grid: pd.DataFrame, - audit_grid: pd.DataFrame, - summary: dict[str, Any], -) -> pd.DataFrame: - selected_id = str(summary["selected_policy"]["candidate_id"]) - audit_selected_id = str(summary["calibration_audit"]["outcome_free_selected_candidate_id"]) - cap = float(summary["design"]["endpoint_budget_cap"]) - output = grid.copy() - output["eligible"] = policy_eligibility_mask( - output, - endpoint_budget_cap=cap, - budget=float(summary["design"]["budget"]), - min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), - ) - output["selected"] = output["candidate_id"].astype(str).eq(selected_id) - audit = audit_grid.copy() - audit["audit_eligible"] = policy_eligibility_mask( - audit, - endpoint_budget_cap=cap, - budget=float(summary["design"]["budget"]), - min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), - ) - audit["audit_selected"] = audit["candidate_id"].astype(str).eq(audit_selected_id) - audit = audit.rename( - columns={ - "expected_objective": "audit_expected_objective", - "n_funded": "audit_n_funded", - "endpoint_budget": "audit_endpoint_budget", - } - ) - output = output.merge( - audit[ - [ - "candidate_id", - "audit_selected", - "audit_eligible", - "audit_expected_objective", - "audit_n_funded", - "audit_endpoint_budget", - ] - ], - on="candidate_id", - how="left", - validate="one_to_one", - ) - return output[ - [ - "selected", - "eligible", - "audit_selected", - "audit_eligible", - "candidate_id", - "risk_tolerance", - "gamma", - "expected_objective", - "n_funded", - "weighted_pd_point", - "weighted_pd_effective", - "endpoint_budget", - "audit_expected_objective", - "audit_n_funded", - "audit_endpoint_budget", - ] - ].sort_values( - ["selected", "eligible", "expected_objective"], - ascending=[False, False, False], - kind="mergesort", - ) - - -def build_temporal_table(evaluation: pd.DataFrame) -> pd.DataFrame: - role_labels = { - "calibration_selected": "Calibration-selected 50/50 CRPTO", - "incumbent_linear": "More-conservative 75% blend", - "point_pd_matched_tau": "Point-PD matched-tau", - } - output = evaluation.copy() - output["policy"] = output["role"].map(role_labels) - return output[ - [ - "period", - "policy", - "risk_tolerance", - "gamma", - "n_funded", - "expected_objective", - "realized_return", - "weighted_outcome", - "weighted_miscoverage", - "weighted_pd_point", - "weighted_pd_effective", - "endpoint_budget", - "markov_loss_threshold", - ] - ] - - -def build_grade_table(allocations: pd.DataFrame) -> pd.DataFrame: - selected = allocations.loc[allocations["role"].eq("calibration_selected")].copy() - rows: list[dict[str, Any]] = [] - for grade, group in selected.groupby("grade", observed=True, sort=True): - rows.append( - { - "grade": str(grade), - "n_funded": int(len(group)), - "exposure": float(group["funded_exposure"].sum()), - "exposure_share": float(group["funded_weight"].sum()), - "weighted_default_rate": _weighted_average(group, "outcome"), - "weighted_miscoverage": _weighted_average(group, "miscoverage"), - "weighted_pd_point": _weighted_average(group, "pd_point"), - "weighted_pd_effective": _weighted_average(group, "pd_effective"), - "weighted_pd_high": _weighted_average(group, "pd_high"), - "realized_return": float(group["realized_return_contribution"].sum()), - } - ) - return pd.DataFrame(rows) - - -def _bootstrap_snapshot( - sample: pd.DataFrame, *, total_exposure: float, lgd: float -) -> dict[str, float]: - weights = sample["funded_exposure"].to_numpy(dtype=float) - weights = weights / max(float(weights.sum()), 1e-12) - outcome = sample["outcome"].to_numpy(dtype=float) - rates = sample["int_rate"].to_numpy(dtype=float) - point = sample["pd_point"].to_numpy(dtype=float) - high = sample["pd_high"].to_numpy(dtype=float) - realized_rate = np.where(outcome.astype(int) == 1, -float(lgd), rates) - return { - "realized_return": float(np.sum(weights * realized_rate) * total_exposure), - "weighted_default_rate": float(np.sum(weights * outcome)), - "weighted_miscoverage": float(np.sum(weights * sample["miscoverage"])), - "Gamma_CP": float(np.sum(weights * (high - point))), - "endpoint_budget": float(np.sum(weights * high)), - } - - -def _bootstrap_summary_rows( - draw_frame: pd.DataFrame, - observed: dict[str, float], - *, - bootstrap_unit: str, - n_units: int, - n_draws: int, - seed: int, - note: str, -) -> list[dict[str, Any]]: - return [ - { - "bootstrap_unit": bootstrap_unit, - "metric": metric, - "observed": observed[metric], - "boot_mean": float(draw_frame[metric].mean()), - "boot_p025": float(draw_frame[metric].quantile(0.025)), - "boot_p50": float(draw_frame[metric].quantile(0.50)), - "boot_p975": float(draw_frame[metric].quantile(0.975)), - "n_units": n_units, - "n_draws": n_draws, - "seed": seed, - "note": note, - } - for metric in draw_frame.columns - ] - - -def build_bootstrap_table( - allocations: pd.DataFrame, - evaluation: pd.DataFrame, - *, - n_draws: int = 5000, - seed: int = 20260709, - lgd: float = 0.45, -) -> pd.DataFrame: - selected = allocations.loc[allocations["role"].eq("calibration_selected")].reset_index( - drop=True - ) - total_exposure = float(selected["funded_exposure"].sum()) - allocation_observed = _bootstrap_snapshot( - selected, - total_exposure=total_exposure, - lgd=lgd, - ) - official = evaluation.loc[ - evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") - ].iloc[0] - observed = { - "realized_return": float(official["realized_return"]), - "weighted_default_rate": float(official["weighted_outcome"]), - "weighted_miscoverage": float(official["weighted_miscoverage"]), - "Gamma_CP": float(official["gamma_cp"]), - "endpoint_budget": float(official["endpoint_budget"]), - } - mismatches = [ - metric - for metric, value in observed.items() - if not np.isclose(allocation_observed[metric], value, rtol=1e-10, atol=1e-8) - ] - if mismatches: - raise ValueError( - "Funded allocations do not reconcile to the full-OOT evaluation: " - + ", ".join(mismatches) - ) - loan_rng = np.random.default_rng(seed) - loan_draws = [ - _bootstrap_snapshot( - selected.iloc[loan_rng.integers(0, len(selected), size=len(selected))].reset_index( - drop=True - ), - total_exposure=total_exposure, - lgd=lgd, - ) - for _ in range(n_draws) - ] - issue_month = pd.to_datetime(selected["issue_d"], errors="raise").dt.to_period("M") - month_groups = [ - selected.loc[issue_month.eq(month)].reset_index(drop=True) - for month in sorted(issue_month.unique()) - ] - month_rng = np.random.default_rng(seed) - month_draws = [ - _bootstrap_snapshot( - pd.concat( - [ - month_groups[index] - for index in month_rng.integers(0, len(month_groups), len(month_groups)) - ], - ignore_index=True, - ), - total_exposure=total_exposure, - lgd=lgd, - ) - for _ in range(n_draws) - ] - rows = _bootstrap_summary_rows( - pd.DataFrame(month_draws), - observed, - bootstrap_unit="origination_month", - n_units=len(month_groups), - n_draws=n_draws, - seed=seed, - note=( - "Fixed-allocation origination-month cluster bootstrap, renormalized to the " - "$1M budget; model, intervals, selector, and solver are not resampled." - ), - ) - rows.extend( - _bootstrap_summary_rows( - pd.DataFrame(loan_draws), - observed, - bootstrap_unit="funded_loan", - n_units=len(selected), - n_draws=n_draws, - seed=seed, - note=( - "Fixed funded-loan contribution bootstrap; model, intervals, selector, " - "and solver are not resampled." - ), - ) - ) - return pd.DataFrame(rows) - - -def build_baseline_table(evaluation: pd.DataFrame) -> pd.DataFrame: - full = evaluation.loc[evaluation["period"].eq("full_oot")].copy() - selected = full.loc[full["role"].eq("calibration_selected")].iloc[0] - labels = { - "calibration_selected": "Calibration-selected 50/50 CRPTO", - "incumbent_linear": "More-conservative 75% blend", - "point_pd_matched_tau": "Point-PD matched-tau", - } - full["policy"] = full["role"].map(labels) - full["return_delta_vs_selected"] = full["realized_return"] - float(selected["realized_return"]) - full["default_delta_vs_selected"] = full["weighted_outcome"] - float( - selected["weighted_outcome"] - ) - full["threshold_delta_vs_selected"] = full["markov_loss_threshold"] - float( - selected["markov_loss_threshold"] - ) - return full[ - [ - "policy", - "n_funded", - "expected_objective", - "realized_return", - "weighted_outcome", - "weighted_miscoverage", - "endpoint_budget", - "markov_loss_threshold", - "return_delta_vs_selected", - "default_delta_vs_selected", - "threshold_delta_vs_selected", - ] - ] - - -def _compact_calibration_audit(summary: dict[str, Any]) -> dict[str, Any]: - audit = summary["calibration_audit"] - rows = {str(row["role"]): row for row in audit["policy_evaluations"]} - - def _policy(role: str) -> dict[str, Any]: - row = rows[role] - return { - "n_funded": int(row["n_funded"]), - "realized_return": float(row["realized_return"]), - "weighted_default_rate": float(row["weighted_outcome"]), - "weighted_miscoverage": float(row["weighted_miscoverage"]), - "endpoint_budget": float(row["endpoint_budget"]), - "observed_accounting_bound": float( - row["endpoint_budget"] + row["weighted_miscoverage"] - ), - } - - return { - "period": audit["period"], - "n_candidates": int(summary["calibration_metadata"]["audit_rows"]), - "outcome_free_selected_candidate_id": audit["outcome_free_selected_candidate_id"], - "same_policy_selected": bool(audit["same_policy_selected"]), - "selected_policy": _policy("calibration_selected"), - "more_conservative_policy": _policy("incumbent_linear"), - "matched_point_pd": _policy("point_pd_matched_tau"), - "claim_boundary": audit["claim_boundary"], - } - - -def build_governance( - summary: dict[str, Any], - exact_summary: dict[str, Any], - evaluation: pd.DataFrame, - bootstrap: pd.DataFrame, - table_paths: dict[str, list[Path]], -) -> dict[str, Any]: - full = evaluation.loc[ - evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") - ].iloc[0] - point = evaluation.loc[ - evaluation["period"].eq("full_oot") & evaluation["role"].eq("point_pd_matched_tau") - ].iloc[0] - return_boot = bootstrap.loc[ - bootstrap["metric"].eq("realized_return") - & bootstrap["bootstrap_unit"].eq("origination_month") - ].iloc[0] - return { - "schema_version": "2026-07-09.7", - "generated_at_utc": summary["generated_at_utc"], - "run_tag": RUN_TAG, - "status": "active_ijds_policy", - "selection_protocol": { - **summary["selection_audit"], - "calibration_metadata": summary["calibration_metadata"], - "selector_forbidden_columns_present": summary["selector_forbidden_columns_present"], - "selector_input_columns": summary["selector_input_columns"], - "endpoint_cap_stability": summary["endpoint_cap_stability"], - "calibration_audit": _compact_calibration_audit(summary), - }, - "selected_policy": summary["selected_policy"], - "full_oot": { - "n_candidates": int(full["n_panel"]), - "n_funded": int(full["n_funded"]), - "total_allocated": float(full["total_allocated"]), - "expected_objective": float(full["expected_objective"]), - "realized_return": float(full["realized_return"]), - "weighted_default_rate": float(full["weighted_outcome"]), - "weighted_miscoverage": float(full["weighted_miscoverage"]), - "weighted_pd_point": float(full["weighted_pd_point"]), - "weighted_pd_effective": float(full["weighted_pd_effective"]), - "Gamma_CP": float(full["gamma_cp"]), - "Gamma_internalized": float(full["gamma_internalized"]), - "Gamma_residual": float(full["gamma_residual"]), - "endpoint_budget": float(full["endpoint_budget"]), - "markov_loss_threshold": float(full["markov_loss_threshold"]), - "observed_accounting_bound": float( - full["endpoint_budget"] + full["weighted_miscoverage"] - ), - "markov_tail_probability_bound": float(np.sqrt(summary["design"]["alpha"])), - }, - "point_pd_contrast": { - "realized_return": float(point["realized_return"]), - "weighted_default_rate": float(point["weighted_outcome"]), - "weighted_miscoverage": float(point["weighted_miscoverage"]), - "endpoint_budget": float(point["endpoint_budget"]), - "markov_loss_threshold": float(point["markov_loss_threshold"]), - "selected_return_cost": float(point["realized_return"] - full["realized_return"]), - "selected_return_cost_pct": float( - (point["realized_return"] - full["realized_return"]) / point["realized_return"] - ), - "selected_default_reduction": float( - point["weighted_outcome"] - full["weighted_outcome"] - ), - "selected_threshold_reduction": float( - point["markov_loss_threshold"] - full["markov_loss_threshold"] - ), - }, - "bootstrap_return_interval": { - "bootstrap_unit": "origination_month", - "p025": float(return_boot["boot_p025"]), - "p975": float(return_boot["boot_p975"]), - "n_units": int(return_boot["n_units"]), - "n_draws": int(return_boot["n_draws"]), - }, - "exact_alpha_reference_replay": exact_summary["reference_replay"], - "paper_tables": { - key: [str(path.relative_to(ROOT).as_posix()) for path in paths] - for key, paths in table_paths.items() - }, - "retired_active_claims": [ - "alpha01 intervals obtained by cross-family average-width scaling", - "8/8 approximate alpha-grid pass as a headline certificate", - "50,010-policy frontier as the active selector", - "0.345084 Markov threshold", - "capped_blended_uncertainty with delta_cap_quantile=0.975", - "OOT-outcome-selected portfolio hyperparameters", - "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger", - ], - "claim_boundary": summary["claim_boundary"], - } - - -def run(*, bootstrap_draws: int, bootstrap_seed: int) -> dict[str, Any]: - summary = load_json(MODEL_DIR / "calibration_selected_policy_summary.json") - exact_summary = load_json(EXACT_MODEL_DIR / "exact_alpha_grid_summary.json") - grid = pd.read_parquet(DATA_DIR / "calibration_policy_selection_grid.parquet") - audit_grid = pd.read_parquet(DATA_DIR / "calibration_policy_audit_grid.parquet") - evaluation = pd.read_csv(DATA_DIR / "calibration_selected_policy_oot_evaluation.csv") - allocations = pd.read_parquet( - DATA_DIR / "calibration_selected_policy_full_oot_allocations.parquet" - ) - tables = { - "alpha": build_alpha_table(exact_summary), - "selector": build_selector_table(grid, audit_grid, summary), - "temporal": build_temporal_table(evaluation), - "grade": build_grade_table(allocations), - "bootstrap": build_bootstrap_table( - allocations, - evaluation, - n_draws=bootstrap_draws, - seed=bootstrap_seed, - ), - "baseline": build_baseline_table(evaluation), - } - table_paths = { - key: write_table(TABLE_NAMES[key], frame, table_dir=TABLE_DIR, root=ROOT) - for key, frame in tables.items() - } - governance = build_governance( - summary, - exact_summary, - evaluation, - tables["bootstrap"], - table_paths, - ) - write_json(MODEL_DIR / GOVERNANCE_NAME, governance) - return governance - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--bootstrap-draws", type=int, default=5000) - parser.add_argument("--bootstrap-seed", type=int, default=20260709) - args = parser.parse_args() - payload = run( - bootstrap_draws=max(100, int(args.bootstrap_draws)), - bootstrap_seed=int(args.bootstrap_seed), - ) - print(json.dumps(payload["full_oot"], indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/build_ijds_policy_support_tie_evidence.py b/scripts/build_ijds_policy_support_tie_evidence.py new file mode 100644 index 0000000..af1ba19 --- /dev/null +++ b/scripts/build_ijds_policy_support_tie_evidence.py @@ -0,0 +1,309 @@ +"""Build tracked evidence from the locked policy-support and tie audit.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import pandas as pd + +from src.utils.isolated_experiment import relative_artifact_descriptor, write_csv_atomic +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_text + +ROOT = Path(__file__).resolve().parents[1] +RUN_TAG = "ijds-policy-support-tie-audit-2026-07-12-v1" +PROTOCOL_TAG = "protocol/ijds-policy-support-tie-audit-2026-07-12-v1" +PROTOCOL_COMMIT = "115eaf1b81ed5f05ffe601e9c20079059c40c200" +MODEL_DIR = ROOT / "models/experiments/ijds_audit" / RUN_TAG +DATA_DIR = ROOT / "data/processed/experiments/ijds_audit" / RUN_TAG +SOURCE_SUMMARY = MODEL_DIR / "policy_support_tie_audit_summary.json" +FAMILY_PATH = DATA_DIR / "policy_family_feasibility.parquet" +DIAGNOSTICS_PATH = DATA_DIR / "point_cap_basis_diagnostics.parquet" +SENSITIVITY_PATH = DATA_DIR / "point_cap_order_sensitivity.parquet" +EVIDENCE_PATH = ROOT / "reports/crpto/ijds_policy_support_tie_evidence.json" +TABLE_DIR = ROOT / "reports/crpto/tables" +MEMO_PATH = ROOT / "docs/research/ijds_policy_support_tie_results_2026-07-12.md" + + +def _json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError(f"Expected a JSON object at {path}.") + return payload + + +def _verify_descriptor(descriptor: Mapping[str, Any]) -> Path: + path = (ROOT / str(descriptor["path"])).resolve() + actual = relative_artifact_descriptor(path, repo_root=ROOT) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Policy-support source mismatch for {path}: {field}.") + return path + + +def _family_table(family: pd.DataFrame) -> pd.DataFrame: + table = ( + family.groupby( + ["role", "gamma", "risk_tolerance", "cap_classification"], + observed=True, + sort=True, + ) + .agg( + cells=("period", "size"), + binding_cells=("cap_binding", "sum"), + minimum_score=("minimum_feasible_score", "min"), + mean_score=("weighted_effective_score", "mean"), + maximum_score=("unconstrained_objective_score", "max"), + objective_mean=("expected_objective", "mean"), + ) + .reset_index() + ) + if int(table["cells"].sum()) != 3_120: + raise RuntimeError("Policy-family summary lost an endpoint cell.") + return table + + +def _endpoint_table(family: pd.DataFrame) -> pd.DataFrame: + keys = ["window_id", "role", "period", "risk_tolerance"] + gamma_075 = family.loc[family["gamma"].eq(0.75), [*keys, "expected_objective"]].rename( + columns={"expected_objective": "gamma_075_objective"} + ) + gamma_100 = family.loc[ + family["gamma"].eq(1.0), [*keys, "expected_objective", "cap_classification"] + ].rename( + columns={ + "expected_objective": "gamma_100_objective", + "cap_classification": "gamma_100_cap_classification", + } + ) + paired = gamma_075.merge(gamma_100, on=keys, validate="one_to_one") + paired["gamma_100_minus_075_objective"] = ( + paired["gamma_100_objective"] - paired["gamma_075_objective"] + ) + table = ( + paired.groupby(["role", "risk_tolerance"], observed=True, sort=True) + .agg( + cells=("period", "size"), + gamma_100_decision_active=( + "gamma_100_cap_classification", + lambda values: int(values.eq("decision_active").sum()), + ), + objective_difference_mean=("gamma_100_minus_075_objective", "mean"), + objective_difference_min=("gamma_100_minus_075_objective", "min"), + objective_difference_max=("gamma_100_minus_075_objective", "max"), + ) + .reset_index() + ) + if len(paired) != 624 or int(table["cells"].sum()) != 624: + raise RuntimeError("Gamma endpoint pairing lost a family cell.") + return table + + +def _support_table(diagnostics: pd.DataFrame) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for column in sorted(name for name in diagnostics if name.startswith("is_")): + selected = diagnostics.loc[diagnostics[column]] + counts = selected["cap_classification"].value_counts() + rows.append( + { + "source": column.removeprefix("is_"), + "cap_month_rows": int(len(selected)), + "decision_active": int(counts.get("decision_active", 0)), + "objective_boundary": int(counts.get("objective_boundary", 0)), + "objective_slack": int(counts.get("objective_slack", 0)), + "primal_degenerate_bases": int(selected["basis_primal_degenerate"].sum()), + "near_zero_bases": int(selected["near_zero_nonbasic_reduced_costs"].gt(0).sum()), + } + ) + return pd.DataFrame(rows) + + +def _memo(evidence: Mapping[str, Any]) -> str: + results = evidence["results"] + family = results["family"] + point = results["point_cap_census"] + order = results["order_sensitivity"] + return f"""# IJDS Policy-Support and Solver-Tie Audit Results + +## Status + +The outcome-free audit completed under `{PROTOCOL_TAG}` at commit +`{PROTOCOL_COMMIT[:7]}`. It read only ID, amount, contractual rate, purpose, +frozen design role, point score, and frozen conformal recipes. No outcome column +entered a solve. These results are pre-freeze structural evidence and do not by +themselves promote a policy or empirical direction. + +## Policy-family domain + +- The audit retained 3,120 cells: eight windows, 26 months, five gamma levels, + and three fixed risk tolerances. +- All 1,872 inherited interior cells were feasible; 1,846 were decision-active. + The 26 slack cells all occur at `gamma=.25` in W8. +- `gamma=0` is objective-slack in all 624 cells under + `tau={{.15,.17,.19}}`. It is correctly treated as a point-score nesting + control, not an uncertainty-aware policy. +- `gamma=1` is feasible and decision-active in all 624 cells. Relative to + `gamma=.75` on the same menu and cap, its plug-in objective is lower in all + 624 cells, by a mean of `{family["gamma_one_minus_075_objective_mean"]:.2f}` + and a range from `{family["gamma_one_minus_075_objective_min"]:.2f}` to + `{family["gamma_one_minus_075_objective_max"]:.2f}` plug-in objective dollars per + monthly USD 1 million budget. +- Parent V4 scores and objectives reconcile to + `{family["maximum_absolute_parent_score_difference"]:.3e}` and + `{family["maximum_absolute_parent_objective_difference"]:.3e}`. + +The endpoint result means the current nine-policy family is computationally +active but semantically incomplete. The next specification must either include +`gamma=1` as a complete-family sensitivity or replace fixed caps with a tagged +normalized-stringency design that includes both endpoints. Silent omission is +no longer defensible. + +## Comparator support + +The tolerance-deduplicated union contains `{point["rows"]}` cap-month pairs in +15 primary months. The earlier exploratory statement of 2,249 solves was not a +complete census; the correct named unique count is +`{point["named_unique_cap_months"]}` and the full union also includes support +endpoints and 2,952 period-specific basis breakpoints. + +- All 45 C0 cap-months are objective-slack for point PD. +- C1 has 1,079 active and one slack cap-month. +- C2 has 1,075 active and four objective-boundary cap-months. +- Every lower development endpoint is active; six upper endpoints are slack. +- Broad `.05` is active in every month, while broad `.12` is slack in every + month. `[.05,.12]` is therefore a stress interval spanning active and slack + regions, not a normative admissible support. + +## Solver ties + +There are `{point["primal_degenerate_bases"]}` primal-degenerate bases, mostly +because basis breakpoints are transition points. None has a nonbasic reduced +cost within `1e-7` of zero; the minimum absolute nonbasic reduced cost is +`{point["minimum_absolute_nonbasic_reduced_cost"]:.6g}`. All +`{order["triggered_rows"]}` triggered caps were rerun after reversing loan-ID +order. Zero were tie-sensitive; maximum exposure distance was +`{order["maximum_allocation_distance"]:.3e}` and maximum absolute objective +difference `{order["maximum_absolute_objective_difference"]:.3e}`. + +Thus primal degeneracy does not explain the portfolio directions in this finite +census. This supports deterministic stability at the evaluated caps, not a +universal uniqueness theorem over every real cap. + +## Required next decision + +Do not freeze the current family. The highest-value challenger is an +outcome-free normalized stringency parameter +`lambda=(q_cap-q_min)/(q_obj-q_min)` over both score endpoints. It directly +addresses the all-slack point endpoint and the arbitrary cross-score meaning of +one numeric tau. It must be separately tagged and reported whether it strengthens +or weakens the V4 conclusion. A simpler fallback is to add `gamma=1` to the +fixed-cap sensitivity and retain the exact support caveat. +""" + + +def build() -> Path: + """Verify immutable sources and emit compact tracked evidence.""" + summary = _json(SOURCE_SUMMARY) + if summary.get("status") != "complete": + raise RuntimeError("Policy-support source run is incomplete.") + if summary.get("protocol_tag") != PROTOCOL_TAG: + raise RuntimeError("Policy-support protocol tag mismatch.") + if summary.get("protocol_commit") != PROTOCOL_COMMIT: + raise RuntimeError("Policy-support protocol commit mismatch.") + if summary.get("outcome_columns_passed") != []: + raise RuntimeError("Policy-support source reports an outcome column.") + for descriptor in summary["artifacts"].values(): + _verify_descriptor(descriptor) + + family = pd.read_parquet(FAMILY_PATH) + diagnostics = pd.read_parquet(DIAGNOSTICS_PATH) + sensitivity = pd.read_parquet(SENSITIVITY_PATH) + if len(family) != 3_120 or len(diagnostics) != 7_297 or len(sensitivity) != 2_941: + raise RuntimeError("Policy-support source cardinality changed.") + family_table = _family_table(family) + endpoint_table = _endpoint_table(family) + support_table = _support_table(diagnostics) + table_paths = { + "family": write_csv_atomic(family_table, TABLE_DIR / "crpto_ijds_policy_family_domain.csv"), + "gamma_endpoint": write_csv_atomic( + endpoint_table, TABLE_DIR / "crpto_ijds_gamma_endpoint_audit.csv" + ), + "comparator_support": write_csv_atomic( + support_table, TABLE_DIR / "crpto_ijds_comparator_support_domain.csv" + ), + } + + gamma_pair = family.loc[ + family["gamma"].eq(1.0), + [ + "window_id", + "role", + "period", + "risk_tolerance", + "expected_objective", + ], + ].merge( + family.loc[ + family["gamma"].eq(0.75), + [ + "window_id", + "role", + "period", + "risk_tolerance", + "expected_objective", + ], + ], + on=["window_id", "role", "period", "risk_tolerance"], + suffixes=("_100", "_075"), + validate="one_to_one", + ) + gamma_difference = gamma_pair["expected_objective_100"] - gamma_pair["expected_objective_075"] + source_results = summary["results"] + results = { + "family": { + **source_results["family"], + "gamma_zero_objective_slack": int( + family.loc[family["gamma"].eq(0.0), "cap_classification"] + .eq("objective_slack") + .sum() + ), + "inherited_slack": int( + family.loc[family["gamma"].isin([0.25, 0.5, 0.75]), "cap_classification"] + .eq("objective_slack") + .sum() + ), + "gamma_one_minus_075_objective_mean": float(gamma_difference.mean()), + "gamma_one_minus_075_objective_min": float(gamma_difference.min()), + "gamma_one_minus_075_objective_max": float(gamma_difference.max()), + }, + "point_cap_census": source_results["point_cap_census"], + "order_sensitivity": source_results["order_sensitivity"], + } + evidence = { + "schema_version": "2026-07-12.1", + "status": "complete_prefreeze_structural_evidence", + "active_claim_status": "not_active_until_family_redesign_decision", + "run_tag": RUN_TAG, + "protocol_tag": PROTOCOL_TAG, + "protocol_commit": PROTOCOL_COMMIT, + "claim_boundary": summary["claim_boundary"], + "outcome_columns_passed": [], + "results": results, + "source_summary": relative_artifact_descriptor(SOURCE_SUMMARY, repo_root=ROOT), + "source_artifacts": summary["artifacts"], + "tables": { + name: relative_artifact_descriptor(path, repo_root=ROOT) + for name, path in table_paths.items() + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + atomic_write_json(EVIDENCE_PATH, evidence) + atomic_write_text(MEMO_PATH, _memo(evidence)) + return EVIDENCE_PATH + + +if __name__ == "__main__": + build() diff --git a/scripts/build_ijds_submission_tex.py b/scripts/build_ijds_submission_tex.py new file mode 100644 index 0000000..b79cdc0 --- /dev/null +++ b/scripts/build_ijds_submission_tex.py @@ -0,0 +1,84 @@ +"""Render the canonical QMD manuscript into the anonymous INFORMS TeX file.""" + +from __future__ import annotations + +import argparse +import hashlib +import subprocess +import tempfile +from pathlib import Path + +from src.utils.pipeline_runtime import atomic_write_text + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "paper" / "CRPTO_ijds.qmd" +TEMPLATE = ROOT / "paper" / "submission" / "informs-pandoc-template.tex" +OUTPUT = ROOT / "paper" / "submission" / "CRPTO_ijds_submission.tex" + +GENERATED_BANNER = """%% ===================================================================== +%% GENERATED FROM paper/CRPTO_ijds.qmd. DO NOT EDIT THIS FILE DIRECTLY. +%% Build: uv run python scripts/build_ijds_submission_tex.py +%% Compile: latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +%% Windows fallback: pdflatex, bibtex, pdflatex, pdflatex. +%% ===================================================================== +""" + + +def _sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def render_submission_tex(*, check: bool = False) -> bool: + """Render the source and return whether the checked output was current.""" + with tempfile.TemporaryDirectory(prefix=".crpto-ijds-", dir=SOURCE.parent) as temporary: + rendered = Path(temporary) / SOURCE.with_suffix(".tex").name + command = [ + "quarto", + "render", + str(SOURCE), + "--to", + "latex", + "--metadata", + "cite-method:natbib", + "--template", + str(TEMPLATE), + "--output-dir", + rendered.parent.name, + ] + subprocess.run(command, cwd=ROOT, check=True) + body = rendered.read_text(encoding="utf-8") + + # The source is one directory above the official output. Quarto preserves + # source-relative image paths, so adjust them deterministically for TeX. + body = body.replace("{../reports/", "{../../reports/") + payload = (GENERATED_BANNER + body).encode("utf-8") + + if check: + if not OUTPUT.exists(): + return False + return _sha256(OUTPUT.read_bytes()) == _sha256(payload) + + atomic_write_text(OUTPUT, payload.decode("utf-8")) + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="fail when the official TeX differs from the generated manuscript", + ) + args = parser.parse_args() + current = render_submission_tex(check=args.check) + if args.check and not current: + print(f"stale generated manuscript: {OUTPUT.relative_to(ROOT)}") + return 1 + print( + f"submission TeX {'is current' if args.check else 'rendered'}: {OUTPUT.relative_to(ROOT)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_multidataset_external_replication.py b/scripts/build_multidataset_external_replication.py deleted file mode 100644 index 82a9b1d..0000000 --- a/scripts/build_multidataset_external_replication.py +++ /dev/null @@ -1,604 +0,0 @@ -"""Build curated multidataset external-replication tables and figures. - -This script reads only local, curated CSV inputs under -``reports/crpto/multidataset/source``. It does not point back to the exploratory -laboratory, does not use credentials, and does not touch the Lending Club -champion artifacts. - -Usage: - uv run python scripts/build_multidataset_external_replication.py -""" - -from __future__ import annotations - -import shutil -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import matplotlib -import matplotlib.pyplot as plt -import matplotlib.ticker as mticker -import pandas as pd -from loguru import logger - -from src.utils.script_helpers import write_json, write_table - -matplotlib.use("Agg") - -ROOT = Path(__file__).resolve().parents[1] -SOURCE_DIR = ROOT / "reports" / "crpto" / "multidataset" / "source" -TABLE_DIR = ROOT / "reports" / "crpto" / "tables" -FIGURE_DIR = ROOT / "reports" / "crpto" / "figures" -BOOK_FIGURE_DIR = ROOT / "book" / "assets" / "figures" / "publication" -STATUS_PATH = ROOT / "models" / "crpto_multidataset_external_status.json" - -TABLE_A25 = "crpto_tableA25_external_replication_gate" -TABLE_A26 = "crpto_tableA26_external_candidate_sensitivity" -TABLE_A27 = "crpto_tableA27_freddie_horizon_sensitivity" -TABLE_A28 = "crpto_tableA28_external_lp_exhaustiveness" -TABLE_A29 = "crpto_tableA29_freddie_mondrian_sparse_group_audit" -TABLE_A30 = "crpto_tableA30_external_metric_intervals" -TABLE_A31 = "crpto_tableA31_external_subperiod_metrics" -TABLE_A32 = "crpto_tableA32_prosper_default_definition_sensitivity" -TABLE_A33 = "crpto_tableA33_freddie_segment_sensitivity" -FIG22 = "crpto_fig22_external_replication" -FIG23 = "crpto_fig23_external_candidate_sensitivity" -FIG24 = "crpto_fig24_freddie_all_candidate_certificate" - -PALETTE = { - "blue": "#0072B2", - "orange": "#E69F00", - "green": "#009E73", - "red": "#D55E00", - "purple": "#CC79A7", - "gray": "#666666", - "light_gray": "#DDDDDD", -} - - -def _rel(path: Path) -> str: - return path.relative_to(ROOT).as_posix() - - -plt.rcParams.update( - { - # TrueType (Type 42) keeps PDF text selectable/extractable and avoids - # Type 3 subset glyphs that publisher font checkers flag. - "pdf.fonttype": 42, - "ps.fonttype": 42, - "font.family": "serif", - "font.serif": ["Times New Roman", "DejaVu Serif", "serif"], - "font.size": 9, - "axes.titlesize": 10, - "axes.labelsize": 9, - "xtick.labelsize": 8, - "ytick.labelsize": 8, - "legend.fontsize": 8, - "figure.dpi": 150, - "savefig.dpi": 300, - "savefig.bbox": "tight", - "savefig.pad_inches": 0.05, - "axes.spines.top": False, - "axes.spines.right": False, - "axes.grid": True, - "grid.alpha": 0.25, - "grid.linestyle": "--", - "lines.linewidth": 1.6, - } -) - - -def _read_source(name: str) -> pd.DataFrame: - path = SOURCE_DIR / name - if not path.exists(): - raise FileNotFoundError(f"Missing curated multidataset source: {path}") - return pd.read_csv(path) - - -def _save_figure(fig: plt.Figure, name: str) -> list[Path]: - FIGURE_DIR.mkdir(parents=True, exist_ok=True) - BOOK_FIGURE_DIR.mkdir(parents=True, exist_ok=True) - written: list[Path] = [] - for ext in ("png", "pdf"): - path = FIGURE_DIR / f"{name}.{ext}" - fig.savefig(path) - shutil.copy2(path, BOOK_FIGURE_DIR / path.name) - logger.info("Wrote {}", path.relative_to(ROOT)) - written.append(path) - plt.close(fig) - return written - - -def _label_dataset(name: str) -> str: - if "Prosper" in name: - return "Prosper" - if "Freddie" in name: - return "Freddie FM48" - return name - - -def _candidate_cap_to_number(value: Any, available: int) -> int: - if str(value).lower() == "all": - return int(available) - return int(value) - - -def _build_external_gate_table(main: pd.DataFrame) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for _, row in main.iterrows(): - dataset = _label_dataset(str(row["dataset"])) - rows.append( - { - "dataset": dataset, - "credit_product": ( - "Marketplace personal loans" - if dataset == "Prosper" - else "Single-family mortgages" - ), - "external_role": "main external economic replication", - "n_rows": int(row["rows_total"]), - "default_rate": float(row["default_rate"]), - "auc_roc": float(row["auc_roc"]), - "pr_auc": float(row["pr_auc"]), - "brier": float(row["brier"]), - "coverage_90": float(row["coverage_90"]), - "min_group_coverage_90": float(row["min_group_coverage_90"]), - "alpha01_coverage": float(row["coverage_alpha01"]), - "oot_candidates": int(row["available_oot_candidates"]), - "lp_candidate_cap": str(row["lp_candidate_cap"]), - "robust_objective": float(row["robust_objective_best"]), - "price_of_robustness_pct": float(row["price_of_robustness_pct"]), - "gate": "pass", - } - ) - return pd.DataFrame(rows) - - -def _build_candidate_table(compact: pd.DataFrame) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for _, row in compact.iterrows(): - dataset = _label_dataset(str(row["dataset"])) - rows.append( - { - "dataset": dataset, - "screen": row["strategy"], - "candidate_cap": str(row["candidate_cap"]), - "available_candidates": int(row["available_candidates"]), - "runs": int(row["runs"]), - "robust_objective_mean": float(row["robust_objective_mean"]), - "robust_objective_min": float(row["robust_objective_min"]), - "robust_objective_max": float(row["robust_objective_max"]), - "nonrobust_objective_mean": float(row["nonrobust_objective_mean"]), - "robust_n_funded_mean": float(row["robust_n_funded_mean"]), - } - ) - return pd.DataFrame(rows) - - -def _build_freddie_horizon_table(freddie: pd.DataFrame) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for _, row in freddie.iterrows(): - rows.append( - { - "variant": row["run_label"], - "color_group": row["variant"], - "window_months": int(row["window_months"]), - "n_rows": int(row["rows_total"]), - "default_rate": float(row["default_rate"]), - "auc_roc": float(row["auc_roc"]), - "pr_auc": float(row["pr_auc"]), - "coverage_90": float(row["coverage_90"]), - "alpha01_coverage": float(row["coverage_alpha01"]), - "coverage90_pass": bool(row["coverage90_pass"]), - "alpha01_pass": bool(row["alpha01_pass"]), - "screening_robust_objective": float(row["robust_objective"]), - "recommendation": row["recommendation"], - } - ) - return pd.DataFrame(rows).sort_values(["window_months", "color_group"]) - - -def _build_lp_exhaustiveness_table(lp: pd.DataFrame) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for _, row in lp.iterrows(): - dataset = "Prosper" if row["dataset"] == "prosper" else "Freddie FM48" - rows.append( - { - "dataset": dataset, - "candidate_cap": str(row["candidate_cap"]), - "available_candidates": int(row["available_candidates"]), - "n_candidates": int(row["n_candidates"]), - "robust_objective": float(row["robust_objective"]), - "nonrobust_objective": float(row["nonrobust_objective"]), - "price_of_robustness_pct": float(row["price_of_robustness_pct"]), - "robust_objective_boot_low": float(row["robust_objective_boot_low"]), - "robust_objective_boot_high": float(row["robust_objective_boot_high"]), - "robust_n_funded": int(row["robust_n_funded"]), - "robust_max_funded_rank": int(row["robust_max_funded_rank"]), - "solver_success": bool(row["robust_solver_success"]) - and bool(row["nonrobust_solver_success"]), - } - ) - return pd.DataFrame(rows) - - -def _build_mondrian_sparse_table(summary: pd.DataFrame) -> pd.DataFrame: - return summary[ - ["scope", "n_groups", "min_coverage_90", "min_coverage_alpha01", "test_rows"] - ].copy() - - -def _build_metric_intervals_table(intervals: pd.DataFrame) -> pd.DataFrame: - return intervals[["dataset", "metric", "estimate", "ci_low", "ci_high", "method", "n"]].copy() - - -def _build_subperiod_table(subperiods: pd.DataFrame) -> pd.DataFrame: - return subperiods[ - [ - "dataset", - "period", - "n", - "defaults", - "default_rate", - "auc_roc", - "pr_auc", - "coverage_90", - "coverage_alpha01", - "economic_candidates", - ] - ].copy() - - -def _build_prosper_default_table(prosper_defaults: pd.DataFrame) -> pd.DataFrame: - return prosper_defaults[ - [ - "variant", - "rows_total", - "default_rate", - "auc_roc", - "coverage_90", - "min_group_coverage_90", - "alpha01_coverage", - "coverage90_pass", - "alpha01_pass", - "available_oot_candidates", - "robust_objective_all_candidates", - "price_of_robustness_pct", - ] - ].copy() - - -def _build_freddie_segment_table(segments: pd.DataFrame) -> pd.DataFrame: - return segments[ - [ - "segment", - "rows_total", - "default_rate", - "auc_roc", - "coverage_90", - "min_group_coverage_90", - "alpha01_coverage", - "coverage90_pass", - "alpha01_pass", - "available_oot_candidates", - "robust_objective_all_candidates", - "price_of_robustness_pct", - "all_lp_solved", - "max_funded_rank", - "funded_outside_top250k", - ] - ].copy() - - -def _plot_external_replication(table: pd.DataFrame) -> list[Path]: - fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.4)) - labels = table["dataset"].tolist() - colors = [PALETTE["orange"], PALETTE["blue"]] - - ax = axes[0] - x = range(len(table)) - ax.bar( - [i - 0.18 for i in x], table["coverage_90"], width=0.34, label="90% coverage", color=colors - ) - ax.bar( - [i + 0.18 for i in x], - table["alpha01_coverage"], - width=0.34, - label="alpha 0.01 coverage", - color=[PALETTE["green"], PALETTE["purple"]], - ) - ax.axhline(0.90, color=PALETTE["red"], linestyle="--", linewidth=1.0, label="90% target") - ax.axhline(0.99, color=PALETTE["gray"], linestyle=":", linewidth=1.0, label="99% target") - ax.set_ylim(0.86, 1.01) - ax.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1.0, decimals=0)) - ax.set_xticks(list(x), labels) - ax.set_ylabel("Empirical coverage") - ax.set_title("Conformal gates") - - ax = axes[1] - objective_m = table["robust_objective"] / 1_000_000 - bars = ax.bar(labels, objective_m, color=colors) - for bar, candidates in zip(bars, table["oot_candidates"], strict=False): - ax.annotate( - f"{candidates:,.0f} OOT", - xy=(bar.get_x() + bar.get_width() / 2, bar.get_height()), - xytext=(0, 4), - textcoords="offset points", - ha="center", - va="bottom", - fontsize=7, - ) - ax.set_ylabel("Robust LP objective (millions)") - ax.set_title("Economic replication") - ax.yaxis.set_major_formatter(mticker.StrMethodFormatter("${x:.1f}M")) - - fig.suptitle("External CRPTO replications preserve conformal gates and positive LP value") - handles, legend_labels = axes[0].get_legend_handles_labels() - fig.legend( - handles, - legend_labels, - loc="lower center", - bbox_to_anchor=(0.5, 0.02), - ncol=4, - frameon=True, - ) - fig.tight_layout(rect=(0, 0.18, 1, 0.93)) - return _save_figure(fig, FIG22) - - -def _plot_candidate_sensitivity(candidate: pd.DataFrame) -> list[Path]: - fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.0), sharey=False) - for ax, dataset in zip(axes, ["Prosper", "Freddie FM48"], strict=True): - subset = candidate.loc[candidate["dataset"] == dataset].copy() - available = int(subset["available_candidates"].max()) - subset["cap_n"] = subset["candidate_cap"].map( - lambda value, available=available: _candidate_cap_to_number(value, available) - ) - for strategy, color, marker in [ - ("top_net_return", PALETTE["blue"], "o"), - ("random", PALETTE["orange"], "s"), - ]: - s = subset.loc[subset["screen"] == strategy].sort_values("cap_n") - if s.empty: - continue - ax.plot( - s["cap_n"], - s["robust_objective_mean"], - marker=marker, - color=color, - label=strategy.replace("_", " "), - ) - if strategy == "random": - ax.fill_between( - s["cap_n"].to_numpy(dtype=float), - s["robust_objective_min"].to_numpy(dtype=float), - s["robust_objective_max"].to_numpy(dtype=float), - color=color, - alpha=0.15, - linewidth=0, - ) - ax.set_xscale("log") - ax.set_title(dataset) - ax.set_xlabel("Candidate cap") - ax.set_ylabel("Robust objective") - ax.yaxis.set_major_formatter( - mticker.FuncFormatter(lambda value, _: f"${value / 1_000:.0f}K") - ) - ax.legend(loc="lower right", frameon=True) - fig.suptitle("Portfolio value stabilizes as candidate pools expand") - fig.tight_layout() - return _save_figure(fig, FIG23) - - -def _plot_freddie_all_candidate_certificate(lp: pd.DataFrame) -> list[Path]: - freddie = lp.loc[lp["dataset"] == "Freddie FM48"].copy() - freddie["cap_n"] = freddie["candidate_cap"].map( - lambda value: _candidate_cap_to_number(value, int(freddie["available_candidates"].max())) - ) - freddie = freddie.sort_values("cap_n") - - fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.0)) - ax = axes[0] - ax.plot( - freddie["cap_n"], - freddie["robust_objective"] / 1_000_000, - marker="o", - color=PALETTE["blue"], - label="Robust", - ) - ax.plot( - freddie["cap_n"], - freddie["nonrobust_objective"] / 1_000_000, - marker="s", - color=PALETTE["orange"], - label="Nonrobust", - ) - ax.set_xscale("log") - ax.set_xlabel("Freddie candidate cap") - ax.set_ylabel("Objective (millions)") - ax.yaxis.set_major_formatter(mticker.StrMethodFormatter("${x:.3f}M")) - ax.set_title("Objective is unchanged") - ax.legend(frameon=True, loc="lower right") - - ax = axes[1] - all_row = freddie.loc[freddie["candidate_cap"].astype(str).str.lower() == "all"].iloc[0] - bars = ax.bar( - ["All OOT candidates", "Worst funded rank"], - [all_row["available_candidates"], all_row["robust_max_funded_rank"]], - color=[PALETTE["gray"], PALETTE["green"]], - ) - ax.set_yscale("log") - ax.set_ylim(300, float(all_row["available_candidates"]) * 3.0) - ax.set_ylabel("Count / rank (log scale)") - ax.set_title("All-candidate audit") - for bar in bars: - value = bar.get_height() - ax.annotate( - f"{value:,.0f}", - xy=(bar.get_x() + bar.get_width() / 2, value), - xytext=(0, 4), - textcoords="offset points", - ha="center", - va="bottom", - fontsize=8, - ) - - fig.suptitle("Freddie FM48 all-candidate LP validates the top-screen result") - fig.tight_layout(rect=(0, 0, 1, 0.92)) - return _save_figure(fig, FIG24) - - -def _write_source_log() -> Path: - SOURCE_DIR.mkdir(parents=True, exist_ok=True) - source_log = SOURCE_DIR / "source_log.md" - source_log.write_text( - "\n".join( - [ - "# Multidataset External Replication Source Log", - "", - "This folder contains only curated, local CSV summaries used by the paper and book.", - "It intentionally does not contain credentials, browser-session references, or paths to the exploratory laboratory.", - "", - "## Public Dataset Sources", - "", - "- Prosper loan-level data access documentation: https://help.prosper.com/hc/en-us/articles/210013083-Where-can-I-download-data-about-loans-through-Prosper", - "- Freddie Mac Single Family Loan-Level Dataset: https://www.freddiemac.com/research/datasets/sf-loanlevel-dataset", - "- Freddie/Mendeley processed mortgage windows: https://data.mendeley.com/datasets/bzr2rxttvz/3", - "- Home Credit Default Risk page, archived only and not used in the main external claim: https://www.kaggle.com/competitions/home-credit-default-risk/data", - "", - "## Editorial Decision", - "", - "Prosper final-status loans and Freddie FM48 are reported as external economic replications. Home Credit is discarded from the IJDS main claim because it lacks a clean investment-return and exposure contract comparable to Lending Club, Prosper, and Freddie.", - "", - "## Extended Audit Layer", - "", - "A28 solves the Freddie FM48 LP on the full OOT candidate universe and documents that the all-candidate optimum funds only loans inside the top-return screen. A29 isolates sparse Mondrian groups. A30--A33 report confidence intervals, OOT subperiods, Prosper default-definition sensitivity, and Freddie red/green segment sensitivity.", - "", - ] - ), - encoding="utf-8", - newline="\n", - ) - logger.info("Wrote {}", source_log.relative_to(ROOT)) - return source_log - - -def build_multidataset_external_replication() -> dict[str, Any]: - start = datetime.now(tz=UTC) - main = _read_source("table_main_external_replications.csv") - compact = _read_source("table_portfolio_candidate_sensitivity_compact.csv") - freddie = _read_source("table_freddie_window_sensitivity.csv") - full_gate = _read_source("table_full_multidataset_gate_audit.csv") - lp_exhaustiveness_source = _read_source("table_external_lp_exhaustiveness.csv") - mondrian_sparse_source = _read_source("table_freddie_mondrian_eligible_summary.csv") - metric_intervals_source = _read_source("table_external_metric_intervals.csv") - subperiod_source = _read_source("table_external_subperiod_metrics.csv") - prosper_default_source = _read_source("table_prosper_default_definition_sensitivity.csv") - freddie_segment_source = _read_source("table_freddie_segment_sensitivity.csv") - - external_gate = _build_external_gate_table(main) - candidate = _build_candidate_table(compact) - freddie_horizon = _build_freddie_horizon_table(freddie) - lp_exhaustiveness = _build_lp_exhaustiveness_table(lp_exhaustiveness_source) - freddie_all = lp_exhaustiveness.loc[ - (lp_exhaustiveness["dataset"] == "Freddie FM48") - & (lp_exhaustiveness["candidate_cap"].astype(str).str.lower() == "all") - & (lp_exhaustiveness["solver_success"]) - ] - if not freddie_all.empty: - all_row = freddie_all.iloc[0] - freddie_mask = external_gate["dataset"] == "Freddie FM48" - external_gate.loc[freddie_mask, "lp_candidate_cap"] = "all" - external_gate.loc[freddie_mask, "robust_objective"] = float(all_row["robust_objective"]) - external_gate.loc[freddie_mask, "price_of_robustness_pct"] = float( - all_row["price_of_robustness_pct"] - ) - mondrian_sparse = _build_mondrian_sparse_table(mondrian_sparse_source) - metric_intervals = _build_metric_intervals_table(metric_intervals_source) - subperiods = _build_subperiod_table(subperiod_source) - prosper_default = _build_prosper_default_table(prosper_default_source) - freddie_segment = _build_freddie_segment_table(freddie_segment_source) - - outputs: list[Path] = [] - outputs.extend( - write_table(TABLE_A25, external_gate, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A26, candidate, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A27, freddie_horizon, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A28, lp_exhaustiveness, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A29, mondrian_sparse, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A30, metric_intervals, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A31, subperiods, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A32, prosper_default, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend( - write_table(TABLE_A33, freddie_segment, table_dir=TABLE_DIR, root=ROOT, float_precision=4) - ) - outputs.extend(_plot_external_replication(external_gate)) - outputs.extend(_plot_candidate_sensitivity(candidate)) - outputs.extend(_plot_freddie_all_candidate_certificate(lp_exhaustiveness)) - outputs.append(_write_source_log()) - - home_credit = full_gate.loc[full_gate["run_label"] == "home_credit_full"].iloc[0] - status = { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "duration_seconds": (datetime.now(tz=UTC) - start).total_seconds(), - "source_dir": _rel(SOURCE_DIR), - "home_credit_policy": "discarded_from_main_claim_archived_only", - "home_credit_archive_metrics": { - "rows_total": int(home_credit["rows_total"]), - "auc_roc": float(home_credit["auc_roc"]), - "coverage_90": float(home_credit["coverage_90"]), - "alpha01_coverage": float(home_credit["coverage_alpha01"]), - }, - "external_replications": external_gate.to_dict(orient="records"), - "freddie_selected_variant": "freddie_fm48_both", - "freddie_all_candidate_lp": { - "available_candidates": int( - lp_exhaustiveness.loc[ - (lp_exhaustiveness["dataset"] == "Freddie FM48") - & (lp_exhaustiveness["candidate_cap"].astype(str).str.lower() == "all"), - "available_candidates", - ].iloc[0] - ), - "robust_objective": float( - lp_exhaustiveness.loc[ - (lp_exhaustiveness["dataset"] == "Freddie FM48") - & (lp_exhaustiveness["candidate_cap"].astype(str).str.lower() == "all"), - "robust_objective", - ].iloc[0] - ), - "max_funded_rank": int( - lp_exhaustiveness.loc[ - (lp_exhaustiveness["dataset"] == "Freddie FM48") - & (lp_exhaustiveness["candidate_cap"].astype(str).str.lower() == "all"), - "robust_max_funded_rank", - ].iloc[0] - ), - }, - "freddie_top_screen_stable_through_candidates": "all", - "prosper_default_definition_sensitivity": prosper_default.to_dict(orient="records"), - "freddie_segment_sensitivity": freddie_segment.to_dict(orient="records"), - "outputs": [_rel(path) for path in outputs], - } - write_json(STATUS_PATH, status) - logger.info("Wrote {}", STATUS_PATH.relative_to(ROOT)) - return status - - -if __name__ == "__main__": - build_multidataset_external_replication() diff --git a/scripts/build_papers_tesis_deep_audit.py b/scripts/build_papers_tesis_deep_audit.py deleted file mode 100644 index ebed162..0000000 --- a/scripts/build_papers_tesis_deep_audit.py +++ /dev/null @@ -1,2040 +0,0 @@ -#!/usr/bin/env python3 -"""Build the June 2026 Papers_tesis deep audit artifacts. - -This is intentionally a claim-governed literature intake, not a generic PDF -summarizer. The script combines local PDF metadata with a curated reading -matrix so the resulting CSV and memo can be regenerated after the source folder -changes. -""" - -from __future__ import annotations - -import argparse -import csv -import re -import subprocess -from collections import Counter -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -DEFAULT_SOURCE_DIR = Path("Papers_tesis") -DEFAULT_AUDIT_PATH = Path("docs/research/papers_tesis_deep_audit_2026-06-06.md") -DEFAULT_MATRIX_PATH = Path("reports/crpto/literature/papers_tesis_source_matrix_2026-06-06.csv") -DEFAULT_CAPTION_PATH = Path( - "reports/crpto/literature/papers_tesis_figure_caption_index_2026-06-06.csv" -) -DEFAULT_CURATED_VISUAL_PATH = Path( - "reports/crpto/literature/papers_tesis_curated_visual_sinks_2026-06-06.csv" -) -REPORT_DATE = "2026-06-06" - - -@dataclass(frozen=True) -class Review: - title: str - authors: str - year: str - status: str - primary_domain: str - bib_key: str = "" - bib_status: str = "audit_only" - canonical_url: str = "" - core_concepts: str = "" - key_claims: str = "" - method_or_evidence: str = "" - conclusions: str = "" - figures_tables_useful: str = "" - limitations: str = "" - crpto_value: str = "" - extended_lab_value: str = "" - quarto_destination: str = "" - decision: str = "" - action_required: str = "none_now" - evidence_gate: str = "No run: literature integration only." - artifact_sink: str = "docs/research/papers_tesis_deep_audit_2026-06-06.md" - stop_rule: str = "Do not reopen a lane unless it changes a manuscript claim." - implementation_or_experiment: str = "" - notes: str = "" - verification_status: str = "local_pdf_text" - tags: tuple[str, ...] = field(default_factory=tuple) - - -def r(**kwargs: Any) -> Review: - return Review(**kwargs) - - -REVIEWS: dict[str, Review] = { - "paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf": r( - title="Credit Scores: Performance and Equity", - authors="Stefania Albanesi; Domonkos F. Vamossy", - year="2024", - status="NBER working paper", - primary_domain="credit scoring / equity", - bib_key="albanesi2024credit", - bib_status="existing", - canonical_url="https://doi.org/10.3386/w32917", - core_concepts="Score performance, equity, misclassification, rank disagreement, observable vulnerable groups.", - key_claims="Better scores can improve accuracy while also changing error allocation across groups.", - method_or_evidence="Empirical credit-score comparison and equity diagnostics.", - conclusions="Useful for metric governance: report discrimination, calibration, utility and equity together.", - figures_tables_useful="Use tables on score performance/equity as template for FICO proxy vs champion diagnostics.", - limitations="Not Lending Club CRPTO evidence and not a legal fair-lending protocol.", - crpto_value="Light motivation: CRPTO is not only AUC; calibrated decision inputs require governance.", - extended_lab_value="Strong appendix support for score-proxy vs champion ML and equity boundary.", - quarto_destination="agenda extendida CRPTO/tesis appendix; Paper CRPTO motivation only.", - decision="append_extended_governance", - action_required="completed_quarto_patch", - tags=("credit", "equity", "governance"), - ), - "paper/Angelopoulos Bates 2023 - Gentle Introduction to Conformal Prediction.pdf": r( - title="Conformal Prediction: A Gentle Introduction", - authors="Anastasios N. Angelopoulos; Stephen Bates", - year="2023", - status="Foundations and Trends in Machine Learning", - primary_domain="conformal foundations", - bib_key="angelopoulos2023", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2107.07511", - core_concepts="Exchangeability, split conformal, prediction sets, finite-sample marginal coverage.", - key_claims="Distribution-free uncertainty can be wrapped around black-box predictors.", - method_or_evidence="Tutorial/survey with proofs, examples and diagnostics.", - conclusions="Best pedagogical citation for explaining why CRPTO's intervals are valid before optimization.", - figures_tables_useful="Use tutorial figures only conceptually; the paper needs its own CRPTO diagram.", - limitations="Introductory; not a portfolio optimizer or credit-risk paper.", - crpto_value="Core theory support for the uncertainty layer.", - extended_lab_value="Proof-audit reference for lab gates.", - quarto_destination="CRPTO theory and glossary.", - decision="promote_crpto_body", - action_required="none_now", - tags=("conformal", "foundation"), - ), - "paper/Angelopoulos et al 2024 - Conformal Risk Control.pdf": r( - title="Conformal Risk Control", - authors="Anastasios N. Angelopoulos; Stephen Bates; Adam Fisch; Lihua Lei; Tal Schuster", - year="2024", - status="ICLR 2024", - primary_domain="conformal risk control", - bib_key="angelopoulos2024risk", - bib_status="existing", - canonical_url="https://openreview.net/forum?id=33XGfHLtZg", - core_concepts="Risk control, monotone bounded losses, post-hoc calibration, user-defined loss.", - key_claims="Conformal calibration can control expected loss beyond ordinary set coverage.", - method_or_evidence="Finite-sample risk-control framework.", - conclusions="Direct ancestor of CRPTO's funded-set weighted miscoverage framing.", - figures_tables_useful="Use conceptual loss-control diagrams as related-work inspiration.", - limitations="Does not itself define a portfolio-funded-set bound.", - crpto_value="Core theory support for mapping uncertainty to a decision-relevant loss.", - extended_lab_value="CRC/LTT decision-loss gates for future experiments.", - quarto_destination="CRPTO theory; agenda extendida CRPTO/tesis gate register.", - decision="promote_crpto_body", - tags=("conformal", "risk_control"), - ), - "paper/Angelopoulos et al 2024 - Theoretical Foundations of Conformal Prediction.pdf": r( - title="Theoretical Foundations of Conformal Prediction", - authors="Anastasios N. Angelopoulos; Rina Foygel Barber; Stephen Bates", - year="2026", - status="Cambridge University Press pre-publication / monograph", - primary_domain="conformal foundations", - bib_key="angelopoulos2024foundations", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2411.11824", - core_concepts="Exchangeability, exact finite-sample coverage, randomized quantiles, modern CP theory.", - key_claims="Conformal validity is a theorem about a procedure and assumptions, not a model-quality claim.", - method_or_evidence="Long-form theoretical reference.", - conclusions="Use as the canonical proof reference for split conformal language and limitations.", - figures_tables_useful="Use definitions/proof structure, not figures, in CRPTO.", - limitations="Too broad for the paper body; cite selectively.", - crpto_value="Canonical theory reference for the proof and notation.", - extended_lab_value="Proof-audit anchor for future online/source claims.", - quarto_destination="CRPTO theory; glossary.", - decision="promote_crpto_body", - tags=("conformal", "foundation"), - ), - "paper/Angelopoulos et al 2025 - Learn Then Test.pdf": r( - title="Learn then Test: Calibrating Predictive Algorithms to Achieve Risk Control", - authors="Anastasios N. Angelopoulos; Stephen Bates; Emmanuel J. Candes; Michael I. Jordan; Lihua Lei", - year="2025", - status="Annals of Applied Statistics", - primary_domain="risk control / post-hoc testing", - bib_key="angelopoulos2025ltt", - bib_status="existing", - canonical_url="https://doi.org/10.1214/24-AOAS1998", - core_concepts="Learn-then-test, finite-sample risk guarantees, post-hoc risk calibration.", - key_claims="Predictive algorithms can be calibrated after learning to satisfy risk constraints.", - method_or_evidence="Statistical testing/calibration framework.", - conclusions="Supports predeclared gates and reviewer-safe validation language.", - figures_tables_useful="Use algorithmic workflow conceptually for gate diagrams.", - limitations="Does not solve downstream optimization by itself.", - crpto_value="Supports exact risk-gate language and post-selection caveats.", - extended_lab_value="Lane-gate template for CRC/LTT decision-loss checks.", - quarto_destination="CRPTO theory; agenda extendida CRPTO/tesis future gate register.", - decision="promote_crpto_body", - tags=("conformal", "risk_control", "gates"), - ), - "paper/Barber et al 2021 - Limits of Distribution-Free Conditional Predictive Inference.pdf": r( - title="The Limits of Distribution-Free Conditional Predictive Inference", - authors="Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas; Ryan J. Tibshirani", - year="2021", - status="Information and Inference", - primary_domain="conditional coverage limits", - bib_key="barber2021limits", - bib_status="existing", - canonical_url="https://doi.org/10.1093/imaiai/iaaa017", - core_concepts="Conditional coverage impossibility, approximate conditional validity, subgroup limits.", - key_claims="Exact distribution-free conditional coverage is impossible without assumptions or restrictions.", - method_or_evidence="Theoretical impossibility/boundary results.", - conclusions="This is a claim-boundary source: it prevents overclaiming subgroup guarantees.", - figures_tables_useful="Use theorem statements, not figures.", - limitations="Boundary paper, not an implementation.", - crpto_value="Keeps Mondrian coverage language honest.", - extended_lab_value="Source/fairness coverage caveat.", - quarto_destination="CRPTO theory; agenda extendida CRPTO/tesis online/source caveats.", - decision="promote_crpto_body", - tags=("conformal", "conditional_coverage", "claim_boundary"), - ), - "paper/Bates et al 2021 - Distribution-Free Risk-Controlling Prediction Sets.pdf": r( - title="Distribution-Free, Risk-Controlling Prediction Sets", - authors="Stephen Bates; Anastasios N. Angelopoulos; Lihua Lei; Jitendra Malik; Michael I. Jordan", - year="2021", - status="arXiv / risk-control foundation", - primary_domain="risk-controlling prediction sets", - bib_key="bates2021rcps", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2101.02703", - core_concepts="RCPS, set-valued prediction, expected loss control, black-box models.", - key_claims="Prediction sets can be calibrated to control a user-defined risk distribution-free.", - method_or_evidence="Post-hoc calibration framework.", - conclusions="Provides the bridge from coverage to risk-control vocabulary.", - figures_tables_useful="Use algorithmic schematic as conceptual support only.", - limitations="General prediction sets; no credit portfolio objective.", - crpto_value="Core CRPTO theory lineage.", - extended_lab_value="Decision-risk gate template.", - quarto_destination="CRPTO theory and related work.", - decision="promote_crpto_body", - tags=("conformal", "risk_control"), - ), - "paper/Bertsimas Gupta Kallus 2018 - Data-Driven Robust Optimization.pdf": r( - title="Data-Driven Robust Optimization", - authors="Dimitris Bertsimas; Vishal Gupta; Nathan Kallus", - year="2018", - status="Mathematical Programming", - primary_domain="data-driven robust optimization", - bib_key="bertsimas2018datadriven", - bib_status="needs_bib_if_cited", - canonical_url="https://doi.org/10.1007/s10107-017-1125-8", - core_concepts="Uncertainty sets learned from data, robustness guarantees, finite-sample feasibility.", - key_claims="Robust sets can be constructed data-driven rather than manually specified.", - method_or_evidence="Optimization theory and computational experiments.", - conclusions="Supports the historical progression from ad-hoc RO to calibrated/data-driven uncertainty sets.", - figures_tables_useful="Useful as related-work table contrasting source of uncertainty sets.", - limitations="Not conformal and not Lending Club-specific.", - crpto_value="Background for why conformal sets are a natural next RO step.", - extended_lab_value="Context only.", - quarto_destination="CRPTO related work if a longer RO paragraph is added.", - decision="context_crpto_related_work", - tags=("robust_optimization",), - ), - "paper/Bertsimas Kallus 2020 - From Predictive to Prescriptive Analytics.pdf": r( - title="From Predictive to Prescriptive Analytics", - authors="Dimitris Bertsimas; Nathan Kallus", - year="2020", - status="Management Science", - primary_domain="prescriptive analytics", - bib_key="bertsimas2020prescriptive", - bib_status="needs_bib_if_cited", - canonical_url="https://doi.org/10.1287/mnsc.2018.3253", - core_concepts="Prescriptive analytics, decision quality, predictive-to-prescriptive bridge.", - key_claims="Prediction alone is insufficient; decision policies should be evaluated by downstream outcomes.", - method_or_evidence="Framework and empirical prescriptive methods.", - conclusions="Useful framing for why CRPTO evaluates funded-set outcomes, not AUC only.", - figures_tables_useful="Related-work schematic only.", - limitations="Not conformal and not a risk-control guarantee.", - crpto_value="Strengthens introduction/positioning.", - extended_lab_value="Supports governed decision-analytics framing.", - quarto_destination="CRPTO related work or thesis framing.", - decision="context_crpto_related_work", - tags=("prescriptive_analytics", "decision"), - ), - "paper/Bertsimas Sim 2004 - The Price of Robustness.pdf": r( - title="The Price of Robustness", - authors="Dimitris Bertsimas; Melvyn Sim", - year="2004", - status="Operations Research", - primary_domain="robust optimization", - bib_key="bertsimas2004", - bib_status="existing", - canonical_url="https://doi.org/10.1287/opre.1030.0065", - core_concepts="Budgeted uncertainty, protection level, tractable robust LP/IP, price of robustness.", - key_claims="Robustness trades objective value for protection in a tunable, tractable way.", - method_or_evidence="Classical robust optimization formulation.", - conclusions="Use as the language ancestor for alpha -> Gamma_CP -> return trade-off.", - figures_tables_useful="Use price-of-robustness framing in CRPTO figures.", - limitations="Budget is chosen, not conformally calibrated.", - crpto_value="Core RO foundation.", - extended_lab_value="Tail/challenger context.", - quarto_destination="CRPTO theory and portfolio chapter.", - decision="promote_crpto_body", - tags=("robust_optimization",), - ), - "paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf": r( - title="Data-Driven Robust Credit Portfolio Optimization for Investment Decisions in P2P Lending", - authors="Guangyu Chi; Lei Ding; Xiangyu Peng", - year="2019", - status="Applied Soft Computing / Elsevier article", - primary_domain="P2P credit portfolio optimization", - bib_key="chi2019p2p", - bib_status="added_2026_06_06", - core_concepts="P2P lending, robust credit portfolio, data-driven scoring, risk-return allocation.", - key_claims="Robust optimization can improve P2P investment decisions under model uncertainty.", - method_or_evidence="Empirical P2P lending optimization study.", - conclusions="Closest credit-domain precedent for robust P2P allocation.", - figures_tables_useful="Use experimental design/tables as comparator template for Lending Club portfolio papers.", - limitations="No conformal guarantees and no funded-set miscoverage bound.", - crpto_value="Important applied-credit related work.", - extended_lab_value="Context for digital-lending robust allocation.", - quarto_destination="CRPTO related work; thesis context.", - decision="append_crpto_related_work", - action_required="completed_bib_quarto_patch", - tags=("credit", "robust_optimization", "p2p"), - ), - "paper/Donti et al 2017 - Task-Based End-to-End Model Learning.pdf": r( - title="Task-based End-to-end Model Learning in Stochastic Optimization", - authors="Priya L. Donti; Brandon Amos; J. Zico Kolter", - year="2017", - status="NeurIPS 2017", - primary_domain="decision-focused learning", - bib_key="donti2017", - bib_status="existing", - canonical_url="https://arxiv.org/abs/1703.04529", - core_concepts="End-to-end learning, differentiable optimization, downstream task loss.", - key_claims="Training against task loss can outperform prediction-error training for decisions.", - method_or_evidence="Decision-focused learning experiments.", - conclusions="DFL lineage for SPO+/PyEPO comparator.", - figures_tables_useful="Use as historical DFL lineage, not as CRPTO method figure.", - limitations="No conformal coverage or credit-specific auditability.", - crpto_value="Comparator lineage only.", - extended_lab_value="Supports PyEPO/DFL lane.", - quarto_destination="CRPTO related work; agenda extendida CRPTO/tesis DFL appendix.", - decision="append_comparator", - tags=("dfl",), - ), - "paper/Elmachtoub Grigas 2022 - Smart Predict Then Optimize.pdf": r( - title="Smart Predict, then Optimize", - authors="Adam N. Elmachtoub; Paul Grigas", - year="2022", - status="Management Science", - primary_domain="SPO+ / predict-then-optimize", - bib_key="elmachtoub2022", - bib_status="existing", - canonical_url="https://doi.org/10.1287/mnsc.2020.3922", - core_concepts="SPO loss, SPO+ surrogate, regret-oriented training, downstream decision quality.", - key_claims="Models can be trained to reduce decision regret rather than prediction error.", - method_or_evidence="Theory and computational experiments for SPO/SPO+.", - conclusions="Primary DFL comparator: SPO+ may reduce regret but does not provide conformal auditability.", - figures_tables_useful="Use benchmark/regret tables as comparator template.", - limitations="No coverage guarantee and no uncertainty-set audit trail.", - crpto_value="Core comparator, not replacement.", - extended_lab_value="Formal PyEPO lane support.", - quarto_destination="CRPTO related work and SPO appendix.", - decision="append_comparator", - tags=("dfl", "spo"), - ), - "paper/Guo et al 2016 - Instance-Based Credit Risk Assessment for Investment Decisions in P2P Lending.pdf": r( - title="Instance-based Credit Risk Assessment for Investment Decisions in P2P Lending", - authors="Yanhui Guo; Wenjun Zhou; Chunyu Luo; Chuanren Liu; Hui Xiong", - year="2016", - status="European Journal of Operational Research", - primary_domain="P2P credit decision support", - bib_key="guo2016p2p", - bib_status="needs_bib_if_cited", - canonical_url="https://doi.org/10.1016/j.ejor.2015.05.050", - core_concepts="Instance-based learning, P2P loan investment, risk-return decision support.", - key_claims="Loan-level decision support can improve P2P investment performance.", - method_or_evidence="Empirical P2P lending recommender/credit assessment.", - conclusions="Useful historical Lending Club/P2P context.", - figures_tables_useful="Portfolio decision tables can inspire thesis background.", - limitations="No conformal or robust bound.", - crpto_value="Applied setting support.", - extended_lab_value="Digital-lending decision context.", - quarto_destination="Related work if credit-domain section expands.", - decision="context_credit_domain", - tags=("credit", "p2p"), - ), - "paper/Hu et al 2026 - Conformal Robustness Control.pdf": r( - title="Conformal Robustness Control", - authors="Yang Hu; Jieren Tan; Changliang Zou; Yajie Bao; Haojie Ren", - year="2026", - status="ICLR 2026", - primary_domain="conformal robustness / decision risk", - bib_key="hu2026crc", - bib_status="added_2026_06_06", - canonical_url="https://openreview.net/forum?id=xRjOrcj08o", - core_concepts="Robustness control, conformal calibration, robust decision criteria.", - key_claims="Conformal machinery can certify robustness-oriented decision properties.", - method_or_evidence="Recent conference method with robust-control framing.", - conclusions="Confirms CRPTO is timely, but should remain future-work unless reproduced.", - figures_tables_useful="Use as frontier map figure/table in related work, not main evidence.", - limitations="Not yet implemented in the project; no Lending Club funded-set result.", - crpto_value="Future-work frontier; supports not overclaiming current method.", - extended_lab_value="Candidate future gate for robustness certificates.", - quarto_destination="agenda extendida CRPTO/tesis future gate; CRPTO related-work frontier.", - decision="park_future_work", - action_required="completed_bib_quarto_patch", - evidence_gate="Only run if a reviewer asks for robustness-control comparison or it changes C4/C6.", - artifact_sink="agenda extendida CRPTO/tesis future experiment register.", - stop_rule="Park if it cannot be evaluated on frozen CRPTO artifacts without retraining a new method.", - implementation_or_experiment="Possible robustness-control audit over V/violation; not executed.", - verification_status="web_primary_and_local_pdf", - tags=("conformal", "robustness", "future_work"), - ), - "paper/Jagtiani Lemieux 2019 - Alternative Data and Machine Learning in Fintech Lending.pdf": r( - title="The Roles of Alternative Data and Machine Learning in Fintech Lending", - authors="Julapa Jagtiani; Catharine Lemieux", - year="2019", - status="Financial Management", - primary_domain="fintech lending / LendingClub", - bib_key="jagtiani2019altdata", - bib_status="existing", - canonical_url="https://doi.org/10.1111/fima.12295", - core_concepts="LendingClub, alternative data, credit grades, FICO relationship, fintech underwriting.", - key_claims="Fintech grades and alternative data provide empirical credit information beyond traditional scores.", - method_or_evidence="Empirical LendingClub study.", - conclusions="Best setting citation for why Lending Club is a meaningful credit-risk lab.", - figures_tables_useful="Use tables about grade/FICO/performance as context, not CRPTO evidence.", - limitations="Does not validate CRPTO or fairness claims.", - crpto_value="Core empirical-setting support.", - extended_lab_value="Metric governance and FICO proxy context.", - quarto_destination="CRPTO introduction and dataset context.", - decision="promote_crpto_body", - tags=("credit", "lendingclub"), - ), - "paper/Johnstone Cox 2021 - Conformal Uncertainty Sets for Robust Optimization.pdf": r( - title="Conformal Uncertainty Sets for Robust Optimization", - authors="Chancellor Johnstone; Bruce Cox", - year="2021", - status="COPA / PMLR", - primary_domain="conformal robust optimization", - bib_key="johnstone2021", - bib_status="existing", - canonical_url="https://proceedings.mlr.press/v152/johnstone21a.html", - core_concepts="Conformal regions as uncertainty sets, robust optimization, finite-sample validity.", - key_claims="Conformal prediction regions can feed robust optimization with validity.", - method_or_evidence="Method paper with simulation evidence.", - conclusions="Closest foundational CP -> RO bridge.", - figures_tables_useful="Use as conceptual predecessor for CRPTO pipeline figure.", - limitations="No credit-specific funded-set weighted bound.", - crpto_value="Core method foundation.", - extended_lab_value="Context only.", - quarto_destination="CRPTO theory and related work.", - decision="promote_crpto_body", - tags=("conformal", "robust_optimization"), - ), - "paper/Patel et al 2024 - Conformal Contextual Robust Optimization.pdf": r( - title="Conformal Contextual Robust Optimization", - authors="Neel Patel; Omar Rayan; Ambuj Tewari", - year="2024", - status="AISTATS 2024", - primary_domain="conformal contextual robust optimization", - bib_key="patel2024", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2310.10003", - core_concepts="Contextual robust optimization, conformal regions, generative uncertainty.", - key_claims="Informative conformal uncertainty regions can improve robust contextual decisions.", - method_or_evidence="Method and benchmark experiments.", - conclusions="Strong neighboring method but with different uncertainty geometry and domains.", - figures_tables_useful="Use related-work contrast table.", - limitations="Not credit-specific and no funded-set PD constraint.", - crpto_value="Direct neighbor in related work.", - extended_lab_value="Future high-dimensional uncertainty context.", - quarto_destination="CRPTO related work.", - decision="promote_crpto_body", - tags=("conformal", "robust_optimization"), - ), - "paper/Sun et al 2024 - Predict-then-Calibrate.pdf": r( - title="Predict-then-Calibrate: A New Perspective of Robust Contextual LP", - authors="Chunlin Sun; Linyu Liu; Xiaocheng Li", - year="2024", - status="arXiv / robust contextual LP", - primary_domain="predict-then-calibrate", - bib_key="sun2024ptc", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2305.15686", - core_concepts="Predict-then-calibrate, robust contextual LP, box/ellipsoid uncertainty sets.", - key_claims="Post-hoc calibration can build valid uncertainty sets for robust contextual optimization.", - method_or_evidence="Theory and experiments for robust LP calibration.", - conclusions="Closest conceptual comparator to post-hoc CRPTO.", - figures_tables_useful="Use contrast table: PtC set membership vs CRPTO funded-set weighted risk.", - limitations="No Lending Club and no Mondrian funded-set bound.", - crpto_value="Core closest-neighbor contrast.", - extended_lab_value="Selector/calibration context.", - quarto_destination="CRPTO related work and theory.", - decision="promote_crpto_body", - tags=("conformal", "robust_optimization", "ptc"), - ), - "paper/Torkian Bamdad Sarfaraz 2025 - AI OR Investment Decisions in Digital Lending.pdf": r( - title="Integrating AI and OR for investment decision-making in emerging digital lending businesses", - authors="Vajiheh Torkian; Shahrooz Bamdad; Amir Homayoun Sarfaraz", - year="2026", - status="Journal of the Operational Research Society; online 2025", - primary_domain="AI/OR digital lending", - bib_key="aior2025lendingclub", - bib_status="added_2026_06_06", - canonical_url="https://doi.org/10.1080/01605682.2025.2498652", - core_concepts="Digital lending, multi-objective risk-return optimization, AI + OR pipeline.", - key_claims="Combining ML and OR improves investment recommendations in digital lending.", - method_or_evidence="Applied multi-objective optimization paper.", - conclusions="Useful applied competitor/context; not a conformal method.", - figures_tables_useful="Use risk-return frontier and experimental setup as agenda extendida CRPTO/tesis comparator inspiration.", - limitations="No conformal coverage, no CRPTO bound, no Lending Club champion evidence.", - crpto_value="Related-work context only.", - extended_lab_value="agenda extendida CRPTO/tesis digital-lending architecture context.", - quarto_destination="agenda extendida CRPTO/tesis synthesis; CRPTO related work if space allows.", - decision="append_extended_context", - action_required="completed_bib_quarto_patch", - verification_status="web_primary_and_local_pdf", - tags=("credit", "optimization", "digital_lending"), - ), - "paper/Zhao et al 2026 - Conformal Robust Optimization and Satisficing.pdf": r( - title="Conformal Robust Optimization and Satisficing for Prescriptive Analytics with Black-Box Predictors", - authors="Lingjie Zhao; Hansheng Jiang; Wei Qi", - year="2026", - status="AISTATS workshop / SSRN working paper", - primary_domain="conformal robust optimization / satisficing", - bib_key="zhao2025robust", - bib_status="existing", - canonical_url="https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5338354", - core_concepts="Conformal robust optimization, satisficing, black-box predictors, parameter mapping.", - key_claims="CRO and conformal robust satisficing can certify robust prescriptive decisions.", - method_or_evidence="Workshop paper with optimization experiments.", - conclusions="Valuable frontier for satisficing margins and policy acceptance thresholds.", - figures_tables_useful="Use satisficing margin idea for appendix, not body.", - limitations="Preprint/workshop and not credit-specific.", - crpto_value="Related-work frontier.", - extended_lab_value="Satisficing/governance candidate.", - quarto_destination="CRPTO related work; agenda extendida CRPTO/tesis future gates.", - decision="append_future_work", - tags=("conformal", "robust_optimization", "satisficing"), - ), - "supplement/Angelopoulos et al 2026 - Conformal Risk Control for Non-Monotonic Losses.pdf": r( - title="Conformal Risk Control for Non-Monotonic Losses", - authors="Anastasios N. Angelopoulos", - year="2026", - status="arXiv preprint", - primary_domain="non-monotonic conformal risk control", - bib_key="angelopoulos2026nonmonotonic", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2602.20151", - core_concepts="Non-monotonic loss, risk control, multi-objective loss families.", - key_claims="CRC ideas can extend beyond monotone losses under new machinery.", - method_or_evidence="Theory preprint.", - conclusions="Useful for future compound loss: return + V + CVaR + source risk.", - figures_tables_useful="Use loss examples for future gate design.", - limitations="Not implemented; would change current CRPTO claim.", - crpto_value="Future-work only.", - extended_lab_value="Strong future gate for composite risk.", - quarto_destination="agenda extendida CRPTO/tesis future experiment register.", - decision="park_future_work", - action_required="none_now", - evidence_gate="Only reopen with a fixed compound loss and prospective split.", - tags=("conformal", "risk_control", "future_work"), - ), - "supplement/Bao et al 2025 - CROMS Optimal Model Selection for Conformalized Robust Optimization.pdf": r( - title="Optimal Model Selection for Conformalized Robust Optimization", - authors="Yajie Bao; Yang Hu; Haojie Ren; Peng Zhao; Changliang Zou", - year="2025", - status="arXiv preprint", - primary_domain="CROMS / model selection", - bib_key="bao2025croms", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2507.04716", - core_concepts="Conformalized robust optimization, decision-aware model selection, downstream risk.", - key_claims="Selecting conformal models should account for robust decision performance.", - method_or_evidence="Theory/algorithm and experiments.", - conclusions="Justifies CRPTO's CROMS-lite selector but not a full implementation claim.", - figures_tables_useful="Use selection diagrams/tables to structure A5/A10 appendix.", - limitations="Project only has a selector screen over artifacts, not CROMS training.", - crpto_value="Appendix selector support.", - extended_lab_value="Lane 2 selector context.", - quarto_destination="CRPTO appendix and agenda extendida CRPTO/tesis selector lane.", - decision="append_crpto_selector", - tags=("conformal", "selector", "robust_optimization"), - ), - "supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf": r( - title="Conformal Prediction Beyond Exchangeability", - authors="Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas; Ryan J. Tibshirani", - year="2023", - status="Annals of Statistics", - primary_domain="beyond exchangeability / weighted conformal", - bib_key="barber2023beyond", - bib_status="added_2026_06_06", - canonical_url="https://doi.org/10.1214/23-AOS2276", - core_concepts="Weighted exchangeability, distribution shift, non-exchangeable conformal validity.", - key_claims="Conformal validity can be extended with weights under structured departures from exchangeability.", - method_or_evidence="Theory paper.", - conclusions="Important boundary/future source for source/shift lanes.", - figures_tables_useful="Use assumptions table for source governance appendix.", - limitations="Requires weight/shift structure not currently guaranteed.", - crpto_value="Caveat/future work.", - extended_lab_value="Strong source/shift conformal lane reference.", - quarto_destination="agenda extendida CRPTO/tesis online/source caveat.", - decision="append_extended_source_shift", - action_required="completed_bib_quarto_patch", - verification_status="web_primary_and_local_pdf", - tags=("conformal", "shift", "source"), - ), - "supplement/Ben-Tal Teboulle 2007 - Optimized Certainty Equivalent.pdf": r( - title="An Old-New Concept of Convex Risk Measures: The Optimized Certainty Equivalent", - authors="Aharon Ben-Tal; Marc Teboulle", - year="2007", - status="Mathematical Finance", - primary_domain="OCE / convex risk measures", - bib_key="bental2007oce", - bib_status="existing", - canonical_url="https://doi.org/10.1111/j.1467-9965.2007.00311.x", - core_concepts="Optimized certainty equivalent, convex risk measures, CVaR relation.", - key_claims="OCE provides a principled convex risk-measure family.", - method_or_evidence="Mathematical risk-measure theory.", - conclusions="Grounds OCE/CVaR tail-risk appendix.", - figures_tables_useful="Use definitions, not figures.", - limitations="Does not choose a credit champion by itself.", - crpto_value="Appendix tail-risk foundation.", - extended_lab_value="Tail challenger foundation.", - quarto_destination="CRPTO appendix; agenda extendida CRPTO/tesis tail lane.", - decision="append_tail_risk", - tags=("tail_risk", "oce"), - ), - "supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf": r( - title="Group-Weighted Conformal Prediction", - authors="Aabesh Bhattacharyya; Rina Foygel Barber", - year="2026", - status="Electronic Journal of Statistics", - primary_domain="group-weighted conformal prediction", - bib_key="bhattacharyya2026groupweighted", - bib_status="added_2026_06_06", - canonical_url="https://doi.org/10.1214/26-EJS2506", - core_concepts="Group-weighted CP, group-based shift, weighted conformal prediction.", - key_claims="When groups drive covariate shift, group-weighted calibration can improve guarantees.", - method_or_evidence="Theory paper with group-weighted guarantees.", - conclusions="Natural candidate for agenda extendida CRPTO/tesis source/grade reweighting, but not current CRPTO.", - figures_tables_useful="Use group-weighting assumptions for source-governance table.", - limitations="Needs target/source group weights and prospective design.", - crpto_value="Future robustness/context only.", - extended_lab_value="Potential source/shift experiment over grade/period/source families.", - quarto_destination="agenda extendida CRPTO/tesis source/shift appendix.", - decision="append_mixed_diagnostic_source_not_better", - action_required="experiment_completed_appendix_diagnostic", - evidence_gate="Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; absolute gate passed, but worst-source coverage fell 0.0111 below Mondrian.", - artifact_sink="docs/research/papers_tesis_deep_audit_2026-06-06.md", - stop_rule="Append as source/shift diagnostic only; do not claim group-weighted source improvement over Mondrian.", - implementation_or_experiment="Group-weighted source-max replay: coverage 0.9387, avg width 0.9394, worst defended source 0.8602.", - verification_status="web_primary_and_local_pdf", - tags=("conformal", "group_weighted", "shift"), - ), - "supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf": r( - title="Using Publicly Available Information to Proxy for Unidentified Race and Ethnicity", - authors="Consumer Financial Protection Bureau", - year="2014", - status="official methodology report", - primary_domain="fair-lending proxy methodology", - bib_key="cfpb2014bisg", - bib_status="needs_bib_if_cited", - canonical_url="https://files.consumerfinance.gov/f/201409_cfpb_report_proxy-methodology.pdf", - core_concepts="BISG-style proxy, surname/geography, proxy validation, protected-attribute limitation.", - key_claims="Race/ethnicity proxying requires richer inputs and careful validation.", - method_or_evidence="Regulatory/official methodology report.", - conclusions="Supports why Lending Club zip3/state is insufficient for legal fair-lending claims.", - figures_tables_useful="Use proxy-method workflow as governance appendix reference.", - limitations="Project lacks surname and fine geography.", - crpto_value="Fairness limitation only.", - extended_lab_value="Strong claim-boundary source for proxy governance.", - quarto_destination="agenda extendida CRPTO/tesis fairness/source governance.", - decision="append_extended_governance", - tags=("fairness", "governance"), - ), - "supplement/Chenreddy Delage 2024 - End-to-End Conditional Robust Optimization.pdf": r( - title="End-to-End Conditional Robust Optimization", - authors="Abhilash Chenreddy; Erick Delage", - year="2024", - status="UAI 2024 / PMLR", - primary_domain="end-to-end conditional robust optimization", - bib_key="chenreddy2024", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2403.04670", - core_concepts="Conditional robust optimization, differentiable optimization, end-to-end training.", - key_claims="End-to-end conditional robust training can improve conditional coverage/objective tradeoffs.", - method_or_evidence="Method with differentiable robust optimization experiments.", - conclusions="Frontier contrast: CRPTO is post-hoc and auditable, not end-to-end.", - figures_tables_useful="Use as frontier comparison table.", - limitations="Different stack and conditional guarantees; not current project method.", - crpto_value="Future-work contrast.", - extended_lab_value="End-to-end lane context.", - quarto_destination="agenda extendida CRPTO/tesis future work; CRPTO related-work frontier.", - decision="park_future_work", - tags=("robust_optimization", "end_to_end"), - ), - "supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf": r( - title="Utility-Directed Conformal Prediction", - authors="Santiago Cortes-Gomez; Carlos Patino; Yewon Byun; Steven Wu; Eric Horvitz; Bryan Wilder", - year="2025", - status="ICLR 2025", - primary_domain="decision-aware conformal prediction", - bib_key="cortesgomez2025utility", - bib_status="added_2026_06_06", - canonical_url="https://openreview.net/forum?id=iOMnn1hSBO", - core_concepts="Utility-directed CP, decision loss, actionable uncertainty, coverage preservation.", - key_claims="Prediction sets can incorporate downstream utility while retaining standard coverage.", - method_or_evidence="Decision-aware conformal method and experiments.", - conclusions="Highly relevant future step: optimize CP usefulness without abandoning coverage.", - figures_tables_useful="Use framework diagram for future decision-loss conformal selector design.", - limitations="Current CRPTO does not implement this training/calibration objective.", - crpto_value="Future-work extension; supports A5 selector direction.", - extended_lab_value="Candidate decision-loss gate or conformal utility experiment.", - quarto_destination="agenda extendida CRPTO/tesis future gates; CRPTO future work.", - decision="park_width_fail", - action_required="experiment_completed_parked", - evidence_gate="Executed with fixed width/violation/tail-miss loss on frozen v4 replay; coverage rose to 0.9954 but avg width 0.9981 failed the 0.98 width gate.", - artifact_sink="docs/research/papers_tesis_deep_audit_2026-06-06.md", - stop_rule="Park as negative width result; no utility-directed selector claim.", - implementation_or_experiment="Utility-directed loss replay: coverage 0.9954, avg width +0.2205 versus Mondrian, worst defended source 0.9499.", - verification_status="web_primary_and_local_pdf", - tags=("conformal", "utility", "decision"), - ), - "supplement/FinRegLab 2023 - Explainability and Fairness in ML Credit Underwriting.pdf": r( - title="Explainability and Fairness in Machine Learning for Credit Underwriting: Policy Analysis", - authors="FinRegLab", - year="2023", - status="policy report", - primary_domain="ML credit governance", - bib_key="finreglab2023fairness", - bib_status="existing", - canonical_url="https://finreglab.org/research/explainability-fairness-in-machine-learning-for-credit-underwriting-policy-analysis/", - core_concepts="Explainability, fairness, adverse action, model governance, underwriting controls.", - key_claims="ML credit systems require governance controls beyond predictive performance.", - method_or_evidence="Policy analysis and stakeholder evidence.", - conclusions="Supports MRM/fairness appendix and explains why claims must be bounded.", - figures_tables_useful="Use control-taxonomy tables for governance mapping.", - limitations="Policy report, not CRPTO empirical evidence.", - crpto_value="Governance appendix support.", - extended_lab_value="Strong governance/fairness boundary source.", - quarto_destination="CRPTO governance appendix; agenda extendida CRPTO/tesis governance.", - decision="append_governance", - tags=("governance", "fairness", "credit"), - ), - "supplement/Gibbs Candes 2021 - Adaptive Conformal Inference Under Distribution Shift.pdf": r( - title="Adaptive Conformal Inference Under Distribution Shift", - authors="Isaac Gibbs; Emmanuel J. Candes", - year="2021", - status="NeurIPS 2021", - primary_domain="adaptive conformal inference", - bib_key="gibbs2021aci", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2106.00170", - core_concepts="ACI, online updating, distribution shift, adaptive quantiles.", - key_claims="Conformal thresholds can be adapted under shift to improve long-run coverage.", - method_or_evidence="Algorithm and experiments.", - conclusions="Future online/drift candidate, not current static CRPTO.", - figures_tables_useful="Use online update schematic for agenda extendida CRPTO/tesis future work.", - limitations="Needs streaming/prospective feedback to claim deployment validity.", - crpto_value="Future-work caveat.", - extended_lab_value="Online/source conformal lane context.", - quarto_destination="agenda extendida CRPTO/tesis online caveat; thesis.", - decision="park_future_work", - tags=("conformal", "online", "shift"), - ), - "supplement/Gibbs Cherian Candes 2025 - Conformal Prediction with Conditional Guarantees.pdf": r( - title="Conformal Prediction with Conditional Guarantees", - authors="Isaac Gibbs; John J. Cherian; Emmanuel J. Candes", - year="2025", - status="JRSS-B", - primary_domain="conditional conformal guarantees", - bib_key="gibbs2024", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2305.12616", - core_concepts="Conditional guarantees, coverage spectrum, restricted conditional targets.", - key_claims="Relaxed conditional goals can be achieved where exact conditional coverage cannot.", - method_or_evidence="Theory paper.", - conclusions="Supports future conditional-tightening language and source-group caveats.", - figures_tables_useful="Use conceptual spectrum for appendix if conditional validity is discussed.", - limitations="Not implemented in project; current Mondrian is finite-group conditional-ish, not exact individual conditional.", - crpto_value="Caveat and future-work support.", - extended_lab_value="Source/group coverage lane context.", - quarto_destination="agenda extendida CRPTO/tesis online/source caveat; CRPTO theory appendix.", - decision="append_claim_boundary", - tags=("conformal", "conditional_coverage"), - ), - "supplement/Guan 2023 - Localized Conformal Prediction.pdf": r( - title="Localized Conformal Prediction", - authors="Leying Guan", - year="2023", - status="Biometrika", - primary_domain="localized conformal prediction", - bib_key="guan2023localized", - bib_status="existing", - canonical_url="https://doi.org/10.1093/biomet/asac040", - core_concepts="Localized conformity scores, local calibration, approximate conditional validity.", - key_claims="Local weighting can improve adaptivity while retaining conformal inference structure.", - method_or_evidence="Generalized local conformal framework.", - conclusions="Candidate comparator to Mondrian if future coverage sharpness matters.", - figures_tables_useful="Use local-vs-global schematic for appendix only.", - limitations="Would require a new localized calibration design.", - crpto_value="Future-work comparator.", - extended_lab_value="Potential source/shift experiment.", - quarto_destination="agenda extendida CRPTO/tesis future gates; thesis.", - decision="append_mixed_diagnostic_wider_than_mondrian", - action_required="experiment_completed_appendix_diagnostic", - evidence_gate="Executed on frozen v4 replay with 2018 calibration and 2019-2020 holdout; coverage and worst-source coverage improved, but width increased 0.0867 versus Mondrian.", - artifact_sink="docs/research/papers_tesis_deep_audit_2026-06-06.md", - stop_rule="Append only as reviewer-facing localized diagnostic; no localized guarantee or champion replacement claim.", - implementation_or_experiment="Localized score-bin replay: coverage 0.9497, avg width 0.8643, worst defended source 0.9252.", - tags=("conformal", "localized"), - ), - "supplement/Jonkers et al 2024 - Conformal Predictive Systems Under Covariate Shift.pdf": r( - title="Conformal Predictive Systems Under Covariate Shift", - authors="Sander Jonkers and coauthors", - year="2024", - status="arXiv / preprint", - primary_domain="covariate shift conformal systems", - bib_key="jonkers2024wcps", - bib_status="needs_bib_if_cited", - core_concepts="Conformal predictive systems, covariate shift, calibration under shift.", - key_claims="Conformal predictive systems can be adapted for shifted covariate distributions.", - method_or_evidence="Method paper.", - conclusions="Useful for source/shift caveat, not current champion.", - figures_tables_useful="Use shift taxonomy only.", - limitations="Requires shift assumptions and is not credit-specific.", - crpto_value="Future-work context.", - extended_lab_value="Source/shift lane context.", - quarto_destination="agenda extendida CRPTO/tesis source/shift appendix if expanded.", - decision="context_source_shift", - tags=("conformal", "shift"), - ), - "supplement/Kiyani et al 2025 - Decision Theoretic Foundations for Conformal Prediction.pdf": r( - title="Decision Theoretic Foundations for Conformal Prediction", - authors="Shayan Kiyani and coauthors", - year="2025", - status="AISTATS / ICML-era preprint", - primary_domain="decision theory for conformal prediction", - bib_key="kiyani2025", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2502.02561", - core_concepts="Decision-theoretic CP, utility, actionability, uncertainty quantification.", - key_claims="Conformal sets can be understood through decisions and utilities, not only coverage.", - method_or_evidence="Theoretical framing.", - conclusions="Good conceptual support for CRPTO's decision-aware framing.", - figures_tables_useful="Use as theory bridge in future-work paragraph.", - limitations="Does not implement CRPTO portfolio optimization.", - crpto_value="Future/positioning support.", - extended_lab_value="Decision-risk lab framing.", - quarto_destination="CRPTO discussion; agenda extendida CRPTO/tesis future gates.", - decision="append_future_work", - tags=("conformal", "decision"), - ), - "supplement/Lekeufack et al 2023 - Conformal Decision Theory.pdf": r( - title="Conformal Decision Theory", - authors="Jordan Lekeufack and coauthors", - year="2023", - status="NeurIPS workshop / preprint", - primary_domain="conformal decision theory", - bib_key="lekeufack2023cdt", - bib_status="needs_bib_if_cited", - core_concepts="Conformal prediction for decisions, calibrated decision policies, set-valued actions.", - key_claims="Conformal uncertainty can be integrated with decision-making objectives.", - method_or_evidence="Conceptual/method paper.", - conclusions="Supports agenda extendida CRPTO/tesis decision-risk framing but is not a CRPTO baseline.", - figures_tables_useful="Use only as conceptual context.", - limitations="No credit portfolio evidence.", - crpto_value="Future-work context.", - extended_lab_value="Decision analytics context.", - quarto_destination="agenda extendida CRPTO/tesis related work if expanded.", - decision="context_decision_conformal", - tags=("conformal", "decision"), - ), - "supplement/Lin Delage Chan 2024 - Conformal Inverse Optimization.pdf": r( - title="Conformal Inverse Optimization", - authors="Bo Lin; Erick Delage; Timothy C. Y. Chan", - year="2024", - status="NeurIPS 2024", - primary_domain="conformal inverse optimization", - bib_key="chan2024inverse", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2402.01489", - core_concepts="Inverse optimization, conformal uncertainty, decision ambiguity.", - key_claims="Conformal methods can quantify uncertainty in inverse optimization.", - method_or_evidence="Theory/algorithm.", - conclusions="Valuable frontier for auditing human/legacy decisions, not current CRPTO.", - figures_tables_useful="Use CREDO/inverse-optimization family as future audit lane.", - limitations="Different problem: inferring preferences/parameters from decisions.", - crpto_value="Future-work context.", - extended_lab_value="Decision audit/source governance context.", - quarto_destination="agenda extendida CRPTO/tesis future gates.", - decision="park_future_work", - tags=("conformal", "inverse_optimization"), - ), - "supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf": r( - title="Multi-Source Conformal Inference Under Distribution Shift", - authors="Meichen Liu; Alexander W. Levis; Sara-Lynne Normand; Larry Han", - year="2024", - status="arXiv preprint", - primary_domain="multi-source conformal inference", - bib_key="liu2024multisource", - bib_status="needs_bib_if_cited", - canonical_url="https://arxiv.org/abs/2402.12105", - core_concepts="Multi-source inference, heterogeneous source distributions, shifted test populations.", - key_claims="Combining calibration sources carefully can maintain valid intervals under source shift.", - method_or_evidence="Theory and experiments.", - conclusions="Directly relevant to agenda extendida CRPTO/tesis source-family holdout but needs stronger data design.", - figures_tables_useful="Use source-combination taxonomy for source governance.", - limitations="Current LC source groups are retrospective, not validated external sources.", - crpto_value="Caveat/future work only.", - extended_lab_value="Source-family holdout lane.", - quarto_destination="agenda extendida CRPTO/tesis online/source caveat.", - decision="append_extended_source_shift", - tags=("conformal", "source", "shift"), - ), - "supplement/Liu et al 2026 - Online Conformal Prediction via Universal Portfolio Algorithms.pdf": r( - title="Online Conformal Prediction via Universal Portfolio Algorithms", - authors="Tuo Liu; Edgar Dobriban; Francesco Orabona", - year="2026", - status="arXiv preprint", - primary_domain="online conformal prediction", - bib_key="liu2026portfolio", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2602.03168", - core_concepts="Online conformal prediction, universal portfolios, regret-to-coverage, parameter-free adaptation.", - key_claims="Online CP can achieve long-run coverage via universal portfolio algorithms.", - method_or_evidence="Theory and experiments.", - conclusions="Excellent future direction for deployment, but current project is retrospective.", - figures_tables_useful="Use online update process as future-work schematic.", - limitations="No production feedback stream in current data.", - crpto_value="Future-work only.", - extended_lab_value="Online lane context and stop-rule support.", - quarto_destination="agenda extendida CRPTO/tesis online caveat; CRPTO future work.", - decision="park_future_work", - tags=("conformal", "online"), - ), - "supplement/Mandi et al 2024 - Decision-Focused Learning Survey.pdf": r( - title="Decision-Focused Learning: Foundations, State of the Art, Benchmark and Future Opportunities", - authors="Jayanta Mandi and coauthors", - year="2024", - status="Journal of Artificial Intelligence Research", - primary_domain="decision-focused learning survey", - bib_key="mandi2024", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2307.13565", - core_concepts="DFL taxonomy, SPO+, gradient-based/gradient-free methods, benchmarks.", - key_claims="DFL is mature but no single method dominates across tasks.", - method_or_evidence="Survey and benchmark synthesis.", - conclusions="Use to defend treating SPO+/PyEPO as comparator, not CRPTO replacement.", - figures_tables_useful="Use taxonomy table for agenda extendida CRPTO/tesis DFL suite framing.", - limitations="Survey does not provide conformal guarantees.", - crpto_value="Comparator framing.", - extended_lab_value="Core PyEPO/DFL suite context.", - quarto_destination="CRPTO related work; agenda extendida CRPTO/tesis PyEPO section.", - decision="append_comparator", - tags=("dfl", "survey"), - ), - "supplement/Rockafellar Uryasev 2000 - Optimization of Conditional Value-at-Risk.pdf": r( - title="Optimization of Conditional Value-at-Risk", - authors="R. Tyrrell Rockafellar; Stanislav Uryasev", - year="2000", - status="The Journal of Risk", - primary_domain="CVaR optimization", - bib_key="rockafellar2000cvar", - bib_status="existing", - canonical_url="https://doi.org/10.21314/JOR.2000.038", - core_concepts="CVaR, tail risk, tractable convex optimization.", - key_claims="CVaR is optimizable and more useful than VaR for tail-risk control.", - method_or_evidence="Optimization theory.", - conclusions="Grounds A12 and agenda extendida CRPTO/tesis tail challenger.", - figures_tables_useful="Use formulation, not figures.", - limitations="Tail-risk improvement does not imply wealth champion.", - crpto_value="Appendix tail-risk diagnostic.", - extended_lab_value="Strong CVaR challenger foundation.", - quarto_destination="CRPTO appendix; agenda extendida CRPTO/tesis tail lane.", - decision="append_tail_risk", - tags=("tail_risk", "cvar"), - ), - "supplement/Romano Patterson Candes 2019 - Conformalized Quantile Regression.pdf": r( - title="Conformalized Quantile Regression", - authors="Yaniv Romano; Evan Patterson; Emmanuel J. Candes", - year="2019", - status="NeurIPS 2019", - primary_domain="CQR", - bib_key="romano2019", - bib_status="existing", - canonical_url="https://arxiv.org/abs/1905.03222", - core_concepts="Conformalized quantile regression, adaptive intervals, split conformal.", - key_claims="Quantile regression plus conformalization yields valid adaptive intervals.", - method_or_evidence="Method and experiments.", - conclusions="Baseline/variant reference for interval adaptivity.", - figures_tables_useful="Use interval-width/adaptivity examples in conformal chapter.", - limitations="Current champion is Mondrian score-decile, not CQR.", - crpto_value="Conformal variant/context.", - extended_lab_value="Potential future variant only.", - quarto_destination="Conformal chapter; CRPTO appendix.", - decision="append_conformal_variant", - tags=("conformal", "cqr"), - ), - "supplement/Schutte et al 2024 - Robust Losses for Decision-Focused Learning.pdf": r( - title="Robust Losses for Decision-Focused Learning", - authors="Wolfgang Schutte and coauthors", - year="2024", - status="NeurIPS / arXiv-era DFL paper", - primary_domain="robust DFL", - bib_key="schutte2024robust", - bib_status="needs_bib_if_cited", - core_concepts="Robust decision-focused losses, decision regret, misspecification.", - key_claims="Robust losses can improve DFL under uncertainty or misspecification.", - method_or_evidence="DFL method and experiments.", - conclusions="agenda extendida CRPTO/tesis DFL challenger context; not a CRPTO body claim.", - figures_tables_useful="Use loss comparison table for DFL appendix only.", - limitations="No conformal auditability or credit-specific bound.", - crpto_value="Comparator/future work.", - extended_lab_value="Potential DFL suite extension.", - quarto_destination="agenda extendida CRPTO/tesis PyEPO/DFL appendix if expanded.", - decision="park_future_dfl", - tags=("dfl", "robustness"), - ), - "supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf": r( - title="Conformal Prediction Under Covariate Shift", - authors="Ryan J. Tibshirani; Rina Foygel Barber; Emmanuel J. Candes; Aaditya Ramdas", - year="2019", - status="NeurIPS 2019", - primary_domain="weighted conformal / covariate shift", - bib_key="tibshirani2019covshift", - bib_status="needs_bib_if_cited", - canonical_url="https://arxiv.org/abs/1904.06019", - core_concepts="Covariate shift, weighted conformal prediction, likelihood-ratio weights.", - key_claims="Conformal coverage can adapt under covariate shift if weights are known/estimated.", - method_or_evidence="Theory and experiments.", - conclusions="Core source for future source/shift gates.", - figures_tables_useful="Use assumption table for source-shift appendix.", - limitations="Requires credible density-ratio/weight estimation.", - crpto_value="Future/caveat only.", - extended_lab_value="Source/shift conformal lane.", - quarto_destination="agenda extendida CRPTO/tesis online/source caveat.", - decision="append_extended_source_shift", - tags=("conformal", "shift"), - ), - "supplement/Yang Jin 2026 - Multi-Distribution Robust Conformal Prediction.pdf": r( - title="Multi-Distribution Robust Conformal Prediction", - authors="Yuqi Yang; Ying Jin", - year="2026", - status="arXiv preprint", - primary_domain="multi-distribution robust conformal prediction", - bib_key="yang2026multidistribution", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2601.02998", - core_concepts="Multiple source distributions, robust coverage, max-p aggregation.", - key_claims="Finite-sample coverage can be made robust over multiple distributions/mixtures.", - method_or_evidence="Theory preprint.", - conclusions="Strong source-robustness future-work citation.", - figures_tables_useful="Use source distribution diagram if agenda extendida CRPTO/tesis source appendix expands.", - limitations="Current project has retrospective source proxies, not true multi-source deployment.", - crpto_value="Future-work context.", - extended_lab_value="Source-governance lane and caveat.", - quarto_destination="CRPTO future work; agenda extendida CRPTO/tesis source caveat.", - decision="append_future_work", - tags=("conformal", "source", "shift"), - ), - "supplement/Yeh et al 2025 - Conformal Risk Training.pdf": r( - title="Conformal Risk Training", - authors="Christopher Yeh; Nicolas Christianson; Adam Wierman; Yisong Yue", - year="2025", - status="NeurIPS 2025", - primary_domain="end-to-end conformal risk training", - bib_key="yeh2025training", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2510.08748", - core_concepts="Conformal risk training, OCE/CVaR, differentiable conformal risk.", - key_claims="Conformal risk can be optimized end-to-end, including OCE-style risks.", - method_or_evidence="Method and experiments.", - conclusions="Future direction for replacing diagnostic tail risk with optimized conformal risk.", - figures_tables_useful="Use risk-training diagram as future-work context only.", - limitations="Current CRPTO is post-hoc; implementing CRT would change method.", - crpto_value="Future-work contrast.", - extended_lab_value="Candidate composite decision-risk training lane.", - quarto_destination="CRPTO future work; agenda extendida CRPTO/tesis future gates.", - decision="park_future_work", - tags=("conformal", "risk_control", "end_to_end"), - ), - "supplement/Yeh et al 2026 - End-to-End Conformal Calibration for Optimization Under Uncertainty.pdf": r( - title="End-to-End Conformal Calibration for Optimization Under Uncertainty", - authors="Christopher Yeh; Nicolas Christianson; Alan Wu; Adam Wierman; Yisong Yue", - year="2025", - status="TMLR; arXiv v2 in 2026", - primary_domain="end-to-end conformal calibration for optimization", - bib_key="yeh2026", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2409.20534", - core_concepts="End-to-end conformal calibration, downstream optimization, learned uncertainty sets.", - key_claims="Calibration can be optimized for decision usefulness while retaining validity.", - method_or_evidence="Method and experiments.", - conclusions="Confirms future direction but not current post-hoc CRPTO.", - figures_tables_useful="Use related-work contrast table only.", - limitations="Different training stack and not credit-specific.", - crpto_value="Future-work frontier.", - extended_lab_value="Lane 3/end-to-end context.", - quarto_destination="CRPTO future work; agenda extendida CRPTO/tesis future gates.", - decision="park_future_work", - tags=("conformal", "end_to_end"), - ), - "supplement/Zhou Orfanoudaki Zhu 2025 - Conformalized Decision Risk Assessment.pdf": r( - title="Conformalized Decision Risk Assessment", - authors="Wenbin Zhou; Agni Orfanoudaki; Shixiang Zhu", - year="2026", - status="ICLR 2026 / arXiv 2025", - primary_domain="decision risk certificates", - bib_key="zhou2025credo", - bib_status="added_2026_06_06", - canonical_url="https://arxiv.org/abs/2505.13243", - core_concepts="CREDO, decision risk certificates, inverse optimization, conformalized risk estimation.", - key_claims="A candidate decision can receive a distribution-free upper bound on probability of suboptimality.", - method_or_evidence="Conformal + inverse optimization framework.", - conclusions="Very relevant for agenda extendida CRPTO/tesis auditability; not part of current CRPTO champion.", - figures_tables_useful="Use risk-certificate diagram as agenda extendida CRPTO/tesis future audit inspiration.", - limitations="Different problem geometry; no Lending Club implementation.", - crpto_value="Future auditability contrast.", - extended_lab_value="Strong future lane for decision certificate rather than policy promotion.", - quarto_destination="agenda extendida CRPTO/tesis future gate register.", - decision="park_future_work", - action_required="completed_bib_quarto_patch", - verification_status="web_primary_and_local_pdf", - tags=("conformal", "decision", "auditability"), - ), - "supplement/Zhou Zhu 2025 - Calibrating Decision Robustness via Inverse Conformal Risk Control.pdf": r( - title="Calibrating Decision Robustness via Inverse Conformal Risk Control", - authors="Yang Zhou; Ruohan Zhu", - year="2026", - status="arXiv preprint", - primary_domain="inverse conformal risk control", - bib_key="zhou2026creme", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2510.07750", - core_concepts="Inverse CRC, decision robustness, robustness calibration.", - key_claims="Decision robustness can be calibrated through inverse conformal risk-control ideas.", - method_or_evidence="Theory/method preprint.", - conclusions="Relevant future lane for robustness calibration.", - figures_tables_useful="Use only as future-work contrast.", - limitations="Not implemented and not credit-specific.", - crpto_value="Future-work context.", - extended_lab_value="Decision robustness gate candidate.", - quarto_destination="agenda extendida CRPTO/tesis future gates.", - decision="park_future_work", - tags=("conformal", "robustness"), - ), - "tesis/Babaei Bamdad 2020 - Multi-Objective Investment Recommendation in P2P Lending.pdf": r( - title="A multi-objective instance-based decision support system for investment recommendation in peer-to-peer lending", - authors="Gholamreza Babaei; Shahrooz Bamdad", - year="2020", - status="Journal article", - primary_domain="P2P investment recommendation", - bib_key="babaei2020p2p", - bib_status="needs_bib_if_cited", - core_concepts="P2P lending, multi-objective recommendation, risk-return tradeoff, NPV.", - key_claims="Decision-support systems can optimize investment recommendations over risk and return.", - method_or_evidence="Applied recommendation/optimization experiment.", - conclusions="Useful thesis context for digital lending optimization.", - figures_tables_useful="Use risk-return recommendation setup as background only.", - limitations="No conformal guarantees.", - crpto_value="Credit-domain context.", - extended_lab_value="Digital-lending optimization context.", - quarto_destination="Thesis/agenda extendida CRPTO/tesis context.", - decision="context_credit_domain", - tags=("credit", "p2p", "optimization"), - ), - "tesis/Basel Committee 2015 - Guidance on Credit Risk and Expected Credit Losses.pdf": r( - title="Guidance on credit risk and accounting for expected credit losses", - authors="Basel Committee on Banking Supervision", - year="2015", - status="official supervisory guidance", - primary_domain="ECL / IFRS9 governance", - bib_key="basel2015ecl", - bib_status="added_2026_06_06", - canonical_url="https://www.bis.org/bcbs/publ/d350.htm", - core_concepts="Expected credit losses, credit risk governance, forward-looking information, controls.", - key_claims="ECL estimation needs governance, data quality, forward-looking information and controls.", - method_or_evidence="Supervisory guidance.", - conclusions="Supports IFRS9-inspired boundary and why contractual IFRS9 is not claimed.", - figures_tables_useful="Use guidance checklist for agenda extendida CRPTO/tesis IFRS9 proxy appendix.", - limitations="Open Lending Club data lacks contractual monthly DPD/EAD/recovery infrastructure.", - crpto_value="Not for CRPTO body.", - extended_lab_value="Strong IFRS9/SICR proxy boundary source.", - quarto_destination="agenda extendida CRPTO/tesis IFRS9 appendix; thesis.", - decision="append_extended_ifrs9", - tags=("ifrs9", "governance"), - ), - "tesis/Ben-Tal El Ghaoui Nemirovski 2009 - Robust Optimization.pdf": r( - title="Robust Optimization", - authors="Aharon Ben-Tal; Laurent El Ghaoui; Arkadi Nemirovski", - year="2009", - status="Princeton University Press book", - primary_domain="robust optimization foundations", - bib_key="bental2009robust", - bib_status="needs_bib_if_cited", - canonical_url="https://press.princeton.edu/books/hardcover/9780691143682/robust-optimization", - core_concepts="Uncertainty sets, robust counterparts, tractability, convex robust optimization.", - key_claims="Robust optimization provides a systematic language for feasible decisions under uncertainty.", - method_or_evidence="Foundational monograph.", - conclusions="Use as deep thesis foundation; CRPTO body can rely on shorter RO citations.", - figures_tables_useful="Use formulations/definitions, not figures.", - limitations="Very broad; not conformal or credit-specific.", - crpto_value="Theoretical background for thesis, optional paper appendix.", - extended_lab_value="Context only.", - quarto_destination="Thesis robust optimization chapter.", - decision="thesis_foundation", - tags=("robust_optimization", "book"), - ), - "tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf": r( - title="How Costly is Noise? Data and Disparities in Consumer Credit", - authors="Laura Blattner; Scott Nelson", - year="2021", - status="working paper / credit economics", - primary_domain="credit data noise / disparities", - bib_key="blattner2021noise", - bib_status="needs_bib_if_cited", - core_concepts="Data noise, credit scores, disparities, unequal information quality.", - key_claims="Noisy credit data can create unequal access and distort credit allocations.", - method_or_evidence="Empirical/economic analysis.", - conclusions="Important governance caveat: model performance can be constrained by source data quality.", - figures_tables_useful="Use disparity/noise tables as governance appendix inspiration.", - limitations="Not Lending Club CRPTO evidence.", - crpto_value="Threat-to-validity/context.", - extended_lab_value="Strong metric/data governance context.", - quarto_destination="agenda extendida CRPTO/tesis governance appendix; thesis.", - decision="append_extended_governance", - tags=("credit", "data_quality", "equity"), - ), - "tesis/Brevoort Grimm Kambara 2016 - Credit Invisibles and the Unscored.pdf": r( - title="Credit Invisibles and the Unscored", - authors="Kenneth P. Brevoort; Philipp Grimm; Michelle Kambara", - year="2016", - status="Cityscape / CFPB research", - primary_domain="credit invisibles / unscored consumers", - bib_key="brevoort2016creditinvisibles", - bib_status="needs_bib_if_cited", - core_concepts="Credit invisibles, unscored consumers, data availability, inclusion.", - key_claims="Large consumer segments lack conventional scores or have weak scoring coverage.", - method_or_evidence="Consumer credit data analysis.", - conclusions="Supports why open accepted-loan data cannot make broad credit-access claims.", - figures_tables_useful="Use population breakdowns as motivation only.", - limitations="Not a CRPTO method source.", - crpto_value="Credit-market context and scope limitation.", - extended_lab_value="Fairness/source governance context.", - quarto_destination="Thesis/agenda extendida CRPTO/tesis governance.", - decision="context_governance", - tags=("credit", "inclusion", "governance"), - ), - "tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf": r( - title="Conformal Prediction Sets Improve Human Decision Making", - authors="Jesse Cresswell and coauthors", - year="2024", - status="preprint / empirical decision support", - primary_domain="human decision support with CP", - bib_key="cresswell2024", - bib_status="needs_bib_if_cited", - core_concepts="Human-AI decision making, prediction sets, uncertainty communication.", - key_claims="Conformal sets can improve human decisions when uncertainty is communicated well.", - method_or_evidence="Human-subjects/decision experiment.", - conclusions="Useful for discussing auditability and committee-facing uncertainty.", - figures_tables_useful="Use decision-support figures as inspiration for MRM/committee communication.", - limitations="Not credit portfolio optimization.", - crpto_value="Motivation for uncertainty-as-decision input.", - extended_lab_value="Governance/human decision appendix context.", - quarto_destination="agenda extendida CRPTO/tesis governance; thesis discussion.", - decision="context_decision_support", - tags=("conformal", "human_decision", "governance"), - ), - "tesis/Einbinder et al 2024 - Label Noise Robustness of Conformal Prediction.pdf": r( - title="Label Noise Robustness of Conformal Prediction", - authors="Bat-Sheva Einbinder; Shai Feldman; Stephen Bates; Anastasios N. Angelopoulos; Asaf Gendler; Yaniv Romano", - year="2024", - status="Journal of Machine Learning Research", - primary_domain="label noise / conformal robustness", - bib_key="einbinder2024labelnoise", - bib_status="existing", - canonical_url="https://jmlr.org/papers/v25/23-1549.html", - core_concepts="Label noise, conformal robustness, noisy outcomes, validity under imperfect labels.", - key_claims="Conformal prediction has robustness properties under forms of label noise.", - method_or_evidence="Theory and experiments.", - conclusions="Useful threat-to-validity note for default labels and hardened shift tests.", - figures_tables_useful="Use noise taxonomy for appendix caveat.", - limitations="Does not validate all Lending Club label issues.", - crpto_value="Threat-to-validity support.", - extended_lab_value="Data/label governance appendix.", - quarto_destination="CRPTO limitations; agenda extendida CRPTO/tesis governance.", - decision="append_claim_boundary", - tags=("conformal", "label_noise"), - ), - "tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf": r( - title="Predictably Unequal? The Effects of Machine Learning on Credit Markets", - authors="Andreas Fuster; Paul Goldsmith-Pinkham; Tarun Ramadorai; Ansgar Walther", - year="2022", - status="Journal of Finance", - primary_domain="ML credit markets / equity", - bib_key="fuster2022predictably", - bib_status="added_2026_06_06", - canonical_url="https://doi.org/10.1111/jofi.13090", - core_concepts="ML credit screening, distributional impacts, mortgage markets, disparities.", - key_claims="More flexible ML can change rate disparities and who benefits from credit-market technology.", - method_or_evidence="Theory and empirical mortgage-market analysis.", - conclusions="Critical fairness/equity context: better predictive technology does not automatically imply equitable outcomes.", - figures_tables_useful="Use disparity mechanism figures/tables in governance discussion.", - limitations="Mortgage market, not Lending Club; no protected attributes in current project.", - crpto_value="Motivation/caveat, not method evidence.", - extended_lab_value="Strong governance/fairness boundary source.", - quarto_destination="agenda extendida CRPTO/tesis governance appendix; CRPTO ethics/motivation if space permits.", - decision="append_extended_governance", - action_required="completed_bib_quarto_patch", - verification_status="web_primary_and_local_pdf", - tags=("credit", "equity", "ml"), - ), - "tesis/Kato 2024 - Conformal Predictive Portfolio Selection.pdf": r( - title="Conformal Predictive Portfolio Selection", - authors="Masahiro Kato", - year="2025", - status="arXiv preprint", - primary_domain="conformal portfolio selection", - bib_key="kato2025", - bib_status="existing", - canonical_url="https://arxiv.org/abs/2410.16333", - core_concepts="Portfolio selection, conformal predictive sets, financial decisions.", - key_claims="Conformal prediction can support portfolio selection under uncertainty.", - method_or_evidence="Finance/portfolio method preprint.", - conclusions="Relevant neighbor in finance; CRPTO differs by credit PD/funded-set risk.", - figures_tables_useful="Use as finance-related-work table.", - limitations="Not credit lending and not Lending Club.", - crpto_value="Related work context.", - extended_lab_value="Finance portfolio context.", - quarto_destination="CRPTO related work if finance CP paragraph expands.", - decision="context_finance_cp", - tags=("conformal", "portfolio"), - ), - "tesis/Kawasumi Kato Duan 2026 - Conformal Prediction for Ordinal Credit Scoring.pdf": r( - title="Conformal Prediction for Ordinal Credit Scoring", - authors="Ryotaro Kawasumi; Masahiro Kato; Yaqing Duan", - year="2026", - status="arXiv preprint", - primary_domain="ordinal credit scoring / conformal prediction", - bib_key="kawasumi2026ordinal", - bib_status="needs_bib_if_cited", - core_concepts="Ordinal credit scoring, conformal prediction, rating categories.", - key_claims="Conformal methods can be adapted to ordinal credit-score outputs.", - method_or_evidence="Short method preprint.", - conclusions="Useful for future grade/rating-set extension, not current PD interval CRPTO.", - figures_tables_useful="Use only as future-work note.", - limitations="No portfolio optimization and very short current evidence.", - crpto_value="Future credit conformal context.", - extended_lab_value="Ordinal score/rating future lane.", - quarto_destination="Thesis future work.", - decision="park_future_work", - tags=("conformal", "credit", "ordinal"), - ), - "tesis/Khandani Kim Lo 2010 - Consumer Credit-Risk Models via Machine-Learning Algorithms.pdf": r( - title="Consumer Credit-Risk Models via Machine-Learning Algorithms", - authors="Amir E. Khandani; Adlar J. Kim; Andrew W. Lo", - year="2010", - status="Journal of Banking & Finance", - primary_domain="consumer credit ML", - bib_key="khandani2010consumer", - bib_status="added_2026_06_06", - canonical_url="https://doi.org/10.1016/j.jbankfin.2010.06.001", - core_concepts="Consumer credit risk, nonlinear ML, transaction data, economic value of risk forecasts.", - key_claims="ML forecasts can materially improve consumer credit-risk management and economic decisions.", - method_or_evidence="Empirical credit-card portfolio study.", - conclusions="Classic support for economic evaluation of credit ML, not just classification metrics.", - figures_tables_useful="Use economic-benefit tables as background for decision-value framing.", - limitations="Credit cards/bank data, not LC/CRPTO.", - crpto_value="Historical credit-ML decision-value context.", - extended_lab_value="Metric/economic governance context.", - quarto_destination="CRPTO motivation or thesis background.", - decision="context_credit_domain", - action_required="completed_bib_quarto_patch", - verification_status="web_primary_and_local_pdf", - tags=("credit", "ml", "economic_value"), - ), - "tesis/Noguer i Alonso 2024 - Conformal Portfolio Optimization.pdf": r( - title="Conformal Portfolio Optimization", - authors="Miquel Noguer i Alonso", - year="2024", - status="SSRN preprint", - primary_domain="conformal portfolio optimization", - bib_key="noguer2024portfolio", - bib_status="existing", - canonical_url="https://papers.ssrn.com/sol3/papers.cfm?abstract_id=5011129", - core_concepts="Conformal prediction, portfolio optimization, finance uncertainty.", - key_claims="Conformal intervals can support portfolio decisions in financial assets.", - method_or_evidence="Short finance preprint.", - conclusions="Finance-adjacent context, but CRPTO's credit/funded-set bound is different.", - figures_tables_useful="Use only as related-work mention.", - limitations="Short, non-credit, not a robust credit allocation benchmark.", - crpto_value="Related work context.", - extended_lab_value="Finance portfolio context.", - quarto_destination="Related work if finance CP paragraph expands.", - decision="context_finance_cp", - tags=("conformal", "portfolio"), - ), -} - - -MATRIX_FIELDS = [ - "folder", - "file", - "relative_path", - "title", - "authors", - "year", - "status", - "primary_domain", - "bib_key", - "bib_status", - "canonical_url", - "verification_status", - "pages", - "word_count", - "figure_caption_count", - "table_caption_count", - "core_concepts", - "key_claims", - "method_or_evidence", - "conclusions", - "figures_tables_useful", - "limitations", - "crpto_value", - "extended_lab_value", - "quarto_destination", - "decision", - "action_required", - "evidence_gate", - "artifact_sink", - "stop_rule", - "implementation_or_experiment", - "notes", - "tags", -] - -CAPTION_FIELDS = [ - "relative_path", - "caption_type", - "caption_index", - "caption_text", - "caption_index_truncated", -] - -CURATED_VISUAL_FIELDS = [ - "relative_path", - "caption_type", - "caption_index", - "caption_text", - "editorial_sink", - "why_useful", - "use_rule", - "claim_boundary", -] - -CURATED_VISUAL_SPECS = [ - { - "relative_path": "paper/Hu et al 2026 - Conformal Robustness Control.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis robustness-certificate appendix", - "why_useful": "Shows the conceptual contrast between conventional CRO and CRC-style robustness control.", - "use_rule": "Use as a schematic inspiration only; do not reproduce the figure without permission.", - "claim_boundary": "Future gate over V/violation, not current CRPTO evidence.", - }, - { - "relative_path": "supplement/Barber Candes Ramdas Tibshirani 2023 - Conformal Prediction Beyond Exchangeability.pdf", - "caption_type": "figure", - "caption_index": 2, - "editorial_sink": "agenda extendida CRPTO/tesis source/shift conformal appendix", - "why_useful": "Coverage-width caption is useful for explaining why non-exchangeability gates must report both validity and efficiency.", - "use_rule": "Use for assumptions table and reviewer response framing; no new run.", - "claim_boundary": "Requires declared weighting or shift structure.", - }, - { - "relative_path": "supplement/Tibshirani et al 2019 - Conformal Prediction Under Covariate Shift.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis source/shift conformal appendix", - "why_useful": "Canonical weighted-conformal covariate-shift coverage caption for density-ratio caveats.", - "use_rule": "Use only to motivate weight-estimation assumptions.", - "claim_boundary": "No source-shift deployment claim without credible weights.", - }, - { - "relative_path": "supplement/Bhattacharyya Barber 2026 - Group-Weighted Conformal Prediction.pdf", - "caption_type": "figure", - "caption_index": 3, - "editorial_sink": "agenda extendida CRPTO/tesis group/source governance appendix", - "why_useful": "Compares older weighted CP guarantees to the new group-weighted guarantee.", - "use_rule": "Use as candidate-experiment design input, not as evidence of LC performance.", - "claim_boundary": "Needs target/source group weights before any run.", - }, - { - "relative_path": "supplement/Liu Levis Normand Han 2024 - Multi-Source Conformal Inference Under Distribution Shift.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis multi-source conformal appendix", - "why_useful": "Illustrates multi-source calibration structure and source-combination logic.", - "use_rule": "Use for taxonomy only.", - "claim_boundary": "LC source families are retrospective, not validated external sources.", - }, - { - "relative_path": "supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis utility-directed CP future gate", - "why_useful": "Contrasts standard CP with utility-directed CP in a decision-aware frame.", - "use_rule": "Use to specify future decision-loss screen.", - "claim_boundary": "No selector change without fixed loss and coverage gate.", - }, - { - "relative_path": "supplement/Cortes-Gomez et al 2025 - Utility-Directed Conformal Prediction.pdf", - "caption_type": "figure", - "caption_index": 4, - "editorial_sink": "agenda extendida CRPTO/tesis utility-directed CP future gate", - "why_useful": "Connects base model accuracy to downstream optimization value.", - "use_rule": "Use as appendix motivation for utility reporting.", - "claim_boundary": "Not evidence that CRPTO currently optimizes utility-directed sets.", - }, - { - "relative_path": "supplement/Guan 2023 - Localized Conformal Prediction.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis localized conformal candidate appendix", - "why_useful": "Visualizes global versus localized conformal bands.", - "use_rule": "Use to frame a localized-vs-Mondrian challenger only if reviewer asks.", - "claim_boundary": "No current localized calibration design.", - }, - { - "relative_path": "paper/Chi Ding Peng 2019 - Data-Driven Robust Credit Portfolio Optimization in P2P Lending.pdf", - "caption_type": "figure", - "caption_index": 2, - "editorial_sink": "CRPTO/agenda extendida CRPTO/tesis applied robust-credit related work", - "why_useful": "Performance-comparison caption anchors the P2P robust portfolio context.", - "use_rule": "Use to motivate domain fit; do not treat as comparable champion evidence.", - "claim_boundary": "Different data, objective and uncertainty construction.", - }, - { - "relative_path": "paper/Albanesi Vamossy 2024 - Credit Scores Performance and Equity.pdf", - "caption_type": "figure", - "caption_index": 4, - "editorial_sink": "agenda extendida CRPTO/tesis score/equity governance appendix", - "why_useful": "Gini-over-time caption is a useful template for score-vs-model governance reporting.", - "use_rule": "Use as dashboard/table design inspiration only.", - "claim_boundary": "Not a legal fair-lending claim.", - }, - { - "relative_path": "tesis/Fuster et al 2022 - Predictably Unequal Effects of Machine Learning on Credit Markets.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "agenda extendida CRPTO/tesis equity/noise governance appendix", - "why_useful": "Clarifies how better prediction technology can create group-specific effects.", - "use_rule": "Use to frame equity caveats in prose.", - "claim_boundary": "No protected-attribute causal claim in LC.", - }, - { - "relative_path": "tesis/Blattner Nelson 2021 - How Costly is Noise Data and Disparities in Consumer Credit.pdf", - "caption_type": "table", - "caption_index": 4, - "editorial_sink": "agenda extendida CRPTO/tesis data-quality/equity governance appendix", - "why_useful": "Links predictive-performance gaps to disadvantaged consumers and data quality.", - "use_rule": "Use as motivation for source/noise limitations.", - "claim_boundary": "Project lacks the paper's protected-group and lender data.", - }, - { - "relative_path": "supplement/CFPB 2014 - Public Information Proxy Race Ethnicity.pdf", - "caption_type": "table", - "caption_index": 5, - "editorial_sink": "agenda extendida CRPTO/tesis fairness-proxy boundary appendix", - "why_useful": "Shows proxy-probability validation needs richer race/ethnicity methodology.", - "use_rule": "Use only to justify the stop rule on legal fair-lending claims.", - "claim_boundary": "LC lacks surname/fine geography and protected labels.", - }, - { - "relative_path": "tesis/Cresswell et al 2024 - Conformal Prediction Sets Improve Human Decision Making.pdf", - "caption_type": "figure", - "caption_index": 2, - "editorial_sink": "agenda extendida CRPTO/tesis governance/committee communication appendix", - "why_useful": "Human-facing uncertainty display can inspire committee explanation design.", - "use_rule": "Use for communication patterns, not as credit decision evidence.", - "claim_boundary": "Different task, subjects and utility function.", - }, -] - - -def run_text(command: list[str], timeout: int = 180) -> str: - try: - result = subprocess.run( - command, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=timeout, - ) - except (FileNotFoundError, subprocess.TimeoutExpired): - return "" - return result.stdout - - -def pdf_pages(path: Path) -> int: - info = run_text(["pdfinfo", str(path)], timeout=30) - for line in info.splitlines(): - if line.startswith("Pages:"): - value = line.split(":", 1)[1].strip() - return int(value) if value.isdigit() else 0 - return 0 - - -def pdf_text(path: Path) -> str: - return run_text(["pdftotext", "-layout", str(path), "-"], timeout=420) - - -def normalize_space(value: str) -> str: - return re.sub(r"\s+", " ", value).strip() - - -def extract_caption_rows( - relative_path: str, text: str, max_per_pdf: int = 14 -) -> tuple[list[dict[str, object]], int, int]: - caption_rows: list[dict[str, object]] = [] - lines = [normalize_space(line) for line in text.splitlines()] - pattern = re.compile(r"^(Figure|Fig\.|Table|Tabla)\s+[A-Z]?\d+[\.: -]+(.+)", re.IGNORECASE) - figure_count = 0 - table_count = 0 - seen: set[str] = set() - for line in lines: - match = pattern.match(line) - if not match: - continue - caption_type = ( - "table" if match.group(1).lower().startswith(("table", "tabla")) else "figure" - ) - if caption_type == "figure": - figure_count += 1 - else: - table_count += 1 - caption = normalize_space(line) - dedupe_key = f"{caption_type}:{caption[:180]}" - if dedupe_key in seen or len(caption) < 20: - continue - seen.add(dedupe_key) - if len(caption_rows) < max_per_pdf: - caption_rows.append( - { - "relative_path": relative_path, - "caption_type": caption_type, - "caption_index": len(caption_rows) + 1, - "caption_text": caption, - "caption_index_truncated": "false", - } - ) - if len(seen) > max_per_pdf: - caption_rows.append( - { - "relative_path": relative_path, - "caption_type": "truncation_note", - "caption_index": max_per_pdf + 1, - "caption_text": f"{len(seen) - max_per_pdf} additional figure/table captions omitted from this compact index.", - "caption_index_truncated": "true", - } - ) - return caption_rows, figure_count, table_count - - -def write_csv(path: Path, rows: list[dict[str, object]], fields: list[str]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") - writer.writeheader() - for row in rows: - writer.writerow({field: row.get(field, "") for field in fields}) - - -def build_curated_visual_rows(caption_rows: list[dict[str, object]]) -> list[dict[str, object]]: - lookup = { - ( - str(row["relative_path"]), - str(row["caption_type"]), - int(str(row["caption_index"])), - ): row - for row in caption_rows - if str(row.get("caption_index", "")).isdigit() - } - - curated: list[dict[str, object]] = [] - missing: list[str] = [] - for spec in CURATED_VISUAL_SPECS: - key = ( - str(spec["relative_path"]), - str(spec["caption_type"]), - int(str(spec["caption_index"])), - ) - caption = lookup.get(key) - if caption is None: - missing.append(f"{key[0]} / {key[1]} #{key[2]}") - continue - curated.append( - { - "relative_path": caption["relative_path"], - "caption_type": caption["caption_type"], - "caption_index": caption["caption_index"], - "caption_text": caption["caption_text"], - "editorial_sink": spec["editorial_sink"], - "why_useful": spec["why_useful"], - "use_rule": spec["use_rule"], - "claim_boundary": spec["claim_boundary"], - } - ) - - if missing: - raise SystemExit("Missing curated caption rows: " + "; ".join(missing)) - return curated - - -def build_rows(source_dir: Path) -> tuple[list[dict[str, object]], list[dict[str, object]]]: - pdfs = sorted(path for path in source_dir.rglob("*.pdf") if path.is_file()) - rows: list[dict[str, object]] = [] - caption_rows: list[dict[str, object]] = [] - - missing = [ - str(path.relative_to(source_dir)) - for path in pdfs - if str(path.relative_to(source_dir)) not in REVIEWS - ] - extra = [key for key in REVIEWS if not (source_dir / key).exists()] - if missing or extra: - message = [] - if missing: - message.append("Missing curated reviews for: " + ", ".join(missing)) - if extra: - message.append("Curated reviews without PDF: " + ", ".join(extra)) - raise SystemExit("\n".join(message)) - - for path in pdfs: - relative_path = str(path.relative_to(source_dir)) - review = REVIEWS[relative_path] - text = pdf_text(path) - words = len(re.findall(r"\b\w+\b", text)) - captions, figure_count, table_count = extract_caption_rows(relative_path, text) - caption_rows.extend(captions) - rows.append( - { - "folder": Path(relative_path).parts[0], - "file": path.name, - "relative_path": relative_path, - "title": review.title, - "authors": review.authors, - "year": review.year, - "status": review.status, - "primary_domain": review.primary_domain, - "bib_key": review.bib_key, - "bib_status": review.bib_status, - "canonical_url": review.canonical_url, - "verification_status": review.verification_status, - "pages": pdf_pages(path), - "word_count": words, - "figure_caption_count": figure_count, - "table_caption_count": table_count, - "core_concepts": review.core_concepts, - "key_claims": review.key_claims, - "method_or_evidence": review.method_or_evidence, - "conclusions": review.conclusions, - "figures_tables_useful": review.figures_tables_useful, - "limitations": review.limitations, - "crpto_value": review.crpto_value, - "extended_lab_value": review.extended_lab_value, - "quarto_destination": review.quarto_destination, - "decision": review.decision, - "action_required": review.action_required, - "evidence_gate": review.evidence_gate, - "artifact_sink": review.artifact_sink, - "stop_rule": review.stop_rule, - "implementation_or_experiment": review.implementation_or_experiment, - "notes": review.notes, - "tags": ";".join(review.tags), - } - ) - return rows, caption_rows - - -def markdown_table( - rows: list[dict[str, object]], fields: list[str], max_rows: int | None = None -) -> str: - selected = rows[:max_rows] if max_rows is not None else rows - header = "| " + " | ".join(fields) + " |" - divider = "| " + " | ".join("---" for _ in fields) + " |" - body = [] - for row in selected: - body.append( - "| " + " | ".join(str(row.get(field, "")).replace("|", "/") for field in fields) + " |" - ) - return "\n".join([header, divider, *body]) - - -def _counter_rows(counter: Counter[str], key_field: str) -> list[dict[str, object]]: - return [{key_field: key, "n": value} for key, value in sorted(counter.items())] - - -def _section(title: str, *body: str) -> list[str]: - return ["", f"## {title}", "", *body] - - -def _decision_groups(rows: list[dict[str, object]]) -> dict[str, list[dict[str, object]]]: - return { - "promote": [row for row in rows if str(row["decision"]).startswith("promote")], - "crpto_appendix": [ - row - for row in rows - if str(row["decision"]).startswith(("append_crpto", "append_comparator", "append_tail")) - ], - "extended": [ - row - for row in rows - if "extended" in str(row["decision"]) or "mixed_diagnostic" in str(row["decision"]) - ], - "future": [ - row - for row in rows - if "future" in str(row["decision"]) or "candidate" in str(row["decision"]) - ], - "experiments": [ - row for row in rows if str(row["action_required"]).startswith("experiment_") - ], - } - - -def _inventory_sections(rows: list[dict[str, object]]) -> list[str]: - folder_counts = Counter(str(row["folder"]) for row in rows) - decision_counts = Counter(str(row["decision"]) for row in rows) - action_counts = Counter(str(row["action_required"]) for row in rows) - domain_counts = Counter(str(row["primary_domain"]) for row in rows) - return [ - *_section( - "Inventario", - markdown_table( - [ - {"folder": folder, "pdfs": count} - for folder, count in sorted(folder_counts.items()) - ], - ["folder", "pdfs"], - ), - ), - *_section( - "Decisiones por destino editorial", - markdown_table(_counter_rows(decision_counts, "decision"), ["decision", "n"]), - ), - *_section( - "Acciones requeridas", - markdown_table( - _counter_rows(action_counts, "action_required"), ["action_required", "n"] - ), - ), - *_section( - "Familias conceptuales", - markdown_table(_counter_rows(domain_counts, "primary_domain"), ["primary_domain", "n"]), - ), - ] - - -def _paper_crpto_sections(groups: dict[str, list[dict[str, object]]]) -> list[str]: - return [ - *_section( - "Lectura integrada para Paper CRPTO", - "El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son:", - "", - markdown_table( - groups["promote"], - ["relative_path", "bib_key", "core_concepts", "crpto_value", "limitations"], - ), - "", - "Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y limites de claim:", - "", - markdown_table( - groups["crpto_appendix"], - ["relative_path", "bib_key", "decision", "crpto_value", "figures_tables_useful"], - ), - ), - *_section( - "Lectura integrada para agenda extendida CRPTO/tesis", - "La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO:", - "", - markdown_table( - groups["extended"], - ["relative_path", "decision", "extended_lab_value", "evidence_gate", "stop_rule"], - ), - ), - *_section( - "Experimentos evidence-gated", - "Esta seccion lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado.", - "", - markdown_table( - groups["experiments"], - [ - "relative_path", - "action_required", - "implementation_or_experiment", - "evidence_gate", - "artifact_sink", - "stop_rule", - ], - ), - ), - ] - - -def _visual_and_matrix_sections( - rows: list[dict[str, object]], - groups: dict[str, list[dict[str, object]]], - curated_visual_rows: list[dict[str, object]], -) -> list[str]: - return [ - *_section( - "Curaduria de figuras/tablas", - "El indice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduria siguiente solo define que visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers.", - "", - markdown_table( - curated_visual_rows, - [ - "relative_path", - "caption_type", - "caption_index", - "editorial_sink", - "why_useful", - "claim_boundary", - ], - ), - ), - *_section( - "Future work y stop rules", - markdown_table( - groups["future"], - ["relative_path", "decision", "crpto_value", "extended_lab_value", "stop_rule"], - ), - ), - *_section( - "Matriz paper-by-paper", - "La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, metodo/evidencia, conclusion, figuras/tablas utiles, limitacion y destino editorial. Para auditoria operativa usar el CSV completo.", - "", - markdown_table( - rows, - [ - "relative_path", - "title", - "status", - "core_concepts", - "key_claims", - "conclusions", - "figures_tables_useful", - "limitations", - "decision", - "action_required", - ], - ), - ), - ] - - -def _bibliography_and_closeout_sections(rows: list[dict[str, object]]) -> list[str]: - bib_counts = Counter(str(row["bib_status"]) for row in rows) - return [ - *_section( - "Control bibliografico", - "Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explicitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografia.", - "", - markdown_table(_counter_rows(bib_counts, "bib_status"), ["bib_status", "n"]), - ), - *_section( - "Fronteras que permanecen falsas", - "- CRPTO no reclama legal fair lending con atributos protegidos directos.", - "- CRPTO no implementa IFRS9 contractual.", - "- agenda extendida CRPTO/tesis no reclama CATE policy value.", - "- agenda extendida CRPTO/tesis no reclama online deployment.", - "- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto.", - "- SPO+/DFL puede ganar regret, pero no reemplaza la garantia/auditabilidad CRPTO.", - ), - *_section( - "Cierre", - "La auditoria agrega valor como integracion bibliografica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empirica del paper actual.", - ), - ] - - -def _audit_lines( - rows: list[dict[str, object]], - matrix_path: Path, - caption_path: Path, - curated_visual_path: Path, - curated_visual_rows: list[dict[str, object]], -) -> list[str]: - groups = _decision_groups(rows) - lines = [ - "# Papers_tesis Deep Audit - 2026-06-06", - "", - "## Resumen ejecutivo", - "", - f"Esta auditoria cubre `61` PDFs locales en `{DEFAULT_SOURCE_DIR}` y fue generada para el corte `{REPORT_DATE}` con `scripts/build_papers_tesis_deep_audit.py`.", - "", - "La decision central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoria, related work, appendices y limites de claim. La agenda extendida CRPTO/tesis absorbe el material que si pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance.", - "", - "Artefactos generados:", - "", - f"- Matriz fuente: `{matrix_path}`", - f"- Indice compacto de captions: `{caption_path}`", - f"- Curaduria de visual sinks: `{curated_visual_path}`", - ] - lines.extend(_inventory_sections(rows)) - lines.extend(_paper_crpto_sections(groups)) - lines.extend(_visual_and_matrix_sections(rows, groups, curated_visual_rows)) - lines.extend(_bibliography_and_closeout_sections(rows)) - lines.append("") - return lines - - -def write_audit( - path: Path, - rows: list[dict[str, object]], - matrix_path: Path, - caption_path: Path, - curated_visual_path: Path, - curated_visual_rows: list[dict[str, object]], -) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - lines = _audit_lines(rows, matrix_path, caption_path, curated_visual_path, curated_visual_rows) - path.write_text("\n".join(lines), encoding="utf-8") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR) - parser.add_argument("--audit-path", type=Path, default=DEFAULT_AUDIT_PATH) - parser.add_argument("--matrix-path", type=Path, default=DEFAULT_MATRIX_PATH) - parser.add_argument("--caption-path", type=Path, default=DEFAULT_CAPTION_PATH) - parser.add_argument("--curated-visual-path", type=Path, default=DEFAULT_CURATED_VISUAL_PATH) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - rows, captions = build_rows(args.source_dir) - curated_visuals = build_curated_visual_rows(captions) - write_csv(args.matrix_path, rows, MATRIX_FIELDS) - write_csv(args.caption_path, captions, CAPTION_FIELDS) - write_csv(args.curated_visual_path, curated_visuals, CURATED_VISUAL_FIELDS) - write_audit( - args.audit_path, - rows, - args.matrix_path, - args.caption_path, - args.curated_visual_path, - curated_visuals, - ) - print(f"Wrote {len(rows)} source rows to {args.matrix_path}") - print(f"Wrote {len(captions)} caption rows to {args.caption_path}") - print(f"Wrote {len(curated_visuals)} curated visual rows to {args.curated_visual_path}") - print(f"Wrote audit memo to {args.audit_path}") - - -if __name__ == "__main__": - main() diff --git a/scripts/build_params_view.py b/scripts/build_params_view.py deleted file mode 100644 index 39583ed..0000000 --- a/scripts/build_params_view.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Build the DVC ``params.yaml`` view from canonical CRPTO configs. - -The project scripts read ``configs/*.yaml`` and promotion JSON artifacts; DVC -uses ``params.yaml`` as a compact cache-key surface. This helper keeps that -surface reproducible without making DVC stages depend on ad hoc manual edits. -""" - -from __future__ import annotations - -import argparse -import difflib -import json -from pathlib import Path -from typing import Any - -import yaml -from loguru import logger - -ROOT = Path(__file__).resolve().parents[1] - - -def _load_yaml(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - if not isinstance(payload, dict): - raise TypeError(f"{path} must contain a YAML mapping.") - return payload - - -def _load_json(path: Path) -> dict[str, Any]: - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise TypeError(f"{path} must contain a JSON object.") - return payload - - -def build_params_view(root: Path = ROOT) -> dict[str, Any]: - """Return the generated ``params.yaml`` mapping.""" - existing = _load_yaml(root / "params.yaml") - pd_config = _load_yaml(root / "configs" / "crpto_pd_model.yaml") - conformal_config = _load_yaml(root / "configs" / "crpto_conformal_policy.yaml") - optimization_config = _load_yaml(root / "configs" / "crpto_optimization.yaml") - promotion = _load_json(root / "models" / "final_project_promotion.json") - - champion = promotion["final_champion"] - robust_region = promotion["robust_region_summary"] - model_params = pd_config["model"]["params"] - policy = conformal_config["policy"] - portfolio_config = optimization_config["portfolio"] - - existing_paper = dict(existing.get("paper", {}) or {}) - existing_conformal = dict(existing.get("conformal", {}) or {}) - existing_portfolio = dict(existing.get("portfolio", {}) or {}) - - return { - "paper": { - "run_tag": promotion["run_tag"], - "champion_policy": existing_paper.get("champion_policy", champion["label"]), - "region_robust": ( - f"{robust_region['n_alpha01_passers']}/{robust_region['n_unique_policies']}" - ), - }, - "champion": { - "return_robust": round(float(champion["realized_total_return"]), 2), - "v_alpha_001": float(champion["alpha01_weighted_miscoverage_V"]), - "gamma_cp_alpha_001": float(champion["alpha01_gamma_cp"]), - "alpha_exact_pass": bool(champion["alpha01_exact_pass"]), - }, - "pd": { - "model": pd_config["model"]["type"], - "catboost": { - "depth": int(model_params["depth"]), - "iterations": int(model_params["iterations"]), - "learning_rate": float(model_params["learning_rate"]), - }, - "calibration": pd_config["calibration"]["method"], - }, - "conformal": { - "alpha": existing_conformal.get("alpha", 0.01), - "coverage_targets": [ - float(policy["target_coverage_90_min"]), - float(policy["target_coverage_95_min"]), - ], - "mondrian": existing_conformal.get("mondrian", {}), - "variant_selection": existing_conformal.get("variant_selection", {}), - }, - "portfolio": { - "policy_mode": existing_portfolio.get("policy_mode", "bound_aware"), - "uncertainty_aversion": existing_portfolio.get("uncertainty_aversion", 0.0), - "max_concentration": float(portfolio_config["max_concentration"]), - "max_portfolio_pd": float(portfolio_config["max_portfolio_pd"]), - "min_budget_utilization": existing_portfolio.get("min_budget_utilization", 0.5), - }, - "optuna": existing.get("optuna", {}), - "book": existing.get("book", {}), - } - - -def dump_params_view(params: dict[str, Any]) -> str: - """Serialize params deterministically for checks and optional rewrites.""" - return yaml.safe_dump(params, sort_keys=False, allow_unicode=True) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--check", action="store_true", help="Fail if params.yaml is stale.") - parser.add_argument("--write", action="store_true", help="Rewrite params.yaml from sources.") - args = parser.parse_args() - - target = ROOT / "params.yaml" - generated = dump_params_view(build_params_view(ROOT)) - - if args.check: - current = dump_params_view(_load_yaml(target)) - if current != generated: - diff = "\n".join( - difflib.unified_diff( - current.splitlines(), - generated.splitlines(), - fromfile="params.yaml", - tofile="generated", - lineterm="", - ) - ) - raise SystemExit(f"params.yaml is stale:\n{diff}") - logger.info("params.yaml is synchronized with canonical configs.") - return - - if args.write: - target.write_text(generated, encoding="utf-8") - logger.info("Wrote {}", target) - return - - logger.info(generated) - - -if __name__ == "__main__": - main() diff --git a/scripts/build_pipeline_results.py b/scripts/build_pipeline_results.py deleted file mode 100644 index 695fc94..0000000 --- a/scripts/build_pipeline_results.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Build canonical pipeline_results artifact from stage outputs. - -Reads canonical artifacts (predictions, conformal intervals, IFRS9 scenarios, -portfolio robustness) and assembles a pipeline_results.pkl summary consumed -by export_streamlit_artifacts.py to produce pipeline_summary.json. - -Usage: - uv run python scripts/build_pipeline_results.py -""" - -from __future__ import annotations - -import pickle -from pathlib import Path -from typing import Any - -import pandas as pd -from loguru import logger -from sklearn.metrics import roc_auc_score - -DATA_DIR = Path("data/processed") -MODEL_DIR = Path("models") - - -def _persist_pipeline_results() -> None: - """Build a pipeline summary artifact from canonical outputs.""" - results: dict[str, Any] = { - "batch_size": 0, - "pd_mean": 0.0, - "pd_auc": 0.0, - "interval_width_mean": 0.0, - "stages": {"S1": 0, "S2": 0, "S3": 0}, - "ecl_expected": 0.0, - "ecl_conservative": 0.0, - "ecl_range": 0.0, - "robust_return": 0.0, - "robust_funded": 0, - "nonrobust_return": 0.0, - "nonrobust_funded": 0, - "price_of_robustness": 0.0, - "pipeline_time_s": 0.0, - } - - preds_path = DATA_DIR / "test_predictions.parquet" - if preds_path.exists(): - preds = pd.read_parquet(preds_path) - if "y_prob_final" in preds.columns: - results["batch_size"] = len(preds) - results["pd_mean"] = float(preds["y_prob_final"].mean()) - if {"y_true", "y_prob_final"}.issubset(preds.columns): - results["pd_auc"] = float(roc_auc_score(preds["y_true"], preds["y_prob_final"])) - - intervals_path = DATA_DIR / "conformal_intervals_mondrian.parquet" - if intervals_path.exists(): - ints = pd.read_parquet(intervals_path) - if "interval_width" in ints.columns: - results["interval_width_mean"] = float(ints["interval_width"].mean()) - elif {"pd_low_90", "pd_high_90"}.issubset(ints.columns): - results["interval_width_mean"] = float((ints["pd_high_90"] - ints["pd_low_90"]).mean()) - elif {"pd_low", "pd_high"}.issubset(ints.columns): - results["interval_width_mean"] = float((ints["pd_high"] - ints["pd_low"]).mean()) - - ifrs9_path = DATA_DIR / "ifrs9_scenario_summary.parquet" - if ifrs9_path.exists(): - ifrs9 = pd.read_parquet(ifrs9_path) - baseline = ifrs9[ifrs9["scenario"] == "baseline"] - severe = ifrs9[ifrs9["scenario"] == "severe"] - if not baseline.empty: - row = baseline.iloc[0] - n_loans = int(row.get("n_loans", 0)) - results["stages"] = { - "S1": int(round(float(row.get("stage1_share", 0.0)) * n_loans)), - "S2": int(round(float(row.get("stage2_share", 0.0)) * n_loans)), - "S3": int(round(float(row.get("stage3_share", 0.0)) * n_loans)), - } - results["ecl_expected"] = float(row.get("total_ecl", 0.0)) - if not severe.empty: - results["ecl_conservative"] = float(severe.iloc[0].get("total_ecl", 0.0)) - results["ecl_range"] = max(0.0, results["ecl_conservative"] - results["ecl_expected"]) - - robust_path = DATA_DIR / "portfolio_robustness_summary.parquet" - if robust_path.exists(): - robust = pd.read_parquet(robust_path) - if not robust.empty: - if "risk_tolerance" in robust.columns: - robust = robust.assign(_dist=(robust["risk_tolerance"] - 0.10).abs()) - row = robust.sort_values("_dist").iloc[0] - else: - row = robust.iloc[0] - results["robust_return"] = float(row.get("best_robust_return", 0.0)) - results["nonrobust_return"] = float(row.get("baseline_nonrobust_return", 0.0)) - results["price_of_robustness"] = float(row.get("price_of_robustness", 0.0)) - results["robust_funded"] = int(row.get("best_robust_funded", 0)) - results["nonrobust_funded"] = int( - row.get("baseline_nonrobust_funded", row.get("best_robust_funded", 0)) - ) - - status_path = MODEL_DIR / "pipeline_run_status.pkl" - if status_path.exists(): - with open(status_path, "rb") as f: - status = pickle.load(f) - try: - results["pipeline_time_s"] = float(status.get("pipeline_time_s", 0.0)) - except Exception: - results["pipeline_time_s"] = 0.0 - - out = MODEL_DIR / "pipeline_results.pkl" - out.parent.mkdir(parents=True, exist_ok=True) - with open(out, "wb") as f: - pickle.dump(results, f) - logger.info(f"Saved pipeline results to {out}") - - -def main() -> None: - _persist_pipeline_results() - - -if __name__ == "__main__": - main() diff --git a/scripts/build_price_of_robustness_cross_dataset.py b/scripts/build_price_of_robustness_cross_dataset.py deleted file mode 100644 index 853c38f..0000000 --- a/scripts/build_price_of_robustness_cross_dataset.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Build the cross-dataset price-of-robustness table (A34). - -Surfaces a finding that is already computed but never displayed: the signed -``price_of_robustness_pct`` of the frozen CRPTO recipe is a *positive* premium on -the external economic panels, and it *increases with the panel default rate* -across frozen applications (no champion search). This reframes the multidataset -layer from a defensive "the gates still pass" claim into a positive, economically -interpretable result: buying the conformal coverage guarantee costs a premium -that scales with the panel's irreducible default risk. - -All inputs are frozen: the multidataset external status JSON. No champion stage -and no external replication run is re-executed. - -Reading of the sign (same convention as the Lending Club champion field): -``price_of_robustness_pct = (nonrobust - robust) / nonrobust``. A positive value -means robustness costs a premium; a negative value means robustness is favorable. -The Lending Club *selected* champion is ``-10.56%`` (favorable) and is reported in -prose as the selected-protocol contrast, not as a frozen-application row. - -Output: reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.{csv,tex} -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pandas as pd -from loguru import logger - -REPO_ROOT = Path(__file__).resolve().parents[1] -TABLES = REPO_ROOT / "reports" / "crpto" / "tables" -STATUS = REPO_ROOT / "models" / "crpto_multidataset_external_status.json" - - -def build_table() -> pd.DataFrame: - status = json.loads(STATUS.read_text(encoding="utf-8")) - - rows: list[dict[str, object]] = [] - - # Freddie FM48 segments (green / combined / red) -- same frozen protocol. - seg_label = { - "green": "Freddie FM48 (green)", - "both": "Freddie FM48 (combined)", - "red": "Freddie FM48 (red)", - } - for seg in status["freddie_segment_sensitivity"]: - name = seg["segment"] - if name not in seg_label: - continue - rows.append( - { - "application": seg_label[name], - "panel_default_rate": float(seg["default_rate"]), - "auc_roc": float(seg["auc_roc"]), - "price_of_robustness_pct": float(seg["price_of_robustness_pct"]), - "protocol": "frozen_application", - } - ) - - # Prosper final-status -- same frozen protocol. - for rep in status["external_replications"]: - if rep["dataset"] != "Prosper": - continue - rows.append( - { - "application": "Prosper final-status", - "panel_default_rate": float(rep["default_rate"]), - "auc_roc": float(rep["auc_roc"]), - "price_of_robustness_pct": float(rep["price_of_robustness_pct"]), - "protocol": "frozen_application", - } - ) - - df = pd.DataFrame(rows).sort_values("panel_default_rate").reset_index(drop=True) - - # Monotonicity check across frozen applications (descriptive, not a fitted law). - price = df["price_of_robustness_pct"].to_numpy() - df.attrs["price_monotone_increasing"] = bool((price[1:] >= price[:-1]).all()) - return df - - -def main() -> None: - TABLES.mkdir(parents=True, exist_ok=True) - df = build_table() - - out = df.copy() - out["panel_default_rate"] = out["panel_default_rate"].round(6) - out["auc_roc"] = out["auc_roc"].round(4) - out["price_of_robustness_pct"] = out["price_of_robustness_pct"].round(6) - - csv_path = TABLES / "crpto_tableA34_price_of_robustness_cross_dataset.csv" - out.to_csv(csv_path, index=False) - - tex = out.to_latex(index=False, escape=True, float_format=lambda x: f"{x:.4f}") - (TABLES / "crpto_tableA34_price_of_robustness_cross_dataset.tex").write_text( - tex, encoding="utf-8" - ) - - logger.info( - "A34 price of robustness: {} frozen applications, monotone-increasing in default rate = {}", - len(out), - df.attrs["price_monotone_increasing"], - ) - for _, r in out.iterrows(): - logger.info( - " {:<26} default={:.4f} price_of_robustness={:+.4f}", - r["application"], - r["panel_default_rate"], - r["price_of_robustness_pct"], - ) - logger.info("Wrote {}", csv_path.relative_to(REPO_ROOT)) - - -if __name__ == "__main__": - main() diff --git a/scripts/build_tail_constrained_reoptimization.py b/scripts/build_tail_constrained_reoptimization.py deleted file mode 100644 index f7d6163..0000000 --- a/scripts/build_tail_constrained_reoptimization.py +++ /dev/null @@ -1,394 +0,0 @@ -"""Build the CVaR/OCE tail-constrained re-optimization frontier (Table A22). - -This closes the P2 roadmap item "funded-set risk OCE/CVaR as a re-optimized -constraint" by turning the A12 tail-risk *diagnostic* into an **active -selection constraint** over the bound-aware robust region. - -What it does, concretely: - -1. Re-solves the 45 alpha01-safe robust-region policies with the Windows-safe - HiGHS backend (same machinery as the A20 audit). The decision is made from - conformal intervals only (``compute_effective_pd`` uses ``pd_point``/ - ``pd_high``); observed labels are never used to choose loans. -2. For each funded set, computes the *decision-time* worst-case loss rate - ``l_i = pd_high_i * LGD - (1 - pd_high_i) * int_rate_i`` (the ex-ante - quantity a risk committee can constrain), and its exposure-weighted CVaR_95 - and entropic OCE. It also reports the realized (label-based) CVaR for - comparison with the A20 audit. -3. Sweeps a CVaR_95 cap and, at each cap, selects the alpha01-passing, - CVaR-feasible policy with the highest *frozen* realized return. This is a - genuine re-optimization of the selection under an added tail constraint and - traces the return kept as the tail budget tightens. - -Authoritative return / V / Gamma_CP / exact-pass values are read from the -frozen shortlist and, for promoted/editorial comparator roles, overlaid from -``final_project_promotion.json``. The re-solve is used only to recover -allocations for tail scoring. No frozen artifact is overwritten and the -economic champion remains the official champion: the CVaR-constrained policy is -reported as a journal challenger. - -Usage:: - - uv run python scripts/build_tail_constrained_reoptimization.py - uv run python scripts/build_tail_constrained_reoptimization.py --max-policies 3 # fast dev -""" - -from __future__ import annotations - -import argparse -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) - -from scripts.build_tail_satisficing_challenger_audit import ( # noqa: E402 - CONFORMAL_INTERVALS_PATH, - DEFAULT_ALPHA, - DEFAULT_LGD, - OPTIMIZATION_CONFIG_PATH, - PROMOTION_PATH, - SHORTLIST_PATH, - _policy_role, - _prepare_portfolio_inputs, -) -from scripts.optimize_portfolio_tradeoff import _solve_single # noqa: E402 -from src.optimization.tail_satisficing_objective import ( # noqa: E402 - entropic_oce, - funded_loss_rate, - weighted_cvar, - weighted_mean, -) -from src.utils.script_helpers import load_json, load_yaml, write_json, write_table # noqa: E402 - -TABLE_DIR = ROOT / "reports" / "crpto" / "tables" -MODEL_DIR = ROOT / "models" -STATUS_PATH = MODEL_DIR / "crpto_tail_constrained_reopt_status.json" -TABLE_A22_NAME = "crpto_tableA22_tail_constrained_reoptimization" - -OCE_THETA = 5.0 -CVAR_TAIL = 0.95 -# Operating-point rule for the headline challenger: tightest tail cap whose -# selected policy stays within this fraction of the economic champion return. -RETURN_TOLERANCE_PCT = 2.0 -PROMOTION_ROLE_KEYS = { - "economic_champion": "final_champion", - "theorem_tight_comparator": "theorem_tight_comparator", - "balanced_comparator": "balanced_comparator", -} -PROMOTION_METRIC_FIELDS = ( - "realized_total_return", - "alpha01_exact_pass", - "alpha01_weighted_miscoverage_V", - "alpha01_gamma_cp", - "alpha01_violation", -) - - -def _sync_official_promotion_metrics( - frame: pd.DataFrame, - promotion: dict[str, Any], - *, - role_col: str, - prefix: str = "", -) -> pd.DataFrame: - """Overlay official rebaseline metrics for promoted editorial roles. - - The robust-region shortlist is retained as provenance, but promoted roles - may have updated V/Gamma values after a formal replay. Applying the promotion - JSON here keeps A22 aligned without mutating the frozen search artifact. - """ - synced = frame.copy() - if role_col not in synced.columns: - return synced - for role, record_key in PROMOTION_ROLE_KEYS.items(): - record = promotion.get(record_key, {}) - if not isinstance(record, dict): - continue - mask = synced[role_col].astype(str).eq(role) - if not mask.any(): - continue - for field in PROMOTION_METRIC_FIELDS: - target_col = f"{prefix}{field}" - if target_col in synced.columns and field in record: - synced.loc[mask, target_col] = record[field] - return synced - - -def worst_case_loss_rate(pd_high: np.ndarray, int_rates: np.ndarray, *, lgd: float) -> np.ndarray: - """Decision-time worst-case net loss rate per unit of exposure. - - Mirrors ``funded_loss_rate`` but replaces the observed default flag with the - conformal upper bound ``pd_high`` (= u_i), so the quantity is available - before labels are observed and is the legitimate object for an ex-ante tail - constraint. - """ - pd_high_array = np.clip(np.asarray(pd_high, dtype=float), 0.0, 1.0) - int_rate_array = np.asarray(int_rates, dtype=float) - return pd_high_array * float(lgd) - (1.0 - pd_high_array) * int_rate_array - - -def _solve_and_score_policy( - *, - row: pd.Series, - loans: pd.DataFrame, - pd_point: np.ndarray, - pd_low: np.ndarray, - pd_high: np.ndarray, - lgd: np.ndarray, - int_rates: np.ndarray, - default_flag: np.ndarray, - optimization_config: dict[str, Any], - promotion: dict[str, Any], -) -> dict[str, Any]: - solved, allocation = _solve_single( - loans=loans, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - default_flag=default_flag, - total_budget=float(optimization_config["portfolio"]["total_budget"]), - max_concentration=float(optimization_config["portfolio"]["max_concentration"]), - risk_tolerance=float(row["risk_tolerance"]), - robust=True, - uncertainty_aversion=float(row["uncertainty_aversion"]), - min_budget_utilization=float(row["min_budget_utilization"]), - pd_cap_slack_penalty=float(row["pd_cap_slack_penalty"]), - time_limit=int(optimization_config["optimization"]["time_limit"]), - threads=int(optimization_config["optimization"]["threads"]), - solver_backend="highs", - policy_mode=str(row["policy_mode"]), - gamma=float(row["gamma"]), - delta_cap_quantile=float(row["delta_cap_quantile"]), - tail_focus_quantile=float(row["tail_focus_quantile"]), - random_seed=42, - ) - loan_amounts = ( - loans["loan_amnt"].to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.full(len(loans), 10_000.0) - ) - exposure = allocation * loan_amounts - decision_loss = worst_case_loss_rate(pd_high, int_rates, lgd=DEFAULT_LGD) - realized_loss = funded_loss_rate(default_flag, int_rates, lgd=DEFAULT_LGD) - return { - "candidate_rank": int(row["candidate_rank"]), - "paper_role": _policy_role(row, promotion), - "policy_mode": str(row["policy_mode"]), - "risk_tolerance": float(row["risk_tolerance"]), - "gamma": float(row["gamma"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - # Authoritative frozen metrics (agree with final_project_promotion.json). - "realized_total_return": float(row["realized_total_return"]), - "alpha01_exact_pass": bool(row["alpha01_exact_pass"]), - "alpha01_weighted_miscoverage_V": float(row["alpha01_weighted_miscoverage_V"]), - "alpha01_gamma_cp": float(row["alpha01_gamma_cp"]), - "n_funded": int(solved["n_funded"]), - "total_allocated": float(solved["total_allocated"]), - "solver_status": str(solved["solver_status"]), - # Decision-time (pd_high-based) tail risk: the constrainable ex-ante object. - "decision_time_cvar95": weighted_cvar(decision_loss, exposure, tail=CVAR_TAIL), - "decision_time_oce_theta5": entropic_oce(decision_loss, exposure, theta=OCE_THETA), - "decision_time_mean_loss_rate": weighted_mean(decision_loss, exposure), - # Realized (label-based) CVaR, comparable to the A20 audit. - "realized_cvar95": weighted_cvar(realized_loss, exposure, tail=CVAR_TAIL), - } - - -def _build_cap_frontier( - scored: pd.DataFrame, champion_return: float -) -> tuple[pd.DataFrame, dict[str, Any]]: - """Trace the efficient return-vs-CVaR frontier under an active tail cap. - - Each alpha01-safe policy's own decision-time CVaR is used as a natural cap - breakpoint; at each cap we select the highest-return feasible policy. Keeping - the first occurrence of each distinct winner yields the efficient-frontier - corners (tightest cap at which a given policy becomes the return-maximizer). - """ - feasible_pool = scored.loc[scored["alpha01_exact_pass"]].copy() - cvar_values = feasible_pool["decision_time_cvar95"].to_numpy(dtype=float) - cvar_min = float(np.min(cvar_values)) - cvar_max = float(np.max(cvar_values)) - caps = np.sort(np.unique(cvar_values)) - - rows: list[dict[str, Any]] = [] - for cap in caps: - admissible = feasible_pool.loc[feasible_pool["decision_time_cvar95"] <= cap + 1e-12] - if admissible.empty: - continue - winner = admissible.sort_values( - ["realized_total_return", "decision_time_cvar95"], ascending=[False, True] - ).iloc[0] - rows.append( - { - "cvar95_cap": float(cap), - "n_feasible_policies": int(len(admissible)), - "selected_candidate_rank": int(winner["candidate_rank"]), - "selected_paper_role": str(winner["paper_role"]), - "selected_policy_mode": str(winner["policy_mode"]), - "selected_risk_tolerance": float(winner["risk_tolerance"]), - "selected_gamma": float(winner["gamma"]), - "selected_uncertainty_aversion": float(winner["uncertainty_aversion"]), - "selected_realized_total_return": float(winner["realized_total_return"]), - "return_delta_vs_champion_pct": ( - (float(winner["realized_total_return"]) - champion_return) - / abs(champion_return) - * 100.0 - ), - "selected_decision_time_cvar95": float(winner["decision_time_cvar95"]), - "selected_decision_time_oce_theta5": float(winner["decision_time_oce_theta5"]), - "selected_realized_cvar95": float(winner["realized_cvar95"]), - "selected_alpha01_weighted_miscoverage_V": float( - winner["alpha01_weighted_miscoverage_V"] - ), - "selected_alpha01_gamma_cp": float(winner["alpha01_gamma_cp"]), - "selected_n_funded": int(winner["n_funded"]), - } - ) - frontier = ( - pd.DataFrame(rows) - .drop_duplicates(subset=["selected_candidate_rank"], keep="first") - .reset_index(drop=True) - ) - - # Headline challenger: tightest cap whose pick stays within tolerance of champion. - within = frontier.loc[ - frontier["return_delta_vs_champion_pct"] >= -RETURN_TOLERANCE_PCT - ].sort_values("selected_decision_time_cvar95") - challenger = (within.iloc[0] if not within.empty else frontier.iloc[-1]).to_dict() - summary = { - "n_alpha01_policies": int(len(feasible_pool)), - "decision_time_cvar95_min": cvar_min, - "decision_time_cvar95_max": cvar_max, - "return_tolerance_pct": RETURN_TOLERANCE_PCT, - "tail_constrained_challenger": challenger, - "selection_rule": ( - "max realized_total_return s.t. alpha01_exact_pass and " - "decision_time_cvar95 <= cap; cap swept over the robust region range" - ), - "promotion_status": "journal_reoptimization_challenger_not_champion", - } - return frontier, summary - - -def build_tail_constrained_reoptimization(max_policies: int = 0) -> dict[str, Any]: - start = datetime.now(tz=UTC) - promotion = load_json(PROMOTION_PATH) - optimization_config = load_yaml(OPTIMIZATION_CONFIG_PATH) - shortlist = pd.read_parquet(SHORTLIST_PATH).sort_values("candidate_rank").reset_index(drop=True) - if max_policies and max_policies > 0: - # Dev mode: keep the champion + comparators + a spread of the region. - head = shortlist.head(max_policies) - shortlist = head.reset_index(drop=True) - logger.warning("DEV MODE: scoring only {} policies", len(shortlist)) - - loans, pd_point, pd_low, pd_high, lgd, int_rates, default_flag = _prepare_portfolio_inputs() - champion_return = float(promotion["final_champion"]["realized_total_return"]) - - scored_rows: list[dict[str, Any]] = [] - for position, row in enumerate((row for _, row in shortlist.iterrows()), start=1): - logger.info( - "Re-solving policy {}/{} (candidate_rank={}, mode={}, gamma={})", - position, - len(shortlist), - row["candidate_rank"], - row["policy_mode"], - row["gamma"], - ) - scored_rows.append( - _solve_and_score_policy( - row=row, - loans=loans, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - default_flag=default_flag, - optimization_config=optimization_config, - promotion=promotion, - ) - ) - scored = _sync_official_promotion_metrics( - pd.DataFrame(scored_rows), - promotion, - role_col="paper_role", - ) - frontier, summary = _build_cap_frontier(scored, champion_return) - frontier = _sync_official_promotion_metrics( - frontier, - promotion, - role_col="selected_paper_role", - prefix="selected_", - ) - - artifacts = write_table(TABLE_A22_NAME, frontier, table_dir=TABLE_DIR, root=ROOT) - status = { - "schema_version": "2026-06-07.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "elapsed_sec": (datetime.now(tz=UTC) - start).total_seconds(), - "alpha": DEFAULT_ALPHA, - "lgd": DEFAULT_LGD, - "cvar_tail": CVAR_TAIL, - "oce_theta": OCE_THETA, - "champion_label": promotion["final_champion"]["label"], - "champion_realized_total_return": champion_return, - "n_policies_scored": int(len(scored)), - "generated_artifacts": [ - str(path.relative_to(ROOT)).replace("\\", "/") for path in artifacts - ], - "source_shortlist": str(SHORTLIST_PATH.relative_to(ROOT)).replace("\\", "/"), - "source_conformal_intervals": str(CONFORMAL_INTERVALS_PATH.relative_to(ROOT)).replace( - "\\", "/" - ), - "reoptimization_summary": summary, - "per_policy_tail_metrics": scored.sort_values("decision_time_cvar95") - .round(6) - .to_dict(orient="records"), - "champion_promotion_changed": False, - "notes": [ - "Decision uses conformal intervals only; labels enter solely via " - "the frozen realized-return diagnostic.", - "Tail risk is the decision-time worst-case loss (pd_high-based), the " - "ex-ante object a risk committee can constrain.", - "No champion search was reopened beyond the 45 frozen robust-region " - "policies; the economic champion stays official.", - "For promoted/editorial comparator rows, V/Gamma/return metrics are " - "overlaid from the current final_project_promotion.json rebaseline " - "while tail-risk quantities remain from the HiGHS re-solve.", - ], - } - write_json(STATUS_PATH, status) - logger.info("Wrote {}", STATUS_PATH.relative_to(ROOT)) - logger.info( - "Tail-constrained challenger: candidate_rank={} return_delta={:.2f}% cvar95={:.4f}", - summary["tail_constrained_challenger"]["selected_candidate_rank"], - summary["tail_constrained_challenger"]["return_delta_vs_champion_pct"], - summary["tail_constrained_challenger"]["selected_decision_time_cvar95"], - ) - return status - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--max-policies", - type=int, - default=0, - help="Limit number of policies scored (dev mode); 0 = all 45.", - ) - args = parser.parse_args() - build_tail_constrained_reoptimization(max_policies=args.max_policies) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_dvc_status_warning.py b/scripts/check_dvc_status_warning.py deleted file mode 100644 index 6a32717..0000000 --- a/scripts/check_dvc_status_warning.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Report DVC drift during local pre-push without blocking the push. - -Protected CRPTO stages are intentionally not re-run as part of ordinary -governance, documentation, or paper-surface edits. This hook keeps the useful -``dvc status`` signal visible while leaving artifact promotion to explicit -run-tagged revalidation. -""" - -from __future__ import annotations - -import subprocess -import sys - - -def main() -> int: - result = subprocess.run( - [sys.executable, "-m", "dvc", "status"], - check=False, - capture_output=True, - text=True, - ) - output = (result.stdout or "").strip() - error = (result.stderr or "").strip() - - if result.returncode == 0: - if output: - print(output) - return 0 - - print("DVC status reports changed deps/outs; treating as non-blocking drift report.") - print("Protected-stage reproduction still requires an explicit run tag and approval.") - if output: - print(output) - if error: - print(error, file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_publication_integrity.py b/scripts/check_publication_integrity.py index 3579b06..d71a73a 100644 --- a/scripts/check_publication_integrity.py +++ b/scripts/check_publication_integrity.py @@ -1,188 +1,421 @@ -"""Check active IJDS manuscript surfaces for claim and narrative drift.""" +"""Check active IJDS surfaces for evidence, narrative, and anonymity drift.""" from __future__ import annotations +import json import re import sys from dataclasses import dataclass from pathlib import Path +import yaml from loguru import logger +from src.ijds_audit.claim_ledger import materialize_claim_ledger +from src.ijds_audit.publication_generation import publication_implementation_descriptors +from src.ijds_audit.publication_sources import load_verified_source_registry + REPO = Path(__file__).resolve().parents[1] +EVIDENCE_PATH = REPO / "reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json" +SOURCE_REGISTRY_PATH = REPO / "configs/ijds_active_evidence_sources.yaml" +PUBLICATION_TARGETS_PATH = REPO / "configs/crpto_publication_targets.yaml" +CLAIM_LEDGER_PATH = REPO / "configs/ijds_claim_ledger.yaml" @dataclass(frozen=True) class SurfaceCheck: - """Text surface and tokens that must or must not appear after normalization.""" - path: Path required: tuple[str, ...] - forbidden: tuple[str, ...] = () - - -COMMON_CLAIM_TOKENS = ( - "$179327.59", - "0.039375", - "0.036875", - "0.176102", - "0.088051", - "0.258051", - "0.294926", - "0.574279", - "196369.14", - "8.678%", - "7.9025", + + +TITLE = "crpto: an identification audit of binary conformal credit portfolio optimization" + +REVIEWER_SURFACES = ( + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", + REPO / "paper/submission/TITLE_PAGE_DRAFT.md", + REPO / "paper/submission/COVER_LETTER_AND_DISCLOSURE.md", +) + +ACTIVE_EDITORIAL_SURFACES = ( + REPO / ".codex/skills/crpto/SKILL.md", + REPO / "CLAUDE.md", + REPO / "AGENTS.md", + REPO / "CONTRIBUTING.md", + REPO / "docs/ACADEMIC_CONTEXT.md", + REPO / "docs/SCOPE_AND_GOVERNANCE.md", + REPO / "docs/research/active_claims_2026-07-14.md", + REPO / "paper/README.md", + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", + REPO / "paper/submission/CLAIM_AUDIT_MATRIX.md", + REPO / "paper/submission/COVER_LETTER_AND_DISCLOSURE.md", + REPO / "paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md", + REPO / "paper/submission/EDITOR_ONLY_REPRODUCIBILITY_CROSSWALK.md", + REPO / "paper/submission/README.md", + REPO / "paper/submission/REPRODUCIBILITY_PACKAGE.md", + REPO / "paper/submission/SCHOLARONE_FINAL_CHECKLIST.md", + REPO / "paper/submission/TITLE_PAGE_DRAFT.md", + REPO / "configs/crpto_publication_targets.yaml", ) -ACTIVE_SURFACE_FORBIDDEN = ( - "four contributions", - "crpto v2", - "-10.56%", - "markov cap", - "0.345084", - "50010", - "27508", - "capped_blended_uncertainty", +RETIRED_CLAIM_TOKENS = ( + "0.838531", + "0.895654", + "0.896973", + "favorable at .25", + "favorable at 0.25", + "5,603.66 higher", + "small favorable", + "borrower-only", + "calibration-in-the-large", + "september 2020 administrative snapshot", + "four independent controls", + "ijds-binary-geometry-frontier-v4-2026-07-12-v2", + "ijds-normalized-objective-frontier-2026-07-13-v2", + "ijds-credit-risk-controls-2026-07-13-v2b", + "active_claims_2026-07-12.md", ) SURFACES = ( SurfaceCheck( - path=REPO / "README.md", - required=( - *COMMON_CLAIM_TOKENS, - "claim ijds activo", - "q=(p+u)/2", - "cap determinista", - ), - forbidden=("## champion congelado",), - ), - SurfaceCheck( - path=REPO / "paper/submission/README.md", - required=( - "pdflatex -> bibtex -> pdflatex -> pdflatex", - "latexmk", - "official-template", - ), - forbidden=ACTIVE_SURFACE_FORBIDDEN, - ), - SurfaceCheck( - path=REPO / "paper/CRPTO_ijds.qmd", - required=( - *COMMON_CLAIM_TOKENS, - "the paper makes three contributions", - "retrospective lockbox replay", - "matched point-pd", - "q_i=(p_i+u_i)/2", - ), - forbidden=ACTIVE_SURFACE_FORBIDDEN, - ), - SurfaceCheck( - path=REPO / "paper/submission/CRPTO_ijds_submission.tex", - required=( - *COMMON_CLAIM_TOKENS, - "the paper makes three contributions", - "retrospective lockbox replay", - "matched point-pd", + REPO / "paper/CRPTO_ijds.qmd", + ( + TITLE, + "reconstructed", + "not a verified point-in-time snapshot", + "two rulers constructed without policy-development or OOT evaluation outcomes", + "objective-matched", + "normalized-score", + "crosses zero", + "not a prospective trial", + "ethical and governance implications", ), - forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper/supplement_ijds.qmd", - required=( - *COMMON_CLAIM_TOKENS, - "a35. exact alpha replay", - "a36. calibration policy selector", - "a40. matched decision audit", - "retrospective lockbox replay", + REPO / "paper/supplement_ijds.qmd", + ( + TITLE, + "reconstructed", + "label-lag sensitivity", + "not independent replications", + "coordinate one", + "missingness-encoding sensitivity", + "second temporal origin", ), - forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper/submission/CLAIM_AUDIT_MATRIX.md", - required=( - "calibration-selected midpoint", - "a40", - "8.678%", - "7.9025", + REPO / "paper/submission/CRPTO_ijds_submission.tex", + ( + TITLE, + "generated from paper/crpto_ijds.qmd", + "objective-matched", + "normalized-score", ), - forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "docs/research/active_claims_2026-07-04.md", - required=( - *COMMON_CLAIM_TOKENS, - "nine round-number candidates", - "retrospective lockbox replay", - "retired headline claims", + REPO / "docs/research/active_claims_2026-07-14.md", + ( + "sole claim registry", + "0.842485", + "0.897726", + "12,076", + "no endpoint has a universal status-indexed outcome ordering", ), - forbidden=("crpto v2", "markov cap", "+27.03%"), ), SurfaceCheck( - path=REPO / "configs/crpto_publication_targets.yaml", - required=( - "exact 90% conformal replay", - "q=(p+u)/2", - "outside the submitted claim", - "not acceptance criteria", + REPO / "configs/crpto_publication_targets.yaml", + ( + "active_claims_2026-07-14.md", + "lineage_and_dvc_authority", + "configs/ijds_active_evidence_sources.yaml", + "policy_winner_allowed: false", ), - forbidden=("crpto v2",), ), ) +REVIEWER_FORBIDDEN_LITERALS = ( + "champion-reopen-", + "protocol/ijds", + "carlos alfredo vergara rojas", + "cavr94", + "eigencharlie", + "c:\\users\\", + "the authors thank the anonymous reviewers and editors", +) + +REVIEWER_FORBIDDEN_PATTERNS = ( + ("full Git commit", re.compile(r"\b[0-9a-f]{40}\b", re.IGNORECASE)), + ("SHA-256 fingerprint", re.compile(r"\b[0-9a-f]{64}\b", re.IGNORECASE)), + ("DVC directory fingerprint", re.compile(r"\b[0-9a-f]{32}\.dir\b", re.IGNORECASE)), +) + def _normalize(text: str) -> str: - """Normalize Markdown and LaTeX enough for robust token checks.""" - lowered = text.lower() - replacements = { + value = text.lower() + for old, new in { r"\$": "$", - "{,}": ",", + r"\%": "%", r"\_": "_", - r"\mathrm": "", - r"\gamma": "gamma", - r"\alpha": "alpha", - "\\": "", + "{,}": ",", + "{[}": "[", + "{]}": "]", "{": "", "}": "", "`": "", - ",": "", - } - for old, new in replacements.items(): - lowered = lowered.replace(old, new) - lowered = lowered.replace("\u2013", "-").replace("\u2014", "-") - return re.sub(r"\s+", " ", lowered) + }.items(): + value = value.replace(old, new) + return re.sub(r"\s+", " ", value) -def _read_normalized(path: Path) -> str: - return _normalize(path.read_text(encoding="utf-8")) +def _evidence() -> dict: + payload: object = json.loads(EVIDENCE_PATH.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{EVIDENCE_PATH} must contain a JSON object") + return payload -def check_publication_integrity() -> list[str]: - """Return active-manuscript integrity failures.""" +def _check_surface_contracts() -> list[str]: failures: list[str] = [] for surface in SURFACES: if not surface.path.is_file(): failures.append(f"{surface.path.relative_to(REPO)} is missing") continue - text = _read_normalized(surface.path) - missing = [token for token in surface.required if token not in text] - present_forbidden = [token for token in surface.forbidden if token in text] - rel = surface.path.relative_to(REPO) - failures.extend(f"{rel}: missing required token '{token}'" for token in missing) + text = _normalize(surface.path.read_text(encoding="utf-8")) + failures.extend( + f"{surface.path.relative_to(REPO)}: missing required token '{token}'" + for token in surface.required + if _normalize(token) not in text + ) + return failures + + +def _check_numeric_sync() -> list[str]: + evidence = _evidence() + design = evidence["design"] + expected = ( + f"{design['primary_oot_candidates']:,}", + f"{design['primary_oot_resolved']:,}", + f"{design['primary_oot_unresolved']:,}", + ) + failures: list[str] = [] + for path in ( + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", + ): + text = _normalize(path.read_text(encoding="utf-8")) + failures.extend( + f"{path.relative_to(REPO)}: missing evidence census '{token}'" + for token in expected + if _normalize(token) not in text + ) + return failures + + +def _check_endpoint_reason_partition() -> list[str]: + """Require the active reason census wherever the endpoint is reported.""" + required = ("307,842", "56,972", "11,551", "47", "478", "364,814", "12,076") + failures: list[str] = [] + for path in ( + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", + REPO / "docs/research/active_claims_2026-07-14.md", + ): + text = _normalize(path.read_text(encoding="utf-8")) + failures.extend( + f"{path.relative_to(REPO)}: incomplete endpoint-reason partition '{token}'" + for token in required + if token not in text + ) + return failures + + +def _check_retired_claims() -> list[str]: + failures: list[str] = [] + for path in ACTIVE_EDITORIAL_SURFACES: + if not path.is_file(): + failures.append(f"{path.relative_to(REPO)} is missing") + continue + text = _normalize(path.read_text(encoding="utf-8")) failures.extend( - f"{rel}: forbidden token still present '{token}'" for token in present_forbidden + f"{path.relative_to(REPO)}: retired claim token '{token}'" + for token in RETIRED_CLAIM_TOKENS + if _normalize(token) in text + ) + return failures + + +def _check_reviewer_anonymity() -> list[str]: + failures: list[str] = [] + for path in REVIEWER_SURFACES: + if not path.is_file(): + continue + raw = path.read_text(encoding="utf-8") + normalized = _normalize(raw) + for literal in REVIEWER_FORBIDDEN_LITERALS: + if literal in normalized: + failures.append(f"{path.relative_to(REPO)}: reviewer identity token '{literal}'") + for label, pattern in REVIEWER_FORBIDDEN_PATTERNS: + if pattern.search(raw): + failures.append(f"{path.relative_to(REPO)}: reviewer surface contains {label}") + return failures + + +def _check_evidence_decision() -> list[str]: + evidence = _evidence() + failures: list[str] = [] + boundary = evidence["claim_boundary"] + for field in ("policy_winner", "confirmatory", "prospective", "causal"): + if boundary[field] is not False: + failures.append(f"active evidence unexpectedly allows {field}") + if evidence["design"]["archive_is_verified_point_in_time_snapshot"] is not False: + failures.append("active evidence misstates the archive as a point-in-time snapshot") + if not evidence["credit_risk_controls"]["all_five_all_eight_upper_below_nominal"]: + failures.append("five-model coverage result no longer holds") + if not evidence["portfolio"]["broad_stress_all_envelopes_cross_zero"]: + failures.append("broad comparator support no longer crosses zero everywhere") + lag = evidence["binary_phase_transition"]["label_lag_sensitivity"] + if not lag["w7_to_w8_threshold_crossing_at_all_admissible_lags"]: + failures.append("phase crossing no longer survives all admissible reporting lags") + tie = evidence["portfolio"]["evaluated_point_cap_solver_stability"] + if tie["near_zero_bases"] != 0 or tie["tie_sensitive_rows"] != 0: + failures.append("evaluated point-cap solver stability no longer holds") + challenger = evidence["decision_challenger"] + interpretation = challenger["interpretation"] + for field in ("preferred_gamma", "preferred_ruler", "preferred_coordinate", "policy_winner"): + if interpretation[field] is not None: + failures.append(f"two-ruler evidence unexpectedly selects {field}") + quarter = next( + row + for row in challenger["rows"] + if row["ruler"] == "objective_matched" and row["coordinate"] == 0.25 + ) + for field in ( + "payoff_direction_pattern", + "default_direction_pattern", + "miscoverage_direction_pattern", + ): + if quarter[field] != "crosses_zero:8": + failures.append(f"objective-matched .25 unexpectedly changed: {field}") + if ( + evidence["evaluation_endpoint"].get("reason_census_partitions_primary_candidates") + is not True + ): + failures.append("endpoint reasons no longer partition the primary candidate census") + endpoint = evidence.get("sensitivity", {}).get("evaluation_endpoint_availability", {}) + if endpoint.get("six_month_endpoint_reconciles_to_active_evaluation") is not True: + failures.append("endpoint sensitivity no longer reconciles to the active evaluation") + if endpoint.get("endpoint_or_result_selected") is not False: + failures.append("endpoint sensitivity unexpectedly selects an endpoint or result") + if endpoint.get("fit_label_lag_crossed_factorially") is not False: + failures.append("separate timing sensitivities are incorrectly reported as factorial") + missingness = evidence.get("sensitivity", {}).get("missingness_encoding", {}) + if missingness.get("all_three_all_eight_upper_below_nominal") is not True: + failures.append("missingness-encoding coverage recurrence no longer holds") + if missingness.get("model_or_encoding_selected") is not False: + failures.append("missingness sensitivity unexpectedly selects a model or encoding") + rolling = evidence.get("sensitivity", {}).get("rolling_origin", {}) + if rolling.get("all_sixteen_upper_below_nominal") is not True: + failures.append("two-origin coverage recurrence no longer holds") + if rolling.get("independent_replication_claim_authorized") is not False: + failures.append("second origin is incorrectly reported as an independent replication") + return failures + + +def _check_claim_ledger() -> list[str]: + """Require every active qualitative claim to resolve and appear only where allowed.""" + evidence = _evidence() + try: + expected = materialize_claim_ledger( + CLAIM_LEDGER_PATH, + evidence=evidence, + repo_root=REPO, ) + except (KeyError, OSError, TypeError, ValueError, RuntimeError) as error: + return [f"active claim ledger failed verification: {error}"] + if evidence.get("claim_ledger") != expected: + return ["evidence manifest claim ledger differs from the executable contract"] + return [] + + +def _check_lineage_sync() -> list[str]: + """Verify identities and DVC pointers against the single source registry.""" + failures: list[str] = [] + try: + registry, registered = load_verified_source_registry( + SOURCE_REGISTRY_PATH, + repo_root=REPO, + ) + except (KeyError, OSError, TypeError, ValueError, RuntimeError) as error: + return [f"active source registry failed verification: {error}"] + evidence = _evidence() + targets = yaml.safe_load(PUBLICATION_TARGETS_PATH.read_text(encoding="utf-8")) + contract = targets.get("active_scientific_contract", {}) if isinstance(targets, dict) else {} + expected_registry_path = SOURCE_REGISTRY_PATH.relative_to(REPO).as_posix() + if contract.get("source_registry") != expected_registry_path: + failures.append("publication target does not consume the active source registry") + if contract.get("lineage_and_dvc_authority") != expected_registry_path: + failures.append("publication target duplicates or omits lineage/DVC authority") + if evidence.get("lineages") != registry["lineages"]: + failures.append("evidence manifest lineages differ from the active source registry") + if evidence.get("sensitivities") != registry.get("sensitivities"): + failures.append("evidence manifest sensitivities differ from the active source registry") + expected_source_registry = { + "schema_version": str(registry["schema_version"]), + "status": str(registry["status"]), + "sources": sorted(registered), + } + if evidence.get("source_registry") != expected_source_registry: + failures.append("evidence manifest source-registry identity changed") + binary = registry["lineages"]["binary_geometry"]["evaluation"] + for field in ("run_tag", "protocol_tag", "protocol_commit"): + if evidence.get(field) != binary[field]: + failures.append(f"active binary evidence {field} differs from the registry") + two_ruler = registry["lineages"]["two_ruler"]["evaluation"] + challenger = evidence.get("decision_challenger", {}) + for field in ("run_tag", "protocol_tag", "protocol_commit"): + if challenger.get(field) != two_ruler[field]: + failures.append(f"two-ruler {field} differs from the registry") + endpoint = registry["sensitivities"]["endpoint_availability"] + endpoint_evidence = evidence.get("sensitivity", {}).get("evaluation_endpoint_availability", {}) + for field in ("run_tag", "protocol_tag", "protocol_commit"): + if endpoint_evidence.get(field) != endpoint[field]: + failures.append(f"endpoint sensitivity {field} differs from the registry") + expected_descriptors = publication_implementation_descriptors(REPO) + evidence_sources = evidence.get("source_artifacts", {}) + for name, descriptor in expected_descriptors.items(): + if evidence_sources.get(name) != descriptor: + failures.append(f"evidence manifest does not bind the current {name}") + for pointer in registry["dvc_pointers"]: + if not (REPO / pointer).is_file(): + failures.append(f"active DVC pointer is missing: {pointer}") return failures +def check_publication_integrity() -> list[str]: + return [ + *_check_surface_contracts(), + *_check_numeric_sync(), + *_check_endpoint_reason_partition(), + *_check_retired_claims(), + *_check_reviewer_anonymity(), + *_check_evidence_decision(), + *_check_claim_ledger(), + *_check_lineage_sync(), + ] + + def main() -> int: - """CLI entry point.""" failures = check_publication_integrity() if failures: for failure in failures: logger.error(failure) return 1 - logger.success("Active IJDS publication surfaces are claim-synchronized.") + logger.info("Active IJDS publication integrity checks passed.") return 0 diff --git a/scripts/compile_ijds_submission.py b/scripts/compile_ijds_submission.py index 9cd7eba..0d0472c 100644 --- a/scripts/compile_ijds_submission.py +++ b/scripts/compile_ijds_submission.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import hashlib +import json import os import re import shutil @@ -13,11 +15,21 @@ from loguru import logger +from scripts.build_ijds_submission_tex import render_submission_tex + ROOT = Path(__file__).resolve().parents[1] SUBMISSION_DIR = ROOT / "paper" / "submission" REPORT_DIR = ROOT / "reports" / "ci" TEX_NAME = "CRPTO_ijds_submission.tex" JOB_NAME = "CRPTO_ijds_submission" +OFFICIAL_TEMPLATE_FILES = ( + "informs4.cls", + "informs2014.bst", + "eqndefns-left.sty", + "informs_Logo.pdf", +) +STYLE_MANIFEST = SUBMISSION_DIR / "informs_style_assets.json" +INFORMS_STYLE_URL = "https://pubsonline.informs.org/authorportal/latex-style-files" @dataclass(frozen=True) @@ -27,10 +39,17 @@ class LatexScan: pages: int | None blg_warnings: tuple[str, ...] log_failures: tuple[str, ...] + artifact_failures: tuple[str, ...] = () @property def ok(self) -> bool: - return not self.blg_warnings and not self.log_failures + return ( + self.pages is not None + and self.pages > 0 + and not self.blg_warnings + and not self.log_failures + and not self.artifact_failures + ) def _run(command: list[str], *, cwd: Path, env: dict[str, str], transcript: Path) -> int: @@ -96,11 +115,65 @@ def _manual_pdflatex_bibtex(cwd: Path, env: dict[str, str], transcript: Path) -> return 0 -def compile_submission(*, prefer_manual: bool = False) -> int: +def _missing_template_files(directory: Path) -> tuple[str, ...]: + """Return official publisher assets absent from a submission directory.""" + return tuple(name for name in OFFICIAL_TEMPLATE_FILES if not (directory / name).is_file()) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _template_asset_drift(directory: Path) -> tuple[str, ...]: + """Return publisher assets that differ from the reviewed style-kit manifest.""" + payload = json.loads((directory / STYLE_MANIFEST.name).read_text(encoding="utf-8")) + assets = payload.get("assets") + if not isinstance(assets, dict) or set(assets) != set(OFFICIAL_TEMPLATE_FILES): + return ("style manifest inventory",) + drift: list[str] = [] + for name in OFFICIAL_TEMPLATE_FILES: + path = directory / name + expected = assets[name] + if path.stat().st_size != int(expected["bytes"]) or _file_sha256(path) != str( + expected["sha256"] + ): + drift.append(name) + return tuple(drift) + + +def compile_submission(*, prefer_manual: bool = False, render: bool = True) -> int: """Compile the official submission with latexmk, falling back to manual passes.""" + if render: + render_submission_tex() if not (SUBMISSION_DIR / TEX_NAME).is_file(): logger.error("Missing {}", (SUBMISSION_DIR / TEX_NAME).relative_to(ROOT)) return 2 + missing = _missing_template_files(SUBMISSION_DIR) + if missing: + logger.error( + "Missing official INFORMS LaTeX assets: {}. Download the current style package " + "from {} and place these files in paper/submission; they are intentionally " + "ignored by Git.", + ", ".join(missing), + INFORMS_STYLE_URL, + ) + return 2 + if not STYLE_MANIFEST.is_file(): + logger.error("Missing tracked INFORMS style manifest: {}", STYLE_MANIFEST.relative_to(ROOT)) + return 2 + drift = _template_asset_drift(SUBMISSION_DIR) + if drift: + logger.error( + "INFORMS style assets differ from the reviewed manifest: {}. Re-download " + "from {} or explicitly review and update informs_style_assets.json.", + ", ".join(drift), + INFORMS_STYLE_URL, + ) + return 2 env = _submission_env() transcript = REPORT_DIR / "ijds-latex-build.txt" @@ -128,12 +201,26 @@ def compile_submission(*, prefer_manual: bool = False) -> int: return code -def scan_submission_logs() -> LatexScan: - """Inspect `.blg` and `.log` outputs for bibliography/reference drift.""" - blg_path = SUBMISSION_DIR / f"{JOB_NAME}.blg" - log_path = SUBMISSION_DIR / f"{JOB_NAME}.log" - blg_text = blg_path.read_text(encoding="utf-8", errors="replace") if blg_path.exists() else "" - log_text = log_path.read_text(encoding="utf-8", errors="replace") if log_path.exists() else "" +def scan_submission_logs(directory: Path | None = None) -> LatexScan: + """Inspect required PDF, `.blg`, and `.log` outputs for build drift.""" + directory = SUBMISSION_DIR if directory is None else directory + pdf_path = directory / f"{JOB_NAME}.pdf" + blg_path = directory / f"{JOB_NAME}.blg" + log_path = directory / f"{JOB_NAME}.log" + required_outputs = ( + ("PDF", pdf_path), + ("LaTeX log", log_path), + ("BibTeX log", blg_path), + ) + artifact_failures: list[str] = [] + for label, path in required_outputs: + if not path.is_file(): + artifact_failures.append(f"missing {label}: {path.name}") + elif path.stat().st_size == 0: + artifact_failures.append(f"empty {label}: {path.name}") + + blg_text = blg_path.read_text(encoding="utf-8", errors="replace") if blg_path.is_file() else "" + log_text = log_path.read_text(encoding="utf-8", errors="replace") if log_path.is_file() else "" blg_warnings = tuple( line.strip() for line in blg_text.splitlines() if line.strip().startswith("Warning--") @@ -152,7 +239,12 @@ def scan_submission_logs() -> LatexScan: if matches: pages = int(matches[-1]) - return LatexScan(pages=pages, blg_warnings=blg_warnings, log_failures=log_failures) + return LatexScan( + pages=pages, + blg_warnings=blg_warnings, + log_failures=log_failures, + artifact_failures=tuple(artifact_failures), + ) def main(argv: list[str] | None = None) -> int: @@ -167,10 +259,18 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Do not compile; only inspect existing .log and .blg files.", ) + parser.add_argument( + "--skip-render", + action="store_true", + help="Compile the existing generated TeX without rendering the QMD again.", + ) args = parser.parse_args(argv) + if args.scan_only and args.skip_render: + parser.error("--skip-render is redundant with --scan-only") + if not args.scan_only: - code = compile_submission(prefer_manual=args.manual) + code = compile_submission(prefer_manual=args.manual, render=not args.skip_render) if code != 0: logger.error("Official IJDS LaTeX compile failed with code {}.", code) return code @@ -180,8 +280,12 @@ def main(argv: list[str] | None = None) -> int: logger.error("BibTeX warnings:\n{}", "\n".join(scan.blg_warnings)) if scan.log_failures: logger.error("LaTeX convergence failures: {}", ", ".join(scan.log_failures)) + if scan.artifact_failures: + logger.error("Required LaTeX output failures: {}", ", ".join(scan.artifact_failures)) if scan.pages is None: - logger.warning("Could not read page count from the LaTeX log.") + logger.error("Could not read page count from the LaTeX log.") + elif scan.pages <= 0: + logger.error("LaTeX log reported a nonpositive page count: {}", scan.pages) else: logger.success("Official IJDS PDF page count: {}", scan.pages) diff --git a/scripts/experiments/ijds_policy_support.py b/scripts/experiments/ijds_policy_support.py deleted file mode 100644 index 10ab494..0000000 --- a/scripts/experiments/ijds_policy_support.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Shared exact-alpha panel loading and policy evaluation for IJDS experiments.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from src.models.conformal_alpha_grid import alpha_interval_columns # noqa: E402 -from src.optimization.certificate_semantics import ( # noqa: E402 - compute_funded_certificate_metrics, -) -from src.optimization.input_alignment import align_candidate_intervals # noqa: E402 -from src.optimization.policy_evaluation import ( # noqa: E402 - PolicyAllocationResult, - solve_policy_allocation, -) -from src.optimization.policy_selection import ( # noqa: E402 - LinearPolicyCandidate, - temporal_period_labels, -) -from src.utils.script_helpers import parse_percent_series, resolve_repo_artifact_path # noqa: E402 - - -def load_policy_panel(config: dict[str, Any], *, root: Path = ROOT) -> pd.DataFrame: - """Load candidates and exact-alpha intervals under a strict ID contract.""" - source = config["source"] - design = config["design"] - execution = config["execution"] - candidate_value = str(source.get("candidate_path", "data/processed/test_fe.parquet")) - candidate_path = resolve_repo_artifact_path(candidate_value, root=root) - exact_grid_value = str(source.get("exact_alpha_grid_path", "")).strip() - if not exact_grid_value: - raise ValueError("Active IJDS policy evaluation requires exact_alpha_grid_path.") - exact_grid_path = resolve_repo_artifact_path(exact_grid_value, root=root) - exact_alignment = align_candidate_intervals( - pd.read_parquet(candidate_path), - pd.read_parquet(exact_grid_path), - max_candidates=0, - random_state=int(execution.get("random_seed", 42)), - ) - if exact_alignment.mode != "id": - raise RuntimeError( - "Active IJDS policy evaluation requires one-to-one candidate/exact-grid ID alignment." - ) - panel = exact_alignment.candidates.copy() - exact = exact_alignment.intervals - low_column, high_column = alpha_interval_columns(float(design["alpha"])) - required = {"y_pred", low_column, high_column} - missing = sorted(required.difference(exact.columns)) - if missing: - raise KeyError(f"Exact alpha grid is missing columns: {missing}") - - panel["_pd_point"] = exact["y_pred"].to_numpy(dtype=float) - panel["_pd_low"] = exact[low_column].to_numpy(dtype=float) - panel["_pd_high"] = exact[high_column].to_numpy(dtype=float) - exact_outcome = pd.to_numeric(exact["y_true"], errors="raise").to_numpy(dtype=float) - if "default_flag" in panel.columns: - candidate_outcome = pd.to_numeric(panel["default_flag"], errors="raise").to_numpy( - dtype=float - ) - if not np.array_equal(candidate_outcome, exact_outcome): - raise ValueError("Candidate default_flag does not match exact-grid y_true by ID.") - panel["_outcome"] = exact_outcome - panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) - panel["_int_rate"] = parse_percent_series(panel["int_rate"]) - panel["_period"] = temporal_period_labels( - panel["issue_d"], - combine_years_from=int(design["combine_years_from"]), - ) - panel.attrs.update( - { - "candidate_path": str(candidate_path), - "exact_alpha_grid_path": str(exact_grid_path), - "alignment_mode": exact_alignment.mode, - } - ) - return panel - - -def solve_candidate( - frame: pd.DataFrame, - candidate: LinearPolicyCandidate, - *, - config: dict[str, Any], - robust: bool = True, -) -> PolicyAllocationResult: - """Solve one declared policy on one aligned panel.""" - design = config["design"] - execution = config["execution"] - return solve_policy_allocation( - loans=frame, - pd_point=frame["_pd_point"].to_numpy(dtype=float), - pd_low=frame["_pd_low"].to_numpy(dtype=float), - pd_high=frame["_pd_high"].to_numpy(dtype=float), - lgd=np.full(len(frame), float(design["lgd"]), dtype=float), - int_rates=frame["_int_rate"].to_numpy(dtype=float), - total_budget=float(design["budget"]), - max_concentration=float(design["max_concentration"]), - risk_tolerance=float(candidate.risk_tolerance), - robust=robust, - uncertainty_aversion=float(candidate.uncertainty_aversion) if robust else 0.0, - min_budget_utilization=float(candidate.min_budget_utilization), - pd_cap_slack_penalty=float(candidate.pd_cap_slack_penalty), - policy_mode=candidate.policy_mode, - gamma=float(candidate.gamma), - delta_cap_quantile=float(candidate.delta_cap_quantile), - tail_focus_quantile=float(candidate.tail_focus_quantile), - time_limit=int(execution["time_limit"]), - threads=int(execution["threads"]), - solver_backend=str(execution["solver_backend"]), - random_seed=int(execution.get("random_seed", 42)), - ) - - -def evaluate_candidate( - frame: pd.DataFrame, - candidate: LinearPolicyCandidate, - *, - config: dict[str, Any], - robust: bool, - period: str, -) -> tuple[dict[str, Any], PolicyAllocationResult]: - """Solve and score one policy on one evaluation period.""" - result = solve_candidate(frame, candidate, config=config, robust=robust) - exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) - total_allocated = float(exposure.sum()) - if total_allocated <= 0.0: - raise RuntimeError(f"Policy {candidate.candidate_id} allocated no capital in {period}.") - weights = exposure / total_allocated - outcomes = frame["_outcome"].to_numpy(dtype=float) - alpha = float(config["design"]["alpha"]) - certificate = compute_funded_certificate_metrics( - weights, - outcomes=outcomes, - pd_point=frame["_pd_point"].to_numpy(dtype=float), - pd_high=frame["_pd_high"].to_numpy(dtype=float), - pd_effective=result.effective_pd, - alpha=alpha, - risk_tolerance=float(candidate.risk_tolerance), - pd_cap_slack=float(result.solution.get("pd_cap_slack", 0.0)), - ) - funded = result.allocation > 0.01 - rates = frame["_int_rate"].to_numpy(dtype=float) - lgd = float(config["design"]["lgd"]) - realized_return = float( - np.sum( - np.where( - funded & (outcomes.astype(int) == 1), - -lgd * exposure, - np.where(funded, rates * exposure, 0.0), - ) - ) - ) - record: dict[str, Any] = { - "period": period, - **candidate.to_record(), - "solver_status": str(result.solution.get("solver_status", "unknown")), - "objective_risk_mode": result.objective_risk_mode, - "expected_objective": float(result.solution.get("objective_value", float("nan"))), - "n_panel": int(len(frame)), - "n_funded": int(certificate.n_funded), - "total_allocated": total_allocated, - "realized_return": realized_return, - "weighted_outcome": certificate.weighted_outcome, - "weighted_miscoverage": certificate.weighted_miscoverage, - "weighted_pd_point": certificate.weighted_pd_point, - "weighted_pd_effective": certificate.weighted_pd_effective, - "gamma_cp": certificate.gamma_cp, - "gamma_internalized": certificate.gamma_internalized, - "gamma_residual": certificate.gamma_residual, - "endpoint_budget": certificate.endpoint_budget, - "markov_loss_threshold": certificate.markov_loss_threshold, - "realized_risk_tolerance_excess": certificate.realized_risk_tolerance_excess, - "screen_V_leq_sqrt_alpha": bool( - certificate.weighted_miscoverage <= certificate.sqrt_alpha + 1e-12 - ), - "screen_risk_excess_leq_alpha": bool( - certificate.realized_risk_tolerance_excess <= alpha + 1e-12 - ), - } - return record, result diff --git a/scripts/experiments/launch_champion_claim_max_after_hpo.sh b/scripts/experiments/launch_champion_claim_max_after_hpo.sh deleted file mode 100644 index e4b2666..0000000 --- a/scripts/experiments/launch_champion_claim_max_after_hpo.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="/mnt/c/Users/carlos/Documents/Paper_CRPTO" -HPO_RUN_TAG="${HPO_RUN_TAG:-champion-reopen-2026-06-19__hpo-wave1}" -RUN_TAG="${RUN_TAG:-${HPO_RUN_TAG}__claim-max-downstream}" -TOP_K="${TOP_K:-4}" -MANDATORY_CASES="${MANDATORY_CASES:-pool93}" -PAPER_FACING_TOP_K="${PAPER_FACING_TOP_K:-3}" -SKIP_CASES="${SKIP_CASES:-}" -POLL_SECONDS="${POLL_SECONDS:-300}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${RUN_TAG}" -LOG_PATH="${LOG_DIR}/claim_max_downstream.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -{ - echo "TMUX_START $(date -Is) cwd=$(pwd)" - echo "HPO_RUN_TAG=${HPO_RUN_TAG} RUN_TAG=${RUN_TAG} TOP_K=${TOP_K} MANDATORY_CASES=${MANDATORY_CASES} PAPER_FACING_TOP_K=${PAPER_FACING_TOP_K} SKIP_CASES=${SKIP_CASES}" -} | tee -a "${LOG_PATH}" - -extra_args=() -if [[ -n "${SKIP_CASES}" ]]; then - extra_args+=(--skip-cases "${SKIP_CASES}") -fi - -set +e -.venv-champion-search/bin/python -u scripts/experiments/run_champion_claim_max_downstream.py \ - --hpo-run-tag "${HPO_RUN_TAG}" \ - --run-tag "${RUN_TAG}" \ - --top-k "${TOP_K}" \ - --mandatory-cases "${MANDATORY_CASES}" \ - --paper-facing-top-k "${PAPER_FACING_TOP_K}" \ - --wait-for-hpo-complete \ - --poll-seconds "${POLL_SECONDS}" \ - "${extra_args[@]}" \ - 2>&1 | tee -a "${LOG_PATH}" -code="${PIPESTATUS[0]}" -set -e - -echo "TMUX_EXIT ${code} $(date -Is)" | tee -a "${LOG_PATH}" -exit "${code}" diff --git a/scripts/experiments/launch_champion_hpo_wave1.sh b/scripts/experiments/launch_champion_hpo_wave1.sh deleted file mode 100644 index 39d5fa5..0000000 --- a/scripts/experiments/launch_champion_hpo_wave1.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="/mnt/c/Users/carlos/Documents/Paper_CRPTO" -RUN_TAG="${RUN_TAG:-champion-reopen-2026-06-19__hpo-wave1}" -N_TRIALS="${N_TRIALS:-96}" -SEED="${SEED:-42}" -TABPREP_SEED="${TABPREP_SEED:-42}" -CASES="${CASES:-pool93,pool93_business80,pool93_woe,pooltop72_tab60,pooltop80_tab90,pooltop93_tab120,pooltop80_business80,pooltop72_business80}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${RUN_TAG}" -LOG_PATH="${LOG_DIR}/hpo_wave1.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -{ - echo "TMUX_START $(date -Is) cwd=$(pwd)" - echo "RUN_TAG=${RUN_TAG} N_TRIALS=${N_TRIALS} SEED=${SEED} CASES=${CASES}" -} | tee -a "${LOG_PATH}" - -set +e -.venv-champion-search/bin/python -u scripts/experiments/run_champion_reopen_hpo.py \ - --config configs/experiments/champion_reopen.yaml \ - --run-tag "${RUN_TAG}" \ - --cases "${CASES}" \ - --seed "${SEED}" \ - --tabprep-seed "${TABPREP_SEED}" \ - --n-trials "${N_TRIALS}" \ - --full-data \ - --resume \ - 2>&1 | tee -a "${LOG_PATH}" -code="${PIPESTATUS[0]}" -set -e - -echo "TMUX_EXIT ${code} $(date -Is)" | tee -a "${LOG_PATH}" -exit "${code}" diff --git a/scripts/experiments/launch_conformal_parallel_after_current_inner.sh b/scripts/experiments/launch_conformal_parallel_after_current_inner.sh deleted file mode 100644 index 7280af2..0000000 --- a/scripts/experiments/launch_conformal_parallel_after_current_inner.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="${REPO_ROOT:-/mnt/c/Users/carlos/Documents/Paper_CRPTO}" -HPO_RUN_TAG="${HPO_RUN_TAG:-champion-reopen-2026-06-19__hpo-wave1}" -CASE_NAME="${CASE_NAME:-pool93}" -CONFORMAL_RUN_TAG="${CONFORMAL_RUN_TAG:-${HPO_RUN_TAG}__claim-max-incremental-conformal__${CASE_NAME}__conformal}" -UPSTREAM_RUN_TAG="${UPSTREAM_RUN_TAG:-${HPO_RUN_TAG}__${CASE_NAME}__seed42}" -PROFILE="${PROFILE:-search_conformal_claim_max}" -PHASE1_WORKERS="${PHASE1_WORKERS:-3}" -THREADS_PER_WORKER="${THREADS_PER_WORKER:-2}" -POLL_SECONDS="${POLL_SECONDS:-300}" -SERIAL_TMUX_SESSION="${SERIAL_TMUX_SESSION:-champion_claim_max_incremental_conformal}" -RUN_LABEL="${RUN_LABEL:-${CONFORMAL_RUN_TAG}__parallel-phase1}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${RUN_LABEL}" -LOG_PATH="${LOG_DIR}/parallel_conformal.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -FIRST_NAMESPACE="${CONFORMAL_RUN_TAG}__phase1__calfrac-0.50__holdout-0.20__seed-42" -FIRST_RESULT="${REPO_ROOT}/models/conformal_gap/${FIRST_NAMESPACE}/conformal_results_mondrian.pkl" - -{ - echo "PARALLEL_CONFORMAL_WAIT_START $(date -Is)" - echo "run_tag=${CONFORMAL_RUN_TAG}" - echo "upstream=${UPSTREAM_RUN_TAG}" - echo "first_checkpoint=${FIRST_RESULT}" - while [[ ! -f "${FIRST_RESULT}" ]]; do - echo "PARALLEL_CONFORMAL_WAITING_FIRST_CHECKPOINT $(date -Is)" - sleep "${POLL_SECONDS}" - done - - echo "PARALLEL_CONFORMAL_FIRST_CHECKPOINT_READY $(date -Is)" - if tmux has-session -t "${SERIAL_TMUX_SESSION}" 2>/dev/null; then - echo "PARALLEL_CONFORMAL_STOPPING_SERIAL_TMUX ${SERIAL_TMUX_SESSION} $(date -Is)" - tmux kill-session -t "${SERIAL_TMUX_SESSION}" || true - fi - - echo "PARALLEL_CONFORMAL_COMMAND_START $(date -Is)" - OMP_NUM_THREADS="${THREADS_PER_WORKER}" \ - MKL_NUM_THREADS="${THREADS_PER_WORKER}" \ - OPENBLAS_NUM_THREADS="${THREADS_PER_WORKER}" \ - NUMEXPR_NUM_THREADS="${THREADS_PER_WORKER}" \ - PYTHONUNBUFFERED=1 \ - nice -n 10 .venv-champion-search/bin/python -u scripts/search/run_conformal_reopen_search.py \ - --run-tag "${CONFORMAL_RUN_TAG}" \ - --pipeline-profile "${PROFILE}" \ - --upstream-canonical-run-tag "${UPSTREAM_RUN_TAG}" \ - --phase1-workers "${PHASE1_WORKERS}" - code="$?" - echo "PARALLEL_CONFORMAL_COMMAND_EXIT ${code} $(date -Is)" - exit "${code}" -} >> "${LOG_PATH}" 2>&1 diff --git a/scripts/experiments/launch_phase2_then_portfolio_after_conformal.sh b/scripts/experiments/launch_phase2_then_portfolio_after_conformal.sh deleted file mode 100644 index 6ff72c1..0000000 --- a/scripts/experiments/launch_phase2_then_portfolio_after_conformal.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="${REPO_ROOT:-/mnt/c/Users/carlos/Documents/Paper_CRPTO}" -HPO_RUN_TAG="${HPO_RUN_TAG:-champion-reopen-2026-06-19__hpo-wave1}" -CASE_NAME="${CASE_NAME:-pool93}" -CONFORMAL_RUN_TAG="${CONFORMAL_RUN_TAG:-${HPO_RUN_TAG}__claim-max-incremental-conformal__${CASE_NAME}__conformal}" -UPSTREAM_RUN_TAG="${UPSTREAM_RUN_TAG:-${HPO_RUN_TAG}__${CASE_NAME}__seed42}" -PROFILE="${PROFILE:-search_conformal_claim_max}" -PYTHON_BIN="${PYTHON_BIN:-${REPO_ROOT}/.venv-champion-search/bin/python}" -CUOPT_PYTHON="${CUOPT_PYTHON:-/home/eigenlinux/.venvs/crpto-cuopt-26-6/bin/python}" -POLL_SECONDS="${POLL_SECONDS:-300}" -PHASE1_WORKERS="${PHASE1_WORKERS:-3}" -THREADS_PER_WORKER="${THREADS_PER_WORKER:-2}" -RUN_LABEL="${RUN_LABEL:-${HPO_RUN_TAG}__${CASE_NAME}__cuopt-frontier-26-06}" -ORCHESTRATOR_LABEL="${ORCHESTRATOR_LABEL:-${CONFORMAL_RUN_TAG}__phase2-then-cuopt}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${ORCHESTRATOR_LABEL}" -LOG_PATH="${LOG_DIR}/phase2_then_cuopt.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -STATUS_PATH="${REPO_ROOT}/models/conformal_gap/${CONFORMAL_RUN_TAG}/conformal_reopen_status.json" - -phase2_ready() { - "${PYTHON_BIN}" - <> "${LOG_PATH}" 2>&1 diff --git a/scripts/experiments/launch_portfolio_cuopt_after_conformal_26_06.sh b/scripts/experiments/launch_portfolio_cuopt_after_conformal_26_06.sh deleted file mode 100644 index 206a4b9..0000000 --- a/scripts/experiments/launch_portfolio_cuopt_after_conformal_26_06.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="${REPO_ROOT:-/mnt/c/Users/carlos/Documents/Paper_CRPTO}" -CONFORMAL_RUN_TAG="${CONFORMAL_RUN_TAG:?Set CONFORMAL_RUN_TAG to the conformal reopen run tag}" -CUOPT_PYTHON="${CUOPT_PYTHON:-/home/eigenlinux/.venvs/crpto-cuopt-26-6/bin/python}" -RUN_LABEL="${RUN_LABEL:-${CONFORMAL_RUN_TAG}__cuopt-frontier-26-06}" -POLL_SECONDS="${POLL_SECONDS:-300}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${RUN_LABEL}" -LOG_PATH="${LOG_DIR}/wait_and_launch.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -STATUS_PATH="${REPO_ROOT}/models/conformal_gap/${CONFORMAL_RUN_TAG}/conformal_reopen_status.json" - -echo "CUOPT_WAIT_START $(date -Is) status=${STATUS_PATH}" | tee -a "${LOG_PATH}" - -while true; do - if [[ -f "${STATUS_PATH}" ]]; then - interval_path="$("${CUOPT_PYTHON}" - <&1 | tee -a "${LOG_PATH}" -import os -import subprocess -from pathlib import Path - -import yaml - -from scripts.experiments.run_champion_claim_max_downstream import _portfolio_command - -repo = Path(os.environ.get("REPO_ROOT", "/mnt/c/Users/carlos/Documents/Paper_CRPTO")) -profile_path = repo / os.environ.get( - "PROFILE_PATH", "configs/profiles/search_portfolio_cuopt_frontier_26_06.yaml" -) -run_label = os.environ["RUN_LABEL"] -conformal_path = Path(os.environ["CONFORMAL_INTERVALS_PATH"]) -cuopt_python = os.environ["CUOPT_PYTHON"] - -profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) or {} -profile.setdefault("execution", {})["python_executable"] = cuopt_python - -output_dir = ( - repo - / "data" - / "processed" - / "experiments" - / "champion_reopen" - / run_label - / "portfolio" -) -model_dir = ( - repo - / "models" - / "experiments" - / "champion_reopen" - / run_label - / "portfolio" -) - -cmd = _portfolio_command( - portfolio_profile=profile, - conformal_intervals_path=conformal_path, - run_label=run_label, - output_dir=output_dir, - model_dir=model_dir, -) -print("COMMAND", " ".join(map(str, cmd)), flush=True) -raise SystemExit(subprocess.run(cmd, cwd=str(repo), check=False).returncode) -PY -code="${PIPESTATUS[0]}" - -echo "CUOPT_FRONTIER_EXIT ${code} $(date -Is)" | tee -a "${LOG_PATH}" -if [[ "${code}" == "0" && "${RUN_EXACT_AFTER_FRONTIER}" == "true" ]]; then - echo "CUOPT_FRONTIER_EXACT_HANDOFF $(date -Is)" | tee -a "${LOG_PATH}" - RUN_LABEL="${RUN_LABEL}" \ - EXACT_PYTHON="${EXACT_PYTHON}" \ - scripts/experiments/launch_portfolio_exact_after_frontier.sh - code="$?" - echo "CUOPT_FRONTIER_EXACT_EXIT ${code} $(date -Is)" | tee -a "${LOG_PATH}" -fi -exit "${code}" diff --git a/scripts/experiments/launch_portfolio_exact_after_frontier.sh b/scripts/experiments/launch_portfolio_exact_after_frontier.sh deleted file mode 100644 index 8432aa4..0000000 --- a/scripts/experiments/launch_portfolio_exact_after_frontier.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="${REPO_ROOT:-/mnt/c/Users/carlos/Documents/Paper_CRPTO}" -RUN_LABEL="${RUN_LABEL:?Set RUN_LABEL to the frontier portfolio run label}" -EXACT_PYTHON="${EXACT_PYTHON:-${REPO_ROOT}/.venv-champion-search/bin/python}" -POLL_SECONDS="${POLL_SECONDS:-60}" -WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-0}" -CONTEXT_PATH="${CONTEXT_PATH:-${REPO_ROOT}/models/experiments/champion_reopen/${RUN_LABEL}/portfolio/portfolio_bound_aware_exact_context.json}" -LOG_DIR="${REPO_ROOT}/reports/run_logs/champion_reopen/${RUN_LABEL}" -LOG_PATH="${LOG_DIR}/exact_after_frontier.log" - -mkdir -p "${LOG_DIR}" -cd "${REPO_ROOT}" - -started_epoch="$(date +%s)" - -{ - echo "EXACT_AFTER_FRONTIER_WAIT_START $(date -Is)" - echo "RUN_LABEL=${RUN_LABEL}" - echo "CONTEXT_PATH=${CONTEXT_PATH}" - echo "EXACT_PYTHON=${EXACT_PYTHON}" -} >> "${LOG_PATH}" 2>&1 - -while [[ ! -f "${CONTEXT_PATH}" ]]; do - now_epoch="$(date +%s)" - elapsed="$((now_epoch - started_epoch))" - if [[ "${WAIT_TIMEOUT_SECONDS}" != "0" && "${elapsed}" -ge "${WAIT_TIMEOUT_SECONDS}" ]]; then - echo "EXACT_AFTER_FRONTIER_TIMEOUT elapsed=${elapsed} $(date -Is)" >> "${LOG_PATH}" 2>&1 - exit 124 - fi - echo "EXACT_AFTER_FRONTIER_WAITING elapsed=${elapsed} $(date -Is)" >> "${LOG_PATH}" 2>&1 - sleep "${POLL_SECONDS}" -done - -echo "EXACT_AFTER_FRONTIER_START $(date -Is)" >> "${LOG_PATH}" 2>&1 -OMP_NUM_THREADS="${EXACT_THREADS:-8}" \ -MKL_NUM_THREADS="${EXACT_THREADS:-8}" \ -OPENBLAS_NUM_THREADS="${EXACT_THREADS:-8}" \ -NUMEXPR_NUM_THREADS="${EXACT_THREADS:-8}" \ -EXACT_THREADS="${EXACT_THREADS:-8}" \ -PYTHONUNBUFFERED=1 \ -"${EXACT_PYTHON}" -u scripts/search/run_portfolio_bound_exact_eval.py \ - --context-path "${CONTEXT_PATH}" >> "${LOG_PATH}" 2>&1 -code="$?" -echo "EXACT_AFTER_FRONTIER_EXIT ${code} $(date -Is)" >> "${LOG_PATH}" 2>&1 -exit "${code}" diff --git a/scripts/experiments/monitor_champion_hpo_wave.py b/scripts/experiments/monitor_champion_hpo_wave.py deleted file mode 100644 index 1d5699b..0000000 --- a/scripts/experiments/monitor_champion_hpo_wave.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Lightweight Optuna/Champion HPO progress monitor. - -This script is intentionally read-only with respect to Optuna studies. It -polls per-case SQLite studies and writes a compact JSON status so long HPO -trials do not look stalled between CatBoost log lines. -""" - -from __future__ import annotations - -import argparse -import json -import sqlite3 -import time -from datetime import UTC, datetime -from pathlib import Path -from statistics import median -from typing import Any - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def _utc_now() -> str: - return datetime.now(tz=UTC).isoformat() - - -def _parse_dt(raw: str | None) -> datetime | None: - if not raw: - return None - try: - return datetime.fromisoformat(str(raw)) - except ValueError: - return None - - -def _duration_seconds(start: str | None, end: str | None) -> float | None: - started = _parse_dt(start) - finished = _parse_dt(end) - if started is None or finished is None: - return None - if started.tzinfo is None: - started = started.replace(tzinfo=UTC) - if finished.tzinfo is None: - finished = finished.replace(tzinfo=UTC) - return max(0.0, (finished - started).total_seconds()) - - -def _elapsed_since_start_seconds(start: str | None) -> float | None: - started = _parse_dt(start) - if started is None: - return None - if started.tzinfo is None: - return max(0.0, (datetime.now() - started).total_seconds()) - return max(0.0, (datetime.now(tz=UTC) - started).total_seconds()) - - -def _study_payload(db_path: Path, *, target_trials: int) -> dict[str, Any]: - if not db_path.exists(): - return { - "case_name": db_path.parents[1].name, - "optuna_db_path": str(db_path), - "exists": False, - } - con = sqlite3.connect(str(db_path)) - try: - state_rows = con.execute( - "select state, count(*) from trials group by state order by state" - ).fetchall() - trials = con.execute( - """ - select number, state, datetime_start, datetime_complete - from trials - order by number - """ - ).fetchall() - finally: - con.close() - - counts = {str(state): int(count) for state, count in state_rows} - done_count = int(counts.get("COMPLETE", 0) + counts.get("PRUNED", 0) + counts.get("FAIL", 0)) - durations = [ - value - for _, state, started, finished in trials - if state in {"COMPLETE", "PRUNED"} and (value := _duration_seconds(started, finished)) - ] - median_trial_sec = float(median(durations)) if durations else None - remaining_trials = max(0, int(target_trials) - done_count) - eta_sec = ( - float(remaining_trials * median_trial_sec) - if median_trial_sec is not None and remaining_trials > 0 - else 0.0 - if remaining_trials == 0 - else None - ) - running = [ - { - "number": int(number), - "datetime_start": started, - "elapsed_sec": _elapsed_since_start_seconds(started), - } - for number, state, started, _ in trials - if state == "RUNNING" - ] - return { - "case_name": db_path.parents[1].name, - "optuna_db_path": str(db_path), - "exists": True, - "target_trials": int(target_trials), - "state_counts": counts, - "done_trials": done_count, - "remaining_trials": remaining_trials, - "median_finished_trial_sec": median_trial_sec, - "eta_sec_from_median_finished_trials": eta_sec, - "running_trials": running, - "last_trials": [ - { - "number": int(number), - "state": str(state), - "datetime_start": started, - "datetime_complete": finished, - } - for number, state, started, finished in trials[-5:] - ], - } - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - tmp.replace(path) - - -def _collect(run_tag: str, *, seed: int, target_trials: int) -> dict[str, Any]: - root = REPO_ROOT / "models" / "experiments" / "champion_reopen" / run_tag - studies = [ - _study_payload(path, target_trials=target_trials) - for path in sorted(root.glob(f"*/seed_{int(seed)}/optuna_study.db")) - ] - active = [ - study - for study in studies - if study.get("exists") and int(study.get("remaining_trials", 0)) > 0 - ] - return { - "updated_at_utc": _utc_now(), - "run_tag": run_tag, - "seed": int(seed), - "target_trials": int(target_trials), - "n_studies": len(studies), - "active_cases": [str(study["case_name"]) for study in active], - "studies": studies, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--run-tag", default="champion-reopen-2026-06-19__hpo-wave1") - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--target-trials", type=int, default=96) - parser.add_argument("--poll-seconds", type=int, default=300) - parser.add_argument( - "--output-path", - default="", - help="Defaults to reports/...//hpo_live_status.json.", - ) - parser.add_argument("--once", action="store_true") - args = parser.parse_args() - - output_path = ( - Path(args.output_path) - if str(args.output_path).strip() - else REPO_ROOT - / "reports" - / "crpto" - / "experiments" - / "champion_reopen" - / str(args.run_tag) - / "hpo_live_status.json" - ) - while True: - _write_json( - output_path, - _collect(str(args.run_tag), seed=int(args.seed), target_trials=int(args.target_trials)), - ) - if args.once: - break - time.sleep(max(30, int(args.poll_seconds))) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/experiments/monitor_champion_reopen.py b/scripts/experiments/monitor_champion_reopen.py deleted file mode 100644 index 320ee6f..0000000 --- a/scripts/experiments/monitor_champion_reopen.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Print a compact leaderboard for champion-reopen experiment outputs.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--run-tag", default="champion-reopen-2026-06-19") - parser.add_argument("--root", default="models/experiments/champion_reopen") - parser.add_argument("--top", type=int, default=15) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - rows = _collect_status_rows(Path(args.root), run_tag=str(args.run_tag)) - if not rows: - print("No champion-reopen training status files found yet.") - return - rows.sort( - key=lambda row: ( - float(row.get("auc_roc", float("-inf"))), - -float(row.get("brier_score", float("inf"))), - -float(row.get("ece", float("inf"))), - ), - reverse=True, - ) - print(f"Champion-reopen leaderboard ({len(rows)} completed cases)") - print("rank auc_roc brier ece seed case run_tag") - for idx, row in enumerate(rows[: int(args.top)], start=1): - print( - f"{idx:>4} " - f"{float(row.get('auc_roc', 0.0)):.6f} " - f"{float(row.get('brier_score', 0.0)):.6f} " - f"{float(row.get('ece', 0.0)):.6f} " - f"{int(row.get('seed', 0)):>5} " - f"{row.get('case_name', '')} " - f"{row.get('run_tag', '')}" - ) - - -def _collect_status_rows(root: Path, *, run_tag: str) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - patterns = [ - f"{run_tag}*/**/selected_feature_training_status.json", - f"{run_tag}*/**/hpo_training_status.json", - ] - for pattern in patterns: - for path in root.glob(pattern): - payload = _read_json(path) - metrics = dict(payload.get("test_metrics", {}) or {}) - rows.append( - { - "path": str(path), - "run_tag": payload.get("run_tag"), - "case_name": payload.get("case_name"), - "seed": payload.get("seed"), - "n_model_features": payload.get("n_model_features"), - "auc_roc": metrics.get("auc_roc"), - "brier_score": metrics.get("brier_score"), - "ece": metrics.get("ece"), - "log_loss": metrics.get("log_loss"), - "pr_auc": metrics.get("pr_auc"), - } - ) - return rows - - -def _read_json(path: Path) -> dict[str, Any]: - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/run_champion_claim_max_downstream.py b/scripts/experiments/run_champion_claim_max_downstream.py deleted file mode 100644 index 4c90501..0000000 --- a/scripts/experiments/run_champion_claim_max_downstream.py +++ /dev/null @@ -1,655 +0,0 @@ -"""Run claim-max conformal and portfolio searches after champion HPO. - -This orchestration layer consumes paper-facing HPO candidate contracts under -``models/search_pd`` and writes only isolated experiment artifacts. It is meant -to wait behind a long HPO tmux run and then evaluate the best PD candidates on -the claims that matter for the paper: return, bound, robust region size, clean -region definition, and return-bound frontier. -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -import time -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] - -CHAMPION_AUC = 0.7138518124963467 -CHAMPION_BRIER = 0.15439302183275685 -CHAMPION_ECE = 0.006998009158194006 - -CUOPT_FLAG_MAP = { - "presolve": "--cuopt-presolve", - "method": "--cuopt-method", - "pdlp_solver_mode": "--cuopt-pdlp-solver-mode", - "pdlp_precision": "--cuopt-pdlp-precision", - "crossover": "--cuopt-crossover", - "first_primal_feasible": "--cuopt-first-primal-feasible", - "save_best_primal_solution": "--cuopt-save-best-primal-solution", - "infeasibility_detection": "--cuopt-infeasibility-detection", - "strict_infeasibility": "--cuopt-strict-infeasibility", - "per_constraint_residual": "--cuopt-per-constraint-residual", - "dual_postsolve": "--cuopt-dual-postsolve", - "dualize": "--cuopt-dualize", - "folding": "--cuopt-folding", - "augmented": "--cuopt-augmented", - "ordering": "--cuopt-ordering", - "cudss_deterministic": "--cuopt-cudss-deterministic", - "eliminate_dense_columns": "--cuopt-eliminate-dense-columns", - "iteration_limit": "--cuopt-iteration-limit", - "num_cpu_threads": "--cuopt-num-cpu-threads", - "num_gpus": "--cuopt-num-gpus", - "log_to_console": "--cuopt-log-to-console", - "log_dir": "--cuopt-log-dir", -} - - -def _utc_now() -> str: - return datetime.now(tz=UTC).isoformat() - - -def _read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - - -def _sanitize(raw: str) -> str: - return str(raw).strip().replace("/", "_").replace("\\", "_").replace(" ", "_") - - -def _parse_case_list(raw: str) -> list[str]: - return [_sanitize(part) for part in str(raw or "").split(",") if str(part).strip()] - - -def _is_paper_facing_case(case_name: str) -> bool: - name = _sanitize(case_name).lower() - return ( - name == "pool93" - or name.startswith("pool93_") - or name == "catboost44" - or name in {"canonical_4", "bureau_behavior_15"} - or "business" in name - ) - - -def _hpo_summary_path(hpo_run_tag: str, seed: int) -> Path: - return ( - REPO_ROOT - / "reports" - / "crpto" - / "experiments" - / "champion_reopen" - / hpo_run_tag - / "summary" - / f"seed_{int(seed)}" - / "hpo_experiment_summary.json" - ) - - -def _status_root(run_tag: str) -> Path: - return ( - REPO_ROOT - / "reports" - / "crpto" - / "experiments" - / "champion_reopen" - / run_tag - / "claim_max_downstream" - ) - - -def _log_root(run_tag: str) -> Path: - return REPO_ROOT / "reports" / "run_logs" / "champion_reopen" / run_tag / "claim_max" - - -def _load_hpo_candidates(hpo_run_tag: str, seed: int) -> list[dict[str, Any]]: - root = REPO_ROOT / "models" / "experiments" / "champion_reopen" / hpo_run_tag - rows: list[dict[str, Any]] = [] - for path in sorted(root.glob(f"*/seed_{int(seed)}/hpo_training_status.json")): - payload = _read_json(path) - metrics = dict(payload.get("test_metrics", {}) or {}) - if not metrics: - continue - case_name = str(payload.get("case_name", path.parents[1].name)) - paper_facing = _is_paper_facing_case(case_name) - auc = float(metrics.get("auc_roc", 0.0)) - brier = float(metrics.get("brier_score", 1.0)) - ece = float(metrics.get("ece", 1.0)) - score = ( - (auc - CHAMPION_AUC) * 1000.0 - - max(0.0, brier - CHAMPION_BRIER) * 80.0 - - max(0.0, ece - CHAMPION_ECE) * 8.0 - + (0.20 if paper_facing else 0.0) - ) - rows.append( - { - "case_name": case_name, - "candidate_run_tag": str(payload["candidate_run_tag"]), - "search_pd_dir": str(payload.get("search_pd_dir", "")), - "n_model_features": int(payload.get("n_model_features", 0)), - "n_generated_features": int(payload.get("n_generated_features", 0)), - "paper_facing": bool(paper_facing), - "auc_roc": auc, - "brier_score": brier, - "ece": ece, - "claim_pd_score": float(score), - "status_path": str(path), - } - ) - rows.sort( - key=lambda row: ( - float(row["claim_pd_score"]), - bool(row["paper_facing"]), - float(row["auc_roc"]), - -float(row["brier_score"]), - -float(row["ece"]), - ), - reverse=True, - ) - for rank, row in enumerate(rows, start=1): - row["rank"] = rank - return rows - - -def _select_downstream_candidates( - candidates: list[dict[str, Any]], - *, - top_k: int, - mandatory_cases: list[str] | tuple[str, ...] = (), - paper_facing_top_k: int = 0, - skip_cases: list[str] | tuple[str, ...] = (), -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Select PD candidates for downstream without losing paper-facing cases. - - The score rank is still the primary gate, but downstream effects can reverse - a small PD ranking difference. This keeps the best ranked candidates and - explicitly preserves candidates that are important for paper interpretation. - """ - - skip_set = {_sanitize(case_name) for case_name in skip_cases} - eligible_candidates = [ - row for row in candidates if _sanitize(str(row["case_name"])) not in skip_set - ] - case_to_row = {_sanitize(str(row["case_name"])): row for row in eligible_candidates} - reasons: dict[str, list[str]] = {} - missing_mandatory: list[str] = [] - - def add(case_name: str, reason: str) -> None: - key = _sanitize(case_name) - if key in skip_set: - return - if key not in case_to_row: - if reason == "mandatory_case": - missing_mandatory.append(key) - return - reasons.setdefault(key, []) - if reason not in reasons[key]: - reasons[key].append(reason) - - for row in eligible_candidates[: max(1, int(top_k))]: - add(str(row["case_name"]), "top_k_pd_score") - - for case_name in mandatory_cases: - add(case_name, "mandatory_case") - - if int(paper_facing_top_k) > 0: - n_added = 0 - for row in eligible_candidates: - if not bool(row.get("paper_facing")): - continue - add(str(row["case_name"]), "paper_facing_top_k") - n_added += 1 - if n_added >= int(paper_facing_top_k): - break - - selected: list[dict[str, Any]] = [] - for row in eligible_candidates: - key = _sanitize(str(row["case_name"])) - if key not in reasons: - continue - selected_row = dict(row) - selected_row["selection_reasons"] = reasons[key] - selected.append(selected_row) - - policy = { - "top_k": max(1, int(top_k)), - "mandatory_cases": list(mandatory_cases), - "paper_facing_top_k": max(0, int(paper_facing_top_k)), - "skip_cases": sorted(skip_set), - "missing_mandatory_cases": missing_mandatory, - "selected_case_names": [str(row["case_name"]) for row in selected], - "n_selected": len(selected), - } - return selected, policy - - -def _load_profile(path: str | Path) -> dict[str, Any]: - target = REPO_ROOT / path if not Path(path).is_absolute() else Path(path) - payload = yaml.safe_load(target.read_text(encoding="utf-8")) or {} - return dict(payload) if isinstance(payload, dict) else {} - - -def _run_logged( - *, - command: list[str], - log_path: Path, - cwd: Path, - env: dict[str, str] | None = None, -) -> int: - log_path.parent.mkdir(parents=True, exist_ok=True) - with log_path.open("a", encoding="utf-8") as log: - log.write(f"\nCOMMAND_START {_utc_now()}\n") - log.write(" ".join(command) + "\n") - log.flush() - proc = subprocess.run( - command, - cwd=str(cwd), - stdout=log, - stderr=subprocess.STDOUT, - text=True, - env=env, - check=False, - ) - log.write(f"COMMAND_EXIT {proc.returncode} {_utc_now()}\n") - return int(proc.returncode) - - -def _conformal_status_path(conformal_run_tag: str) -> Path: - return ( - REPO_ROOT / "models" / "conformal_gap" / conformal_run_tag / "conformal_reopen_status.json" - ) - - -def _conformal_intervals_for_status(status: dict[str, Any]) -> Path: - namespace = str(status["final_namespace"]) - return ( - REPO_ROOT - / "data" - / "processed" - / "conformal_gap" - / namespace - / "conformal_intervals_mondrian.parquet" - ) - - -def _profile_section(portfolio_profile: dict[str, Any], key: str) -> dict[str, Any]: - return dict(portfolio_profile.get(key, {}) or {}) - - -def _append_option(command: list[str], flag: str, value: Any) -> None: - if value is not None and str(value).strip(): - command.extend([flag, str(value)]) - - -def _append_int_option(command: list[str], flag: str, value: Any) -> None: - if value is not None and str(value).strip(): - command.extend([flag, str(int(value))]) - - -def _portfolio_base_command( - *, - portfolio_profile: dict[str, Any], - conformal_intervals_path: Path, - run_label: str, - output_dir: Path, - model_dir: Path, - grids: dict[str, Any], - frontier: dict[str, Any], - incumbent: dict[str, Any], - execution: dict[str, Any], -) -> list[str]: - python_executable = str(execution.get("python_executable") or sys.executable) - policy_modes = ",".join(str(x) for x in portfolio_profile.get("candidate_policy_families", [])) - return [ - python_executable, - "scripts/search/run_portfolio_bound_aware_search.py", - "--config", - "configs/crpto_optimization.yaml", - "--conformal-intervals-path", - str(conformal_intervals_path), - "--run-label", - run_label, - "--output-dir", - str(output_dir), - "--model-dir", - str(model_dir), - "--incumbent-policy-path", - "models/champion_portfolio_policy.json", - "--risk-grid", - str(grids["risk_grid"]), - "--gamma-grid", - str(grids["gamma_grid"]), - "--aversion-grid", - str(grids["aversion_grid"]), - "--delta-cap-grid", - str(grids["delta_cap_grid"]), - "--tail-focus-grid", - str(grids["tail_focus_grid"]), - "--alpha-grid", - str(grids["alpha_grid"]), - "--random-states", - str(grids["random_states"]), - "--max-candidates", - str(int(frontier.get("proxy_candidates_per_conformal_finalist", 100000))), - "--shortlist-top-k", - str(int(frontier.get("exact_rerank_top_k", 1000))), - "--bucket-return-k", - str(int(frontier.get("bucket_return_k", 200))), - "--bucket-proxy-k", - str(int(frontier.get("bucket_proxy_k", 200))), - "--bucket-family-k", - str(int(frontier.get("bucket_family_k", 100))), - "--bucket-region-k", - str(int(frontier.get("bucket_region_k", 200))), - "--incumbent-risk-neighbors", - str(incumbent["risk_neighbors"]), - "--incumbent-gamma-neighbors", - str(incumbent["gamma_neighbors"]), - "--incumbent-policy-modes", - str(incumbent["policy_modes"]), - "--policy-modes", - policy_modes, - "--solver-backend", - str(execution.get("solver_backend", "highs")), - "--exact-solver-backend", - str(execution.get("exact_solver_backend", "highs")), - ] - - -def _append_frontier_options( - command: list[str], - *, - grids: dict[str, Any], - frontier: dict[str, Any], -) -> None: - _append_int_option(command, "--exact-max-candidates", frontier.get("exact_max_candidates")) - exact_random_states = frontier.get("exact_random_states", grids.get("exact_random_states")) - _append_option(command, "--exact-random-states", exact_random_states) - _append_int_option(command, "--exact-checkpoint-every", frontier.get("exact_checkpoint_every")) - _append_int_option(command, "--exact-threads", frontier.get("exact_threads")) - _append_option(command, "--budget-profiles", grids.get("budget_profiles")) - - -def _append_execution_options(command: list[str], *, execution: dict[str, Any]) -> None: - if bool(execution.get("frontier_only", False)): - command.append("--frontier-only") - _append_option(command, "--exact-python-executable", execution.get("exact_python_executable")) - - -def _append_cuopt_options(command: list[str], *, cuopt: dict[str, Any]) -> None: - for key, flag in CUOPT_FLAG_MAP.items(): - _append_option(command, flag, cuopt.get(key)) - for key, value in dict(cuopt.get("extra_parameters", {}) or {}).items(): - command.extend(["--cuopt-extra-parameter", f"{key}={value}"]) - - -def _portfolio_command( - *, - portfolio_profile: dict[str, Any], - conformal_intervals_path: Path, - run_label: str, - output_dir: Path, - model_dir: Path, -) -> list[str]: - grids = _profile_section(portfolio_profile, "grids") - frontier = _profile_section(portfolio_profile, "frontier") - incumbent = _profile_section(portfolio_profile, "incumbent_region") - execution = _profile_section(portfolio_profile, "execution") - cuopt = _profile_section(portfolio_profile, "cuopt") - command = _portfolio_base_command( - portfolio_profile=portfolio_profile, - conformal_intervals_path=conformal_intervals_path, - run_label=run_label, - output_dir=output_dir, - model_dir=model_dir, - grids=grids, - frontier=frontier, - incumbent=incumbent, - execution=execution, - ) - _append_frontier_options(command, grids=grids, frontier=frontier) - _append_execution_options(command, execution=execution) - _append_cuopt_options(command, cuopt=cuopt) - return command - - -def _write_status(status_path: Path, payload: dict[str, Any]) -> None: - payload = {"updated_at_utc": _utc_now(), **payload} - _write_json(status_path, payload) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hpo-run-tag", default="champion-reopen-2026-06-19__hpo-wave1") - parser.add_argument("--run-tag", default="") - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--top-k", type=int, default=4) - parser.add_argument( - "--mandatory-cases", - default="pool93", - help="Comma-separated cases that must run downstream if their HPO artifacts exist.", - ) - parser.add_argument( - "--paper-facing-top-k", - type=int, - default=3, - help="Also include this many paper-facing cases by PD score, deduplicated with top-k.", - ) - parser.add_argument( - "--skip-cases", - default="", - help="Comma-separated cases managed by another lane and excluded from this watcher.", - ) - parser.add_argument("--wait-for-hpo-complete", action="store_true") - parser.add_argument("--poll-seconds", type=int, default=300) - parser.add_argument("--conformal-profile", default="search_conformal_claim_max") - parser.add_argument( - "--portfolio-profile-path", - default="configs/profiles/search_portfolio_claim_max.yaml", - ) - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--skip-portfolio", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - hpo_run_tag = _sanitize(args.hpo_run_tag) - run_tag = _sanitize(args.run_tag or f"{hpo_run_tag}__claim-max-downstream") - status_dir = _status_root(run_tag) - log_dir = _log_root(run_tag) - status_path = status_dir / "runtime_status.json" - portfolio_profile = _load_profile(args.portfolio_profile_path) - mandatory_cases = _parse_case_list(args.mandatory_cases) - skip_cases = _parse_case_list(args.skip_cases) - - while args.wait_for_hpo_complete and not _hpo_summary_path(hpo_run_tag, args.seed).exists(): - candidates = _load_hpo_candidates(hpo_run_tag, args.seed) - selected_preview, selection_policy_preview = _select_downstream_candidates( - candidates, - top_k=args.top_k, - mandatory_cases=mandatory_cases, - paper_facing_top_k=args.paper_facing_top_k, - skip_cases=skip_cases, - ) - _write_status( - status_path, - { - "stage_name": "champion_claim_max_downstream", - "state": "waiting", - "phase": "waiting_for_hpo_summary", - "hpo_run_tag": hpo_run_tag, - "completed_hpo_candidates": len(candidates), - "hpo_summary_path": str(_hpo_summary_path(hpo_run_tag, args.seed)), - "selection_policy_preview": selection_policy_preview, - "selected_candidate_preview": selected_preview, - }, - ) - time.sleep(max(30, int(args.poll_seconds))) - - candidates = _load_hpo_candidates(hpo_run_tag, args.seed) - selected, selection_policy = _select_downstream_candidates( - candidates, - top_k=args.top_k, - mandatory_cases=mandatory_cases, - paper_facing_top_k=args.paper_facing_top_k, - skip_cases=skip_cases, - ) - _write_status( - status_path, - { - "stage_name": "champion_claim_max_downstream", - "state": "completed" - if args.dry_run and selected - else ("running" if selected else "blocked"), - "phase": "selected_pd_candidates", - "hpo_run_tag": hpo_run_tag, - "run_tag": run_tag, - "selected_candidates": selected, - "selection_policy": selection_policy, - "n_available_hpo_candidates": len(candidates), - "dry_run": bool(args.dry_run), - }, - ) - if not selected: - return 2 - if args.dry_run: - return 0 - - results: list[dict[str, Any]] = [] - for candidate in selected: - case_name = _sanitize(str(candidate["case_name"])) - conformal_run_tag = _sanitize(f"{run_tag}__{case_name}__conformal") - conformal_status_path = _conformal_status_path(conformal_run_tag) - conformal_log = log_dir / f"{case_name}__conformal.log" - candidate_result: dict[str, Any] = { - "candidate": candidate, - "conformal_run_tag": conformal_run_tag, - "conformal_status_path": str(conformal_status_path), - } - if not conformal_status_path.exists(): - command = [ - sys.executable, - "scripts/search/run_conformal_reopen_search.py", - "--run-tag", - conformal_run_tag, - "--pipeline-profile", - str(args.conformal_profile), - "--upstream-canonical-run-tag", - str(candidate["candidate_run_tag"]), - ] - code = _run_logged(command=command, log_path=conformal_log, cwd=REPO_ROOT) - candidate_result["conformal_exit_code"] = int(code) - if code != 0: - candidate_result["state"] = "conformal_failed" - results.append(candidate_result) - _write_status( - status_path, - { - "stage_name": "champion_claim_max_downstream", - "state": "running", - "phase": "candidate_failed", - "latest_result": candidate_result, - "results": results, - }, - ) - continue - conformal_status = _read_json(conformal_status_path) - intervals_path = _conformal_intervals_for_status(conformal_status) - candidate_result["conformal_status"] = conformal_status - candidate_result["conformal_intervals_path"] = str(intervals_path) - if args.skip_portfolio: - candidate_result["state"] = "conformal_complete_portfolio_skipped" - results.append(candidate_result) - continue - portfolio_label = _sanitize(f"{run_tag}__{case_name}__portfolio") - portfolio_output = ( - REPO_ROOT - / "data" - / "processed" - / "experiments" - / "champion_reopen" - / run_tag - / case_name - / "portfolio" - ) - portfolio_model = ( - REPO_ROOT - / "models" - / "experiments" - / "champion_reopen" - / run_tag - / case_name - / "portfolio" - ) - selection_path = portfolio_model / "portfolio_bound_aware_selection.json" - if not selection_path.exists(): - command = _portfolio_command( - portfolio_profile=portfolio_profile, - conformal_intervals_path=intervals_path, - run_label=portfolio_label, - output_dir=portfolio_output, - model_dir=portfolio_model, - ) - code = _run_logged( - command=command, - log_path=log_dir / f"{case_name}__portfolio.log", - cwd=REPO_ROOT, - ) - candidate_result["portfolio_exit_code"] = int(code) - if code != 0: - candidate_result["state"] = "portfolio_failed" - results.append(candidate_result) - continue - candidate_result["portfolio_selection_path"] = str(selection_path) - candidate_result["state"] = "portfolio_complete" - if selection_path.exists(): - selection = _read_json(selection_path) - candidate_result["portfolio_selection"] = { - "selection_reason": selection.get("selection_reason"), - "selected_policy": selection.get("selected_policy"), - "selected_metrics": selection.get("selected_metrics"), - "region_summary_path": selection.get("region_summary_path"), - } - results.append(candidate_result) - _write_status( - status_path, - { - "stage_name": "champion_claim_max_downstream", - "state": "running", - "phase": "candidate_complete", - "latest_result": candidate_result, - "results": results, - }, - ) - - _write_status( - status_path, - { - "stage_name": "champion_claim_max_downstream", - "state": "completed", - "phase": "complete", - "hpo_run_tag": hpo_run_tag, - "run_tag": run_tag, - "selected_candidates": selected, - "selection_policy": selection_policy, - "results": results, - }, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/experiments/run_champion_reopen.py b/scripts/experiments/run_champion_reopen.py deleted file mode 100644 index a429956..0000000 --- a/scripts/experiments/run_champion_reopen.py +++ /dev/null @@ -1,356 +0,0 @@ -"""Orchestrate the CRPTO champion-reopen experiment waves. - -The script is intentionally conservative: it writes command manifests and -runtime status under experiment-only roots, and only launches expensive work -when ``--execute`` is supplied. -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -import time -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from src.utils.pipeline_runtime import atomic_write_json, write_runtime_status # noqa: E402 - - -@dataclass(frozen=True) -class ReopenCommand: - """One resumable command in a champion-reopen stage.""" - - name: str - stage: str - command: list[str] - expected_output: str - log_path: str - reason: str - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--config", - default="configs/experiments/champion_reopen.yaml", - help="Champion-reopen config.", - ) - parser.add_argument("--run-tag", default=None, help="Override config run_tag.") - parser.add_argument( - "--stage", - choices=["plan", "smoke", "feature_search", "seed_replay", "calibration", "all"], - default="plan", - ) - parser.add_argument( - "--execute", action="store_true", help="Run commands after writing manifest." - ) - parser.add_argument( - "--resume", action="store_true", help="Skip commands whose expected output exists." - ) - parser.add_argument( - "--sample-rows", - type=int, - default=None, - help="Override smoke sample rows for the smoke stage.", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - config = _load_config(Path(args.config)) - run_tag = str(args.run_tag or config.get("run_tag", "champion-reopen-2026-06-19")) - started = time.perf_counter() - commands = _build_commands( - config=config, - stage=str(args.stage), - run_tag=run_tag, - sample_rows_override=args.sample_rows, - ) - manifest_path = _write_command_manifest( - config=config, - run_tag=run_tag, - stage=str(args.stage), - commands=commands, - execute=bool(args.execute), - resume=bool(args.resume), - ) - status_path = _orchestration_status_path(config, run_tag=run_tag) - write_runtime_status( - "champion_reopen_orchestration", - phase="manifest_written", - state="running" if args.execute and commands else "planned", - run_tag=run_tag, - status_path=status_path, - extra={ - "stage": str(args.stage), - "execute": bool(args.execute), - "resume": bool(args.resume), - "command_count": len(commands), - "manifest_path": str(manifest_path), - }, - ) - if not args.execute: - print(f"Wrote command manifest: {manifest_path}") - return - - executed: list[dict[str, Any]] = [] - for item in commands: - expected = Path(item.expected_output) - if args.resume and expected.exists(): - executed.append({**asdict(item), "state": "skipped_existing_output"}) - continue - log_path = Path(item.log_path) - log_path.parent.mkdir(parents=True, exist_ok=True) - with log_path.open("w", encoding="utf-8") as log: - log.write("$ " + " ".join(item.command) + "\n\n") - log.flush() - proc = subprocess.run( - item.command, - cwd=REPO_ROOT, - stdout=log, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - state = "completed" if proc.returncode == 0 else "failed" - executed.append({**asdict(item), "state": state, "returncode": proc.returncode}) - write_runtime_status( - "champion_reopen_orchestration", - phase="command_complete", - state="running" if proc.returncode == 0 else "failed", - run_tag=run_tag, - status_path=status_path, - extra={ - "stage": str(args.stage), - "latest_command": item.name, - "latest_returncode": proc.returncode, - "executed": executed, - "elapsed_seconds": time.perf_counter() - started, - }, - ) - if proc.returncode != 0: - raise SystemExit(proc.returncode) - - write_runtime_status( - "champion_reopen_orchestration", - phase="complete", - state="completed", - run_tag=run_tag, - status_path=status_path, - extra={ - "stage": str(args.stage), - "executed": executed, - "elapsed_seconds": time.perf_counter() - started, - }, - ) - print(f"Completed stage {args.stage}; manifest: {manifest_path}") - - -def _load_config(path: Path) -> dict[str, Any]: - config = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(config, dict): - raise TypeError(f"Config must be a mapping: {path}") - config.setdefault("champion_reopen", {}) - config.setdefault("output", {}) - return config - - -def _build_commands( - *, - config: Mapping[str, Any], - stage: str, - run_tag: str, - sample_rows_override: int | None, -) -> list[ReopenCommand]: - if stage == "plan": - return [] - if stage == "all": - stages = ["smoke", "feature_search", "seed_replay"] - return [ - command - for substage in stages - for command in _build_commands( - config=config, - stage=substage, - run_tag=run_tag, - sample_rows_override=sample_rows_override, - ) - ] - if stage == "smoke": - sample_rows = int( - sample_rows_override - if sample_rows_override is not None - else config["champion_reopen"].get("smoke_sample_rows", 50000) - ) - return [ - _feature_selection_command( - config=config, - stage=stage, - run_tag=f"{run_tag}__smoke", - seed=42, - tabprep_seed=42, - cases=config["champion_reopen"]["smoke_cases"], - sample_rows=sample_rows, - reason="Smoke test ranking, guardrails, calibration, and output isolation.", - ) - ] - if stage == "feature_search": - return [ - _feature_selection_command( - config=config, - stage=stage, - run_tag=f"{run_tag}__feature_search", - seed=42, - tabprep_seed=42, - cases=config["champion_reopen"]["feature_search_cases"], - sample_rows=0, - reason="Full-data feature subset tournament on the canonical TabPrep space.", - ) - ] - if stage == "seed_replay": - seed_replay_seeds = config["champion_reopen"].get( - "seed_replay_seeds", config.get("seeds", [42, 52, 62, 72, 82]) - ) - return [ - _feature_selection_command( - config=config, - stage=stage, - run_tag=f"{run_tag}__seed_replay", - seed=int(seed), - tabprep_seed=42, - cases=config["champion_reopen"]["seed_replay_cases"], - sample_rows=0, - reason="Replay selected subsets across CatBoost seeds with fixed TabPrep features.", - ) - for seed in seed_replay_seeds - ] - if stage == "calibration": - cases = config["champion_reopen"]["seed_replay_cases"] - return [ - _feature_selection_command( - config=config, - stage=stage, - run_tag=f"{run_tag}__calibration_{method}", - seed=42, - tabprep_seed=42, - cases=cases, - sample_rows=0, - calibration_method=str(method), - reason=f"Calibration tournament lane for {method}.", - ) - for method in config.get("calibration", {}).get("candidates", []) - ] - raise ValueError(f"Unsupported stage: {stage}") - - -def _feature_selection_command( - *, - config: Mapping[str, Any], - stage: str, - run_tag: str, - seed: int, - tabprep_seed: int, - cases: Sequence[str], - sample_rows: int, - reason: str, - calibration_method: str | None = None, -) -> ReopenCommand: - command = [ - sys.executable, - "scripts/experiments/run_tabprep_feature_selection_catboost.py", - "--config", - "configs/experiments/champion_reopen.yaml", - "--run-tag", - run_tag, - "--seed", - str(seed), - "--tabprep-seed", - str(tabprep_seed), - "--variant", - "balanced_1500", - "--selector-model", - str(config["champion_reopen"]["selector_model"]), - "--ranking-method", - str(config["champion_reopen"].get("ranking_method", "pvc")), - "--shap-rows", - str(config["champion_reopen"].get("shap_rows", 30000)), - "--cases", - ",".join(cases), - ] - if sample_rows > 0: - command.extend(["--sample-rows", str(sample_rows)]) - else: - command.append("--full-data") - if calibration_method: - command.extend(["--calibration-method", calibration_method]) - summary = _summary_path(config, run_tag=run_tag, seed=seed) - return ReopenCommand( - name=f"{stage}:{run_tag}:seed_{seed}", - stage=stage, - command=command, - expected_output=str(summary), - log_path=str(_log_path(run_tag=run_tag, name=f"{stage}_seed_{seed}")), - reason=reason, - ) - - -def _write_command_manifest( - *, - config: Mapping[str, Any], - run_tag: str, - stage: str, - commands: Sequence[ReopenCommand], - execute: bool, - resume: bool, -) -> Path: - root = Path(config["output"]["report_dir"]) / run_tag / "orchestration" - payload = { - "run_tag": run_tag, - "stage": stage, - "execute": bool(execute), - "resume": bool(resume), - "command_count": len(commands), - "commands": [asdict(command) for command in commands], - "monitor_commands": [ - f"tail -f reports/run_logs/champion_reopen/{run_tag}*/*.log", - f"{sys.executable} scripts/experiments/monitor_champion_reopen.py --run-tag {run_tag}", - ], - } - return atomic_write_json(root / "command_manifest.json", payload) - - -def _summary_path(config: Mapping[str, Any], *, run_tag: str, seed: int) -> Path: - return ( - Path(config["output"]["report_dir"]) - / run_tag - / "summary" - / f"seed_{seed}" - / "selected_feature_experiment_summary.json" - ) - - -def _orchestration_status_path(config: Mapping[str, Any], *, run_tag: str) -> Path: - return Path(config["output"]["model_dir"]) / run_tag / "orchestration" / "runtime_status.json" - - -def _log_path(*, run_tag: str, name: str) -> Path: - safe = name.replace(":", "_").replace("/", "_") - return Path("reports/run_logs/champion_reopen") / run_tag / f"{safe}.log" - - -def _unique(items: Iterable[str]) -> list[str]: - return list(dict.fromkeys(str(item) for item in items)) - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/run_champion_reopen_hpo.py b/scripts/experiments/run_champion_reopen_hpo.py deleted file mode 100644 index efdb5cc..0000000 --- a/scripts/experiments/run_champion_reopen_hpo.py +++ /dev/null @@ -1,1002 +0,0 @@ -"""Run resumable CatBoost HPO for champion-reopen feature finalists. - -This runner stays paper-facing by writing only experiment/search artifacts. It -reuses the exact TabPrep feature construction and feature-subset resolver used -by the seed replay, then mirrors each tuned candidate into ``models/search_pd`` -so downstream conformal/portfolio scripts can consume it through -``UPSTREAM_CANONICAL_RUN_TAG`` without touching canonical artifacts. -""" - -from __future__ import annotations - -import argparse -import gc -import json -import shutil -import sys -import time -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from scripts.experiments.run_tabprep_challenger import ( # noqa: E402 - _apply_calibrator, - _build_fairness_report, - _case_dir, - _combine_features, - _model_params, - _prediction_frame, - _prepare_catboost_frame, - _resolve_sample_rows, - _sample_split, - _select_calibrator, - _validate_output_roots, -) -from scripts.experiments.run_tabprep_feature_selection_catboost import ( # noqa: E402 - _build_case_features, - _business_ranking, - _feature_importance_frame, - _load_config, - _load_selector_rankings, - _resolve_catboost_features, - _resolve_core_features, - _resolve_woe_features, -) -from src.evaluation.metrics import classification_metrics # noqa: E402 -from src.features.feature_config_io import load_feature_config # noqa: E402 -from src.features.feature_engineering import TARGET # noqa: E402 -from src.features.tabprep_challenger import ( # noqa: E402 - TabPrepChallengerTransformer, - resolve_tabprep_categorical_features, - resolve_tabprep_input_features, - validate_no_forbidden_features, -) -from src.models.optuna_tuning import train_catboost_tuned_optuna # noqa: E402 -from src.models.pd_contract import build_contract_payload # noqa: E402 -from src.models.pd_model import temporal_train_val_split # noqa: E402 -from src.utils.pipeline_runtime import ( # noqa: E402 - atomic_write_json, - atomic_write_parquet, - atomic_write_pickle, - write_runtime_checkpoint, - write_runtime_status, -) - -DEFAULT_HPO_CASES = [ - "pool93", - "pool93_business80", - "pool93_woe", - "pooltop72_tab60", - "pooltop80_tab90", - "pooltop93_tab120", - "pooltop80_business80", - "pooltop72_business80", -] - -DEFAULT_LOCAL_REFINE: dict[str, Any] = { - "enqueue_base_trial": True, - "fixed_params": { - "bootstrap_type": "MVS", - "grow_policy": "SymmetricTree", - }, - "iterations": {"low": 2600, "high": 5400, "step": 100}, - "learning_rate": {"low": 0.012, "high": 0.075, "log": True}, - "depth": {"choices": [7, 8, 9, 10]}, - "l2_leaf_reg": {"low": 25.0, "high": 320.0, "log": True}, - "min_data_in_leaf": {"low": 80, "high": 340, "step": 5}, - "rsm": {"low": 0.50, "high": 0.92}, - "random_strength": {"low": 1.0e-9, "high": 0.005, "log": True}, - "border_count": {"choices": [128, 148, 192, 254]}, - "subsample": {"low": 0.60, "high": 0.93}, - "leaf_estimation_iterations": {"choices": [2, 3, 4, 5, 6]}, - "penalties_coefficient": [0.50, 0.75, 1.0, 1.25, 1.5], - "feature_weights": { - "loan_to_income": [1.0, 1.15, 1.30], - "annual_inc": [1.0, 1.10, 1.20], - "dti": [1.0, 1.10, 1.20], - "installment_burden": [1.0, 1.10, 1.20], - "fico_score": [1.0, 1.10, 1.20], - }, - "first_feature_use_penalties": { - "delinq_recency": [0.0, 0.20, 0.50], - "recent_chargeoff": [0.0, 0.20, 0.50], - }, -} - -INCUMBENT_ENQUEUE_TRIALS = [ - { - "iterations": 3450, - "learning_rate": 0.030590100735681216, - "depth": 9, - "l2_leaf_reg": 125.73596604846647, - "min_data_in_leaf": 185, - "rsm": 0.66554189303626, - "random_strength": 1.7202215535069166e-06, - "border_count": 148, - "bootstrap_type": "MVS", - "subsample": 0.771052860633893, - "grow_policy": "SymmetricTree", - "leaf_estimation_iterations": 3, - }, - { - "iterations": 4200, - "learning_rate": 0.057321202729872456, - "depth": 8, - "l2_leaf_reg": 119.37272987133554, - "min_data_in_leaf": 135, - "rsm": 0.5716653769355704, - "random_strength": 1.3208942645900998e-07, - "border_count": 254, - "bootstrap_type": "MVS", - "subsample": 0.678878683066026, - "grow_policy": "SymmetricTree", - "leaf_estimation_iterations": 5, - }, -] - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", default="configs/experiments/champion_reopen.yaml") - parser.add_argument("--run-tag", default="champion-reopen-2026-06-19__hpo-wave1") - parser.add_argument("--cases", default="") - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--tabprep-seed", type=int, default=42) - parser.add_argument("--variant", default="balanced_1500") - parser.add_argument( - "--selector-model", - default=( - "models/experiments/tabprep/tabprep-catboost-full-2026-06-17/" - "balanced_1500/seed_42/pd_tabprep_challenger.cbm" - ), - ) - parser.add_argument("--ranking-method", choices=["pvc", "shap_blend"], default="pvc") - parser.add_argument("--shap-rows", type=int, default=30000) - parser.add_argument("--n-trials", type=int, default=None) - parser.add_argument("--timeout-minutes", type=int, default=None) - parser.add_argument("--sample-rows", type=int, default=None) - parser.add_argument("--full-data", action="store_true") - parser.add_argument("--resume", action="store_true") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - config = _load_config(Path(args.config)) - hpo_cfg = _resolve_hpo_cfg(config, args=args) - cases = _resolve_cases(config, args=args) - sample_rows = 0 if args.full_data else _resolve_sample_rows(config, args.sample_rows) - run_tag = str(args.run_tag) - seed = int(args.seed) - tabprep_seed = int(args.tabprep_seed) - started = time.perf_counter() - runtime_paths = _runtime_paths(config, run_tag=run_tag, seed=seed) - _write_runtime( - runtime_paths=runtime_paths, - phase="start", - state="running", - run_tag=run_tag, - extra={ - "seed": seed, - "tabprep_seed": tabprep_seed, - "variant": str(args.variant), - "cases": cases, - "sample_rows": sample_rows, - "hpo": hpo_cfg, - }, - ) - - train = pd.read_parquet(config["data"]["train_path"]) - calibration = pd.read_parquet(config["data"]["calibration_path"]) - test = pd.read_parquet(config["data"]["test_path"]) - if sample_rows > 0: - train = _sample_split(train, sample_rows=sample_rows, seed=seed) - calibration = _sample_split(calibration, sample_rows=sample_rows, seed=seed) - test = _sample_split(test, sample_rows=sample_rows, seed=seed) - - feature_config = load_feature_config( - yaml_path=Path(config["data"]["feature_config_path"]), - prefer="yaml", - ) - pool_features = resolve_tabprep_input_features( - train, - feature_config=feature_config, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - validate_no_forbidden_features( - pool_features, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - categorical_features = resolve_tabprep_categorical_features( - pool_features, - feature_config=feature_config, - ) - train_columns = [str(column) for column in train.columns] - core_features = _resolve_core_features(feature_config, train_columns) - catboost_features = _resolve_catboost_features(feature_config, train_columns) - woe_features = _resolve_woe_features(feature_config, train_columns) - - train_fit, train_val = temporal_train_val_split( - train, - val_fraction=float(config["validation"]["val_fraction"]), - date_col=str(config["validation"]["date_col"]), - ) - del train - gc.collect() - - cache_payload = ( - _load_selected_tabprep_cache( - config=config, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - variant=str(args.variant), - sample_rows=sample_rows, - cases=cases, - expected_rows={ - "train_fit": len(train_fit), - "train_val": len(train_val), - "calibration": len(calibration), - "test": len(test), - }, - ) - if args.resume - else None - ) - if cache_payload is not None: - logger.info( - "Loaded selected TabPrep cache for {}: {} generated columns", - run_tag, - len(cache_payload["generated_features"]), - ) - generated_train_fit = cache_payload["generated_train_fit"] - generated_train_val = cache_payload["generated_train_val"] - generated_calibration = cache_payload["generated_calibration"] - generated_test = cache_payload["generated_test"] - generated_features = cache_payload["generated_features"] - case_features = cache_payload["case_features"] - else: - transformer = TabPrepChallengerTransformer( - variant=str(args.variant), - input_features=pool_features, - categorical_features=categorical_features, - target=TARGET, - random_state=tabprep_seed, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - logger.info("Fitting TabPrep {} once for HPO finalists", args.variant) - generated_train_fit = transformer.fit_transform( - train_fit, - train_fit[TARGET].astype(int), - issue_dates=train_fit.get(str(config["validation"]["date_col"])), - ) - generated_train_val = transformer.transform(train_val) - generated_calibration = transformer.transform(calibration) - generated_test = transformer.transform(test) - generated_features_full = list(generated_train_fit.columns) - manifest = transformer.feature_manifest() - - pool_ranking, generated_ranking, ranking_diagnostics = _load_selector_rankings( - selector_model_path=Path(args.selector_model), - pool_features=pool_features, - generated_features=generated_features_full, - ranking_method=str(args.ranking_method), - shap_rows=int(args.shap_rows), - train_val=train_val, - train_fit=train_fit, - generated_train_val=generated_train_val, - categorical_features=categorical_features, - ) - business_ranking = _business_ranking(generated_ranking, manifest) - case_features = _build_case_features( - cases=cases, - core_features=core_features, - catboost_features=catboost_features, - pool_features=pool_features, - pool_ranking=pool_ranking, - woe_features=woe_features, - generated_ranking=generated_ranking, - business_ranking=business_ranking, - ) - _write_hpo_manifest( - config=config, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - case_features=case_features, - ranking_diagnostics=ranking_diagnostics, - manifest=manifest, - hpo_cfg=hpo_cfg, - ) - selected_generated_features = _selected_generated_features( - case_features=case_features, - generated_features=generated_features_full, - ) - generated_train_fit = generated_train_fit.loc[:, selected_generated_features].copy() - generated_train_val = generated_train_val.loc[:, selected_generated_features].copy() - generated_calibration = generated_calibration.loc[:, selected_generated_features].copy() - generated_test = generated_test.loc[:, selected_generated_features].copy() - generated_features = selected_generated_features - _write_selected_tabprep_cache( - config=config, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - variant=str(args.variant), - sample_rows=sample_rows, - cases=cases, - case_features=case_features, - generated_features=generated_features, - matrices={ - "train_fit": generated_train_fit, - "train_val": generated_train_val, - "calibration": generated_calibration, - "test": generated_test, - }, - ) - gc.collect() - - results: list[dict[str, Any]] = [] - for case_name, feature_set in case_features.items(): - status_path = ( - _case_dir( - Path(config["output"]["model_dir"]), - run_tag=run_tag, - case_name=case_name, - seed=seed, - ) - / "hpo_training_status.json" - ) - if args.resume and status_path.exists(): - logger.info("Skipping {} because status exists: {}", case_name, status_path) - results.append(json.loads(status_path.read_text(encoding="utf-8"))) - continue - _write_runtime( - runtime_paths=runtime_paths, - phase="case_running", - state="running", - run_tag=run_tag, - extra={ - "latest_case": case_name, - "completed_cases": [row["case_name"] for row in results], - "elapsed_seconds": time.perf_counter() - started, - }, - ) - generated_subset = [feature for feature in feature_set if feature in generated_features] - base_subset = [feature for feature in feature_set if feature not in generated_features] - result = _run_hpo_case( - case_name=case_name, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - config=config, - hpo_cfg=hpo_cfg, - train_fit=train_fit, - train_val=train_val, - calibration=calibration, - test=test, - base_features=base_subset, - categorical_features=categorical_features, - generated_train_fit=generated_train_fit[generated_subset], - generated_train_val=generated_train_val[generated_subset], - generated_calibration=generated_calibration[generated_subset], - generated_test=generated_test[generated_subset], - selection_sources={ - "core_features": core_features, - "pool_features": pool_features, - "woe_features": woe_features, - "generated_features": generated_subset, - }, - ) - results.append(result) - _write_runtime( - runtime_paths=runtime_paths, - phase="case_complete", - state="running", - run_tag=run_tag, - extra={ - "latest_case": case_name, - "completed_cases": [row["case_name"] for row in results], - "latest_auc": result.get("test_metrics", {}).get("auc_roc"), - "best_auc": max( - float(row.get("test_metrics", {}).get("auc_roc", float("-inf"))) - for row in results - ), - "elapsed_seconds": time.perf_counter() - started, - }, - ) - gc.collect() - - summary_dir = _case_dir( - Path(config["output"]["report_dir"]), - run_tag=run_tag, - case_name="summary", - seed=seed, - ) - summary = { - "run_tag": run_tag, - "seed": seed, - "tabprep_seed": tabprep_seed, - "elapsed_seconds": time.perf_counter() - started, - "cases": list(case_features), - "results": results, - } - atomic_write_json(summary_dir / "hpo_experiment_summary.json", _json_ready(summary)) - _write_checkpoint( - runtime_paths=runtime_paths, - checkpoint_name="summary", - payload=summary, - ) - _write_runtime( - runtime_paths=runtime_paths, - phase="complete", - state="completed", - run_tag=run_tag, - extra={ - "elapsed_seconds": time.perf_counter() - started, - "summary_path": str(summary_dir / "hpo_experiment_summary.json"), - "completed_cases": [row["case_name"] for row in results], - }, - ) - - -def _resolve_cases(config: Mapping[str, Any], *, args: argparse.Namespace) -> list[str]: - if str(args.cases).strip(): - return [case.strip() for case in str(args.cases).split(",") if case.strip()] - configured = config.get("champion_reopen_hpo", {}).get("cases") - if configured: - return [str(case).strip() for case in configured if str(case).strip()] - return list(DEFAULT_HPO_CASES) - - -def _resolve_hpo_cfg(config: Mapping[str, Any], *, args: argparse.Namespace) -> dict[str, Any]: - raw = dict(config.get("champion_reopen_hpo", {}) or {}) - hpo = dict(raw.get("optuna", {}) or {}) - hpo.setdefault("n_trials", 180) - hpo.setdefault("timeout_minutes", 0) - hpo.setdefault("sampler", "tpe") - hpo.setdefault("pruner", "median") - hpo.setdefault("n_startup_trials", 40) - hpo.setdefault("multivariate_tpe", True) - hpo.setdefault("group_tpe", True) - hpo.setdefault("constant_liar", True) - hpo.setdefault("pruner_n_startup_trials", 20) - hpo.setdefault("pruner_n_warmup_steps", 75) - hpo.setdefault("use_pruning_callback", True) - hpo.setdefault("load_if_exists", True) - hpo.setdefault("refit_full_train", True) - hpo.setdefault("gc_after_trial", True) - hpo.setdefault("storage_heartbeat_interval", 60) - hpo.setdefault("storage_grace_period", 240) - hpo.setdefault("sqlite_timeout_seconds", 180) - hpo.setdefault("retry_failed_trials", 2) - hpo.setdefault("n_jobs", 1) - hpo.setdefault("search_space_mode", "local_refine") - hpo.setdefault("search_space_version", "cb_local_refine_champion_reopen_v1") - hpo.setdefault("local_refine", DEFAULT_LOCAL_REFINE) - hpo.setdefault( - "constraints_policy", - { - "max_brier_delta": 0.001, - "max_ece_delta": 0.0025, - "min_auc_delta": -0.001, - }, - ) - hpo.setdefault("enqueue_trials", INCUMBENT_ENQUEUE_TRIALS) - if args.n_trials is not None: - hpo["n_trials"] = int(args.n_trials) - if args.timeout_minutes is not None: - hpo["timeout_minutes"] = int(args.timeout_minutes) - return hpo - - -def _runtime_paths(config: Mapping[str, Any], *, run_tag: str, seed: int) -> dict[str, Path]: - model_dir = _case_dir( - Path(config["output"]["model_dir"]), - run_tag=run_tag, - case_name="runtime", - seed=seed, - ) - _validate_output_roots([model_dir]) - return { - "status": model_dir / "runtime_status.json", - "checkpoints": model_dir / "checkpoints", - } - - -def _write_runtime( - *, - runtime_paths: Mapping[str, Path], - phase: str, - state: str, - run_tag: str, - extra: Mapping[str, Any] | None = None, -) -> None: - write_runtime_status( - "champion_reopen_hpo", - phase=phase, - state=state, - run_tag=run_tag, - status_path=runtime_paths["status"], - extra=_json_ready(dict(extra or {})), - ) - - -def _write_checkpoint( - *, - runtime_paths: Mapping[str, Path], - checkpoint_name: str, - payload: Mapping[str, Any], -) -> None: - write_runtime_checkpoint( - "champion_reopen_hpo", - checkpoint_name, - _json_ready(dict(payload)), - checkpoint_dir=runtime_paths["checkpoints"], - ) - - -def _selected_generated_features( - *, - case_features: Mapping[str, Sequence[str]], - generated_features: Sequence[str], -) -> list[str]: - generated_set = set(map(str, generated_features)) - selected = { - str(feature) - for features in case_features.values() - for feature in features - if str(feature) in generated_set - } - return [feature for feature in generated_features if feature in selected] - - -def _tabprep_cache_dir(config: Mapping[str, Any], *, run_tag: str, seed: int) -> Path: - cache_dir = _case_dir( - Path(config["output"]["data_dir"]), - run_tag=run_tag, - case_name="tabprep_selected_cache", - seed=seed, - ) - _validate_output_roots([cache_dir]) - return cache_dir - - -def _tabprep_cache_paths(config: Mapping[str, Any], *, run_tag: str, seed: int) -> dict[str, Path]: - cache_dir = _tabprep_cache_dir(config, run_tag=run_tag, seed=seed) - return { - "dir": cache_dir, - "meta": cache_dir / "cache_meta.json", - "train_fit": cache_dir / "generated_train_fit.parquet", - "train_val": cache_dir / "generated_train_val.parquet", - "calibration": cache_dir / "generated_calibration.parquet", - "test": cache_dir / "generated_test.parquet", - } - - -def _cache_signature( - *, - run_tag: str, - seed: int, - tabprep_seed: int, - variant: str, - sample_rows: int, - cases: Sequence[str], -) -> dict[str, Any]: - return { - "run_tag": str(run_tag), - "seed": int(seed), - "tabprep_seed": int(tabprep_seed), - "variant": str(variant), - "sample_rows": int(sample_rows), - "cases": [str(case) for case in cases], - } - - -def _load_selected_tabprep_cache( - *, - config: Mapping[str, Any], - run_tag: str, - seed: int, - tabprep_seed: int, - variant: str, - sample_rows: int, - cases: Sequence[str], - expected_rows: Mapping[str, int], -) -> dict[str, Any] | None: - paths = _tabprep_cache_paths(config, run_tag=run_tag, seed=seed) - required = [paths[key] for key in ["meta", "train_fit", "train_val", "calibration", "test"]] - if not all(path.exists() for path in required): - return None - try: - meta = json.loads(paths["meta"].read_text(encoding="utf-8")) - expected = _cache_signature( - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - variant=variant, - sample_rows=sample_rows, - cases=cases, - ) - for key, value in expected.items(): - if meta.get(key) != value: - logger.warning( - "Ignoring selected TabPrep cache because {} differs: cached={} expected={}", - key, - meta.get(key), - value, - ) - return None - matrices = { - split: pd.read_parquet(paths[split]) - for split in ["train_fit", "train_val", "calibration", "test"] - } - row_counts = {split: len(frame) for split, frame in matrices.items()} - for split, expected_count in expected_rows.items(): - if int(row_counts.get(split, -1)) != int(expected_count): - logger.warning( - "Ignoring selected TabPrep cache because {} rows differ: cached={} expected={}", - split, - row_counts.get(split), - expected_count, - ) - return None - generated_features = [str(feature) for feature in meta.get("generated_features", [])] - case_features = { - str(case): [str(feature) for feature in features] - for case, features in dict(meta.get("case_features", {}) or {}).items() - } - if not generated_features and any( - feature.startswith("tabprep__") - for features in case_features.values() - for feature in features - ): - return None - return { - "generated_train_fit": matrices["train_fit"], - "generated_train_val": matrices["train_val"], - "generated_calibration": matrices["calibration"], - "generated_test": matrices["test"], - "generated_features": generated_features, - "case_features": case_features, - "meta": meta, - } - except Exception as exc: - logger.warning("Failed to load selected TabPrep cache; recomputing. reason={}", exc) - return None - - -def _write_selected_tabprep_cache( - *, - config: Mapping[str, Any], - run_tag: str, - seed: int, - tabprep_seed: int, - variant: str, - sample_rows: int, - cases: Sequence[str], - case_features: Mapping[str, Sequence[str]], - generated_features: Sequence[str], - matrices: Mapping[str, pd.DataFrame], -) -> None: - paths = _tabprep_cache_paths(config, run_tag=run_tag, seed=seed) - logger.info( - "Writing selected TabPrep cache for {} with {} generated columns", - run_tag, - len(generated_features), - ) - for split in ["train_fit", "train_val", "calibration", "test"]: - atomic_write_parquet(matrices[split], paths[split], index=False) - meta = { - **_cache_signature( - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - variant=variant, - sample_rows=sample_rows, - cases=cases, - ), - "generated_features": [str(feature) for feature in generated_features], - "n_generated_features": int(len(generated_features)), - "case_features": { - str(case): [str(feature) for feature in features] - for case, features in case_features.items() - }, - "row_counts": {split: int(len(frame)) for split, frame in matrices.items()}, - "paths": {key: str(path) for key, path in paths.items() if key != "dir"}, - } - atomic_write_json(paths["meta"], _json_ready(meta)) - - -def _run_hpo_case( - *, - case_name: str, - run_tag: str, - seed: int, - tabprep_seed: int, - config: Mapping[str, Any], - hpo_cfg: Mapping[str, Any], - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - calibration: pd.DataFrame, - test: pd.DataFrame, - base_features: Sequence[str], - categorical_features: Sequence[str], - generated_train_fit: pd.DataFrame, - generated_train_val: pd.DataFrame, - generated_calibration: pd.DataFrame, - generated_test: pd.DataFrame, - selection_sources: Mapping[str, Any], -) -> dict[str, Any]: - model_dir = _case_dir( - Path(config["output"]["model_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - data_dir = _case_dir( - Path(config["output"]["data_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - report_dir = _case_dir( - Path(config["output"]["report_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - _validate_output_roots([model_dir, data_dir, report_dir]) - - generated_features = list(generated_train_fit.columns) - model_features = [*base_features, *generated_features] - cat_features = [feature for feature in categorical_features if feature in model_features] - - x_train = _combine_features(train_fit, base_features, generated_train_fit) - x_val = _combine_features(train_val, base_features, generated_train_val) - x_cal = _combine_features(calibration, base_features, generated_calibration) - x_test = _combine_features(test, base_features, generated_test) - - x_train_cb = _prepare_catboost_frame(x_train, model_features, cat_features) - x_val_cb = _prepare_catboost_frame(x_val, model_features, cat_features) - x_cal_cb = _prepare_catboost_frame(x_cal, model_features, cat_features) - x_test_cb = _prepare_catboost_frame(x_test, model_features, cat_features) - - params = _model_params(config, model_features=model_features, seed=seed) - storage_db = (model_dir / "optuna_study.db").resolve() - storage_db.parent.mkdir(parents=True, exist_ok=True) - study_storage = f"sqlite:///{storage_db.as_posix()}" - study_name = f"champion_reopen_hpo_{case_name}_seed{seed}" - model, train_metrics = train_catboost_tuned_optuna( - x_train_cb, - train_fit[TARGET].astype(int), - x_val_cb, - train_val[TARGET].astype(int), - X_test=x_test_cb, - y_test=test[TARGET].astype(int), - cat_features=cat_features, - base_params=params, - n_trials=int(hpo_cfg["n_trials"]), - sampler=str(hpo_cfg["sampler"]), - pruner=str(hpo_cfg["pruner"]), - timeout_minutes=int(hpo_cfg["timeout_minutes"]), - n_startup_trials=int(hpo_cfg["n_startup_trials"]), - multivariate_tpe=bool(hpo_cfg["multivariate_tpe"]), - group_tpe=bool(hpo_cfg["group_tpe"]), - constant_liar=bool(hpo_cfg["constant_liar"]), - pruner_n_startup_trials=int(hpo_cfg["pruner_n_startup_trials"]), - pruner_n_warmup_steps=int(hpo_cfg["pruner_n_warmup_steps"]), - use_pruning_callback=bool(hpo_cfg["use_pruning_callback"]), - study_storage=study_storage, - study_name=study_name, - load_if_exists=bool(hpo_cfg["load_if_exists"]), - refit_full_train=bool(hpo_cfg["refit_full_train"]), - gc_after_trial=bool(hpo_cfg["gc_after_trial"]), - storage_heartbeat_interval=int(hpo_cfg["storage_heartbeat_interval"]), - storage_grace_period=int(hpo_cfg["storage_grace_period"]), - sqlite_timeout_seconds=int(hpo_cfg["sqlite_timeout_seconds"]), - retry_failed_trials=int(hpo_cfg["retry_failed_trials"]), - n_jobs=int(hpo_cfg["n_jobs"]), - search_space_mode=str(hpo_cfg["search_space_mode"]), - local_refine_space=dict(hpo_cfg["local_refine"]), - constraints_policy=dict(hpo_cfg["constraints_policy"]), - search_space_version=str(hpo_cfg["search_space_version"]), - enqueue_trials=list(hpo_cfg["enqueue_trials"]), - ) - - raw_cal = model.predict_proba(x_cal_cb)[:, 1] - raw_test = model.predict_proba(x_test_cb)[:, 1] - calibrator, calibration_report = _select_calibrator( - method=str(config["calibration"]["method"]), - candidates=[str(item) for item in config["calibration"]["candidates"]], - y_cal=calibration[TARGET].astype(int).to_numpy(), - raw_cal=raw_cal, - ) - calibrated_test = np.asarray(_apply_calibrator(calibrator, raw_test), dtype=float) - calibrated_cal = np.asarray(_apply_calibrator(calibrator, raw_cal), dtype=float) - test_metrics = classification_metrics(test[TARGET].astype(int).to_numpy(), calibrated_test) - cal_metrics = classification_metrics( - calibration[TARGET].astype(int).to_numpy(), - calibrated_cal, - ) - fairness = _build_fairness_report(test, y_prob=calibrated_test, config=config) - - model_dir.mkdir(parents=True, exist_ok=True) - data_dir.mkdir(parents=True, exist_ok=True) - report_dir.mkdir(parents=True, exist_ok=True) - - model_path = model_dir / "pd_selected_tabprep_hpo.cbm" - calibrator_path = model_dir / "pd_selected_tabprep_hpo_calibrator.pkl" - prediction_path = data_dir / "test_predictions.parquet" - status_path = model_dir / "hpo_training_status.json" - model.save_model(str(model_path)) - atomic_write_pickle(calibrator_path, calibrator) - predictions = _prediction_frame( - test, - raw_prob=raw_test, - calibrated_prob=calibrated_test, - case_name=case_name, - seed=seed, - ) - atomic_write_parquet(predictions, prediction_path) - if not fairness.empty: - atomic_write_parquet(fairness, report_dir / "fairness_report.parquet") - atomic_write_parquet( - _feature_importance_frame(model), report_dir / "feature_importance.parquet" - ) - - candidate_run_tag = _candidate_run_tag(run_tag=run_tag, case_name=case_name, seed=seed) - search_pd_dir = Path("models/search_pd") / candidate_run_tag - search_data_dir = Path("data/processed/search_pd") / candidate_run_tag - search_pd_dir.mkdir(parents=True, exist_ok=True) - search_data_dir.mkdir(parents=True, exist_ok=True) - candidate_model_path = search_pd_dir / "pd_candidate_model.cbm" - candidate_calibrator_path = search_pd_dir / "pd_candidate_calibrator.pkl" - shutil.copy2(model_path, candidate_model_path) - shutil.copy2(model_path, search_pd_dir / "pd_local_hpo_tuned.cbm") - shutil.copy2(calibrator_path, candidate_calibrator_path) - shutil.copy2(calibrator_path, search_pd_dir / "pd_local_hpo_calibrator.pkl") - - calibration_matrix_path = search_data_dir / "calibration_model_matrix.parquet" - test_matrix_path = search_data_dir / "test_model_matrix.parquet" - atomic_write_parquet( - _matrix_with_identity(x_cal_cb, calibration), - calibration_matrix_path, - index=False, - ) - atomic_write_parquet( - _matrix_with_identity(x_test_cb, test), - test_matrix_path, - index=False, - ) - atomic_write_parquet(predictions, search_data_dir / "test_predictions.parquet") - contract = build_contract_payload( - model_path=candidate_model_path, - calibrator_path=candidate_calibrator_path, - feature_names=list(model_features), - categorical_features=list(cat_features), - split_shapes={ - "train_fit": tuple(x_train_cb.shape), - "train_val": tuple(x_val_cb.shape), - "calibration": tuple(x_cal_cb.shape), - "test": tuple(x_test_cb.shape), - }, - ) - contract["run_tag"] = candidate_run_tag - contract["source_experiment_run_tag"] = run_tag - contract["case_name"] = case_name - contract["seed"] = int(seed) - contract["tabprep_seed"] = int(tabprep_seed) - contract["model_matrix_paths"] = { - "calibration": str(calibration_matrix_path), - "test": str(test_matrix_path), - } - atomic_write_json(search_pd_dir / "pd_model_contract.json", _json_ready(contract)) - - status = { - "run_tag": run_tag, - "candidate_run_tag": candidate_run_tag, - "case_name": case_name, - "seed": seed, - "tabprep_seed": tabprep_seed, - "model_path": str(model_path), - "calibrator_path": str(calibrator_path), - "prediction_path": str(prediction_path), - "search_pd_dir": str(search_pd_dir), - "search_data_dir": str(search_data_dir), - "upstream_canonical_run_tag": candidate_run_tag, - "n_base_features": len(base_features), - "n_generated_features": len(generated_features), - "n_model_features": len(model_features), - "categorical_features": cat_features, - "generated_features": generated_features, - "training_metrics": train_metrics, - "test_metrics": test_metrics, - "calibration_metrics": cal_metrics, - "calibration_selection": calibration_report, - "selection_sources": selection_sources, - "optuna_storage": study_storage, - "optuna_study_name": study_name, - "hpo": dict(hpo_cfg), - } - atomic_write_json(status_path, _json_ready(status)) - atomic_write_json(search_pd_dir / "pd_training_status.json", _json_ready(status)) - logger.info( - "{} HPO seed {} done: test AUC={:.6f}, Brier={:.6f}, ECE={:.6f}, features={} generated={}", - case_name, - seed, - test_metrics["auc_roc"], - test_metrics["brier_score"], - test_metrics["ece"], - len(model_features), - len(generated_features), - ) - return status - - -def _matrix_with_identity(matrix: pd.DataFrame, source: pd.DataFrame) -> pd.DataFrame: - out = matrix.reset_index(drop=True).copy() - source_reset = source.reset_index(drop=True) - for col in ["id", "issue_d", "grade", TARGET]: - if col in source_reset.columns and col not in out.columns: - out[col] = source_reset[col].to_numpy() - return out - - -def _candidate_run_tag(*, run_tag: str, case_name: str, seed: int) -> str: - raw = f"{run_tag}__{case_name}__seed{int(seed)}" - return raw.replace("/", "_").replace(" ", "_") - - -def _write_hpo_manifest( - *, - config: Mapping[str, Any], - run_tag: str, - seed: int, - tabprep_seed: int, - case_features: Mapping[str, Sequence[str]], - ranking_diagnostics: pd.DataFrame, - manifest: pd.DataFrame, - hpo_cfg: Mapping[str, Any], -) -> None: - report_dir = _case_dir( - Path(config["output"]["report_dir"]), - run_tag=run_tag, - case_name="hpo_manifest", - seed=seed, - ) - report_dir.mkdir(parents=True, exist_ok=True) - atomic_write_json( - report_dir / "hpo_manifest.json", - _json_ready( - { - "run_tag": run_tag, - "seed": seed, - "tabprep_seed": tabprep_seed, - "cases": {name: list(features) for name, features in case_features.items()}, - "hpo": dict(hpo_cfg), - } - ), - ) - if not ranking_diagnostics.empty: - atomic_write_parquet(ranking_diagnostics, report_dir / "pool_ranking_diagnostics.parquet") - atomic_write_parquet(manifest, report_dir / "generated_feature_manifest.parquet") - - -def _json_ready(value: Any) -> Any: - if isinstance(value, dict): - return {str(key): _json_ready(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_json_ready(item) for item in value] - if isinstance(value, np.generic): - return value.item() - if isinstance(value, np.ndarray): - return [_json_ready(item) for item in value.tolist()] - if isinstance(value, Path): - return str(value) - if isinstance(value, float) and not np.isfinite(value): - return None - return value - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/run_ijds_allocation_granularity_sensitivity.py b/scripts/experiments/run_ijds_allocation_granularity_sensitivity.py new file mode 100644 index 0000000..1f24907 --- /dev/null +++ b/scripts/experiments/run_ijds_allocation_granularity_sensitivity.py @@ -0,0 +1,317 @@ +"""Run the two-phase USD 25 allocation-granularity sensitivity.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml + +from src.ijds_audit.allocation_granularity import ( + floor_allocations_to_lot, + granularity_contrast_bounds, + rounded_solve_records, +) +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.protocol import configured_archive_outcomes, load_outcome_universe +from src.utils.isolated_experiment import ( + environment_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, +) +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = ( + ROOT / "configs/experiments/ijds_allocation_granularity_sensitivity_2026-07-16.yaml" +) +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("freeze", "evaluate")) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Allocation-granularity config must be a mapping.") + sensitivity = payload.get("allocation_granularity", {}) + if float(sensitivity.get("lot_size_usd", 0.0)) != 25.0: + raise ValueError("The allocation-granularity lot must remain USD 25.") + if float(sensitivity.get("committed_budget_usd", 0.0)) != 1_000_000.0: + raise ValueError("The allocation-granularity budget must remain USD 1 million.") + if sensitivity.get("rounding_rule") != "floor_each_exposure_hold_residual_as_cash": + raise ValueError("Allocation-granularity rounding rule changed.") + if sensitivity.get("outcome_based_selection") is not False: + raise ValueError("Allocation granularity cannot select from outcomes.") + periods = tuple(str(value) for value in sensitivity.get("periods", ())) + if len(periods) != 15 or len(set(periods)) != 15: + raise ValueError("Allocation granularity must retain all 15 primary OOT months.") + return payload + + +def _verified_descriptor(descriptor: Mapping[str, Any], *, repo_root: Path) -> Path: + path = resolve_repo_input(str(descriptor["path"]), repo_root=repo_root) + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor.get(field): + raise RuntimeError(f"Allocation-granularity parent mismatch for {field}.") + return path + + +def _run_dirs(config: Mapping[str, Any], root: Path) -> tuple[Path, Path]: + return ( + resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=str(config["run_tag"]), + ), + resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(config["run_tag"]), + ), + ) + + +def freeze(config_path: Path, *, repo_root: Path) -> Path: + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + allocation_path = _verified_descriptor(config["parent"]["allocations"], repo_root=root) + record_path = _verified_descriptor(config["parent"]["solve_records"], repo_root=root) + allocations = pd.read_parquet(allocation_path) + records = pd.read_parquet(record_path) + sensitivity = config["allocation_granularity"] + budget = float(sensitivity["committed_budget_usd"]) + rounded, audit = floor_allocations_to_lot( + allocations, + lot_size=float(sensitivity["lot_size_usd"]), + committed_budget=budget, + tolerance=float(sensitivity["numerical_tolerance"]), + ) + rounded_records = rounded_solve_records( + records, + rounded, + audit, + committed_budget=budget, + ) + if len(audit) != int(sensitivity["expected_portfolios"]): + raise RuntimeError("Allocation-granularity portfolio census changed.") + artifact_paths = { + "rounded_allocations": atomic_write_parquet( + rounded, + paths.data_dir / "outcome_free/rounded_allocations.parquet", + ), + "rounded_solve_records": atomic_write_parquet( + rounded_records, + paths.data_dir / "outcome_free/rounded_solve_records.parquet", + ), + "granularity_audit": atomic_write_parquet( + audit, + paths.data_dir / "outcome_free/granularity_audit.parquet", + ), + } + freeze_payload = { + "schema_version": str(config["schema_version"]), + "status": "allocation_granularity_frozen_before_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "historical_archive_previously_inspected": True, + "outcome_based_selection": False, + "outcome_columns_passed_to_rounding": [], + "rounding_rule": str(sensitivity["rounding_rule"]), + "lot_size_usd": float(sensitivity["lot_size_usd"]), + "committed_budget_usd": budget, + "results": { + "portfolios": int(len(audit)), + "source_rows": int(len(allocations)), + "rounded_positive_rows": int(len(rounded)), + "changed_rows": int(audit["changed_positions"].sum()), + "cash_residual_min": float(audit["cash_residual"].min()), + "cash_residual_mean": float(audit["cash_residual"].mean()), + "cash_residual_max": float(audit["cash_residual"].max()), + "cash_share_max": float(audit["cash_share"].max()), + }, + "parent": { + "allocations": relative_artifact_descriptor(allocation_path, repo_root=root), + "solve_records": relative_artifact_descriptor(record_path, repo_root=root), + }, + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_paths.items() + }, + "implementation": implementation_provenance( + config_path=resolved_config, + repo_root=root, + relative_paths=[ + Path("src/ijds_audit/allocation_granularity.py"), + Path("src/evaluation/policy_contrast_bounds.py"), + Path("scripts/experiments/run_ijds_allocation_granularity_sensitivity.py"), + Path( + "docs/research/ijds_allocation_granularity_sensitivity_protocol_2026-07-16.md" + ), + ], + ), + "environment": environment_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze_payload) + + +def evaluate(config_path: Path, *, repo_root: Path) -> Path: + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + data_dir, model_dir = _run_dirs(config, root) + freeze_path = model_dir / "protocol_freeze.json" + if not freeze_path.is_file(): + raise FileNotFoundError("Run the allocation-granularity freeze phase first.") + summary_path = model_dir / "allocation_granularity_summary.json" + evaluation_dir = data_dir / "evaluation" + if summary_path.exists() or evaluation_dir.exists(): + raise FileExistsError("Allocation-granularity evaluation outputs are immutable.") + freeze_payload = json.loads(freeze_path.read_text(encoding="utf-8")) + for field, expected in { + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "status": "allocation_granularity_frozen_before_outcome_join", + }.items(): + if freeze_payload.get(field) != expected: + raise RuntimeError(f"Allocation-granularity freeze mismatch for {field}.") + if freeze_payload.get("outcome_columns_passed_to_rounding") != []: + raise RuntimeError("Allocation-granularity freeze reports outcome leakage.") + rounded_path = _verified_descriptor( + freeze_payload["artifacts"]["rounded_allocations"], + repo_root=root, + ) + continuous_path = _verified_descriptor(config["parent"]["allocations"], repo_root=root) + continuous = pd.read_parquet(continuous_path) + rounded = pd.read_parquet(rounded_path) + + v4_config_path = resolve_repo_input(config["parent"]["v4_config"], repo_root=root) + parent = load_v4_config(v4_config_path) + raw_path = resolve_repo_input(parent["source"]["raw_path"], repo_root=root) + universe = load_outcome_universe(parent, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, parent) + sensitivity = config["allocation_granularity"] + contrasts = granularity_contrast_bounds( + continuous, + rounded, + outcomes, + committed_budget=float(sensitivity["committed_budget_usd"]), + periods=tuple(str(value) for value in sensitivity["periods"]), + lgd=float(sensitivity["lgd"]), + ) + if len(contrasts) != int(sensitivity["expected_tracks"]): + raise RuntimeError("Allocation-granularity track census changed.") + numeric_bounds = [ + "realized_payoff_rate_difference_lower", + "realized_payoff_rate_difference_upper", + "weighted_default_difference_lower", + "weighted_default_difference_upper", + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ] + if not bool(np.isfinite(contrasts[numeric_bounds].to_numpy(dtype=float)).all()): + raise RuntimeError("Allocation-granularity contrasts contain non-finite bounds.") + contrast_path = atomic_write_parquet( + contrasts, + evaluation_dir / "granularity_contrasts.parquet", + ) + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_allocation_granularity_sensitivity", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "outcome_based_selection": False, + "scope": ( + "Deterministic floor to USD 25 lots with residual cash; this diagnoses the " + "continuous relaxation and is not a new optimized integer policy." + ), + "results": { + "tracks": int(len(contrasts)), + "cash_share_max": float(contrasts["cash_share"].max()), + "payoff_rate_perturbation_abs_max": float( + contrasts[ + [ + "realized_payoff_rate_difference_lower", + "realized_payoff_rate_difference_upper", + ] + ] + .abs() + .to_numpy(dtype=float) + .max() + ), + "default_rate_perturbation_abs_max": float( + contrasts[ + ["weighted_default_difference_lower", "weighted_default_difference_upper"] + ] + .abs() + .to_numpy(dtype=float) + .max() + ), + "miscoverage_rate_perturbation_abs_max": float( + contrasts[ + [ + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ] + ] + .abs() + .to_numpy(dtype=float) + .max() + ), + }, + "freeze": relative_artifact_descriptor(freeze_path, repo_root=root), + "artifacts": { + "granularity_contrasts": relative_artifact_descriptor(contrast_path, repo_root=root) + }, + "environment": environment_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(summary_path, summary) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + path = ( + freeze(args.config, repo_root=args.repo_root) + if args.phase == "freeze" + else evaluate(args.config, repo_root=args.repo_root) + ) + print(path) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_binary_geometry_frontier_v4.py b/scripts/experiments/run_ijds_binary_geometry_frontier_v4.py new file mode 100644 index 0000000..d88b6a5 --- /dev/null +++ b/scripts/experiments/run_ijds_binary_geometry_frontier_v4.py @@ -0,0 +1,87 @@ +"""Run one phase of the tagged IJDS V4 binary-geometry audit.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.protocol import evaluate_frozen, freeze_outcome_free +from src.utils.isolated_experiment import ( + git_provenance, + resolve_isolated_run_dir, + resolve_repo_input, +) +from src.utils.pipeline_runtime import atomic_write_json + +ROOT = Path(__file__).resolve().parents[2] +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("freeze", "evaluate")) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def _write_failure_receipt( + *, + config_path: Path, + repo_root: Path, + phase: str, + error: Exception, +) -> Path: + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_v4_config(resolved_config) + model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(config["run_tag"]), + ) + model_dir.mkdir(parents=True, exist_ok=True) + path = model_dir / f"{phase}_failure_receipt.json" + if path.exists(): + raise FileExistsError(f"Failure receipt already exists: {path}") from error + details = getattr(error, "protocol_details", {}) + if not isinstance(details, dict): + details = {"unstructured_details": str(details)} + payload: dict[str, Any] = { + "schema_version": str(config["schema_version"]), + "status": "protocol_phase_failed_without_result_adaptation", + "phase": str(phase), + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "error_type": type(error).__name__, + "error_message": str(error), + "protocol_details": details, + "git": git_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(path, payload) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + runner = freeze_outcome_free if args.phase == "freeze" else evaluate_frozen + try: + print(runner(config_path=args.config, repo_root=args.repo_root)) + except Exception as error: + receipt = _write_failure_receipt( + config_path=args.config, + repo_root=args.repo_root, + phase=str(args.phase), + error=error, + ) + print(receipt) + raise + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py deleted file mode 100644 index 21f86e7..0000000 --- a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py +++ /dev/null @@ -1,597 +0,0 @@ -"""Select a simple CRPTO policy without consulting OOT outcomes.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import pickle -import subprocess -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from loguru import logger - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.experiments.ijds_policy_support import ( # noqa: E402 - evaluate_candidate, - load_policy_panel, - solve_candidate, -) -from scripts.generate_conformal_intervals import ( # noqa: E402 - _build_probability_lookups, - _build_tuning_split, - _load_conformal_inputs, -) -from src.models.conformal_alpha_grid import ( # noqa: E402 - FrozenConformalRecipe, - compute_exact_alpha_intervals, -) -from src.optimization.policy_evaluation import PolicyAllocationResult # noqa: E402 -from src.optimization.policy_selection import ( # noqa: E402 - FORBIDDEN_POLICY_SELECTION_COLUMNS, - LinearPolicyCandidate, - build_linear_policy_grid, - endpoint_cap_stability, - select_policy_result_ex_ante, -) -from src.utils.script_helpers import ( # noqa: E402 - parse_percent_series, - resolve_repo_artifact_path, - write_json, -) - -DEFAULT_CONFIG = ( - ROOT / "configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml" -) -FORBIDDEN_SELECTOR_COLUMNS = FORBIDDEN_POLICY_SELECTION_COLUMNS -SELECTOR_INPUT_COLUMNS = ( - "id", - "loan_amnt", - "purpose", - "grade", - "term", - "verification_status", - "issue_d", - "_pd_point", - "_pd_low", - "_pd_high", - "_loan_amount", - "_int_rate", -) - - -def _load_config(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise TypeError("Experiment config must contain a mapping.") - return payload - - -def _load_pickle_mapping(path: Path) -> dict[str, Any]: - with path.open("rb") as handle: - payload = pickle.load(handle) - if not isinstance(payload, dict): - raise TypeError("Frozen conformal results must contain a mapping.") - return payload - - -def _experiment_paths(run_tag: str) -> tuple[Path, Path]: - data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" - model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" - data_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - return data_dir, model_dir - - -def _endpoint_budget_cap(config: dict[str, Any]) -> float: - design = config["design"] - if "endpoint_budget_cap" in design: - return float(design["endpoint_budget_cap"]) - if "markov_threshold_cap" in design: - return float(design["markov_threshold_cap"]) - float(np.sqrt(design["alpha"])) - raise KeyError("Experiment design requires endpoint_budget_cap.") - - -def _selector_input_frame(frame: pd.DataFrame) -> pd.DataFrame: - """Expose only columns required to solve the outcome-free policy grid.""" - required = {"loan_amnt", "purpose", "_pd_point", "_pd_low", "_pd_high", "_int_rate"} - missing = sorted(required.difference(frame.columns)) - if missing: - raise KeyError(f"Calibration selector input is missing columns: {missing}") - return frame.loc[:, [column for column in SELECTOR_INPUT_COLUMNS if column in frame]].copy() - - -def _load_calibration_selection_panel( - config: dict[str, Any], -) -> tuple[pd.DataFrame, pd.Series, FrozenConformalRecipe, dict[str, Any]]: - source = config["source"] - design = config["design"] - os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) - results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) - payload = _load_pickle_mapping(results_path) - recipe = FrozenConformalRecipe.from_results_payload(payload) - inputs = _load_conformal_inputs( - calibration_fraction=recipe.calibration_fraction, - calibrator_override_path=(str(payload.get("calibrator_override_path", "")).strip() or None), - ) - split = _build_tuning_split( - cal_df=inputs.cal_df, - test_df=inputs.test_df, - X_cal=inputs.X_cal, - y_cal=inputs.y_cal, - group_cal_base=inputs.group_cal_base, - y_prob_cal_raw=inputs.y_prob_cal_raw, - tuning_holdout_ratio=recipe.tuning_holdout_ratio, - tuning_random_state=recipe.tuning_random_state, - ) - probability_fit, probability_tune, _probability_test = _build_probability_lookups( - inputs, - split, - ) - intervals = compute_exact_alpha_intervals( - recipe=recipe, - target_alpha=float(design["alpha"]), - y_cal=split.y_cal_fit, - interval_probability_cal=probability_fit["calibrated"], - interval_probability_eval=probability_tune["calibrated"], - partition_probability_cal=probability_fit[recipe.partition_probability_source], - partition_probability_eval=probability_tune[recipe.partition_probability_source], - base_groups_cal=split.group_cal_fit_base, - base_groups_eval=split.group_tune_base, - issue_dates_eval=split.issue_tune, - ) - panel = inputs.cal_df.iloc[split.idx_cal_tune].reset_index(drop=True).copy() - panel["_pd_point"] = intervals.point - panel["_pd_low"] = intervals.low - panel["_pd_high"] = intervals.high - panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) - panel["_int_rate"] = parse_percent_series(panel["int_rate"]) - outcomes = pd.Series(split.y_tune.to_numpy(dtype=float), name="_outcome") - months = pd.to_datetime(panel["issue_d"], errors="raise").dt.to_period("M").astype(str) - panel = _selector_input_frame(panel) - panel["_month"] = months.to_numpy(dtype=str) - metadata = { - "conformal_results_path": str(results_path.relative_to(ROOT)), - "calibration_fit_rows": int(len(split.idx_cal_fit)), - "calibration_holdout_rows": int(len(split.idx_cal_tune)), - "calibration_holdout_start": str(pd.to_datetime(split.issue_tune).min().date()), - "calibration_holdout_end": str(pd.to_datetime(split.issue_tune).max().date()), - "target_alpha": intervals.target_alpha, - "used_alpha": intervals.used_alpha, - "partition": recipe.partition, - } - return panel, outcomes, recipe, metadata - - -def _measure_ex_ante_solution( - frame: pd.DataFrame, - candidate: LinearPolicyCandidate, - result: PolicyAllocationResult, - *, - alpha: float, -) -> dict[str, Any]: - exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) - total_allocated = float(exposure.sum()) - if total_allocated <= 0.0: - raise RuntimeError(f"Policy {candidate.candidate_id} allocated no calibration capital.") - weights = exposure / total_allocated - point = frame["_pd_point"].to_numpy(dtype=float) - high = frame["_pd_high"].to_numpy(dtype=float) - weighted_point = float(np.sum(weights * point)) - weighted_effective = float(np.sum(weights * result.effective_pd)) - endpoint_budget = float(np.sum(weights * high)) - return { - **candidate.to_record(), - "solver_status": str(result.solution.get("solver_status", "unknown")), - "objective_risk_mode": result.objective_risk_mode, - "expected_objective": float(result.solution["objective_value"]), - "n_panel": int(len(frame)), - "n_funded": int(np.count_nonzero(result.allocation > 0.01)), - "total_allocated": total_allocated, - "weighted_pd_point": weighted_point, - "weighted_pd_effective": weighted_effective, - "gamma_cp": float(np.sum(weights * (high - point))), - "gamma_internalized": float(np.sum(weights * (result.effective_pd - point))), - "gamma_residual": float(np.sum(weights * (high - result.effective_pd))), - "endpoint_budget": endpoint_budget, - "markov_loss_threshold": endpoint_budget + float(np.sqrt(alpha)), - "effective_pd_cap_slack": float(candidate.risk_tolerance - weighted_effective), - } - - -def _run_calibration_grid( - panel: pd.DataFrame, - candidates: list[LinearPolicyCandidate], - *, - config: dict[str, Any], - output_path: Path, -) -> pd.DataFrame: - selector_panel = _selector_input_frame(panel) - rows: list[dict[str, Any]] = [] - for candidate in candidates: - logger.info("Calibration selector evaluating {}", candidate.candidate_id) - result = solve_candidate(selector_panel, candidate, config=config) - rows.append( - _measure_ex_ante_solution( - selector_panel, - candidate, - result, - alpha=float(config["design"]["alpha"]), - ) - ) - output = pd.DataFrame(rows) - if FORBIDDEN_SELECTOR_COLUMNS.intersection(output.columns): - raise AssertionError("Calibration selector artifact contains outcome-derived columns.") - output.to_parquet(output_path, index=False) - return output - - -def _match_candidate( - candidates: list[LinearPolicyCandidate], - settings: dict[str, Any], -) -> LinearPolicyCandidate: - matches = [ - candidate - for candidate in candidates - if np.isclose(candidate.risk_tolerance, float(settings["risk_tolerance"])) - and np.isclose(candidate.gamma, float(settings["gamma"])) - and np.isclose(candidate.uncertainty_aversion, float(settings["uncertainty_aversion"])) - ] - if len(matches) != 1: - raise ValueError(f"Policy settings must match exactly one candidate, got {len(matches)}.") - return matches[0] - - -def _comparison_policies( - selected: LinearPolicyCandidate, - incumbent: LinearPolicyCandidate, -) -> tuple[tuple[str, LinearPolicyCandidate, bool], ...]: - point = LinearPolicyCandidate( - candidate_id="point-pd", - risk_tolerance=selected.risk_tolerance, - gamma=0.0, - uncertainty_aversion=0.0, - policy_mode="point_estimate", - ) - return ( - ("calibration_selected", selected, True), - ("incumbent_linear", incumbent, True), - ("point_pd_matched_tau", point, False), - ) - - -def _evaluate_fixed_policies( - panel: pd.DataFrame, - selected: LinearPolicyCandidate, - incumbent: LinearPolicyCandidate, - *, - config: dict[str, Any], -) -> tuple[pd.DataFrame, pd.DataFrame]: - periods = ["full_oot", *list(config["design"]["period_order"])] - rows: list[dict[str, Any]] = [] - allocation_frames: list[pd.DataFrame] = [] - for period in periods: - frame = ( - panel.reset_index(drop=True) - if period == "full_oot" - else panel.loc[panel["_period"].astype(str).eq(period)].reset_index(drop=True) - ) - for role, candidate, robust in _comparison_policies(selected, incumbent): - record, result = evaluate_candidate( - frame, - candidate, - config=config, - robust=robust, - period=period, - ) - rows.append({"role": role, **record}) - if period == "full_oot": - allocation_frames.append( - _funded_allocation_frame( - frame, - result, - role=role, - lgd=float(config["design"]["lgd"]), - ) - ) - return pd.DataFrame(rows), pd.concat(allocation_frames, ignore_index=True) - - -def _evaluate_calibration_audit( - panel: pd.DataFrame, - selected: LinearPolicyCandidate, - incumbent: LinearPolicyCandidate, - *, - config: dict[str, Any], - period: str, -) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - frame = panel.reset_index(drop=True) - for role, candidate, robust in _comparison_policies(selected, incumbent): - record, _result = evaluate_candidate( - frame, - candidate, - config=config, - robust=robust, - period=period, - ) - rows.append({"role": role, **record}) - return pd.DataFrame(rows) - - -def _funded_allocation_frame( - frame: pd.DataFrame, - result: PolicyAllocationResult, - *, - role: str, - lgd: float, -) -> pd.DataFrame: - allocation = result.allocation - funded = allocation > 0.01 - selected = frame.loc[funded].reset_index(drop=True).copy() - selected_allocation = allocation[funded] - exposure = selected_allocation * selected["_loan_amount"].to_numpy(dtype=float) - total_exposure = float(exposure.sum()) - outcome = selected["_outcome"].to_numpy(dtype=float) - rates = selected["_int_rate"].to_numpy(dtype=float) - point = selected["_pd_point"].to_numpy(dtype=float) - high = selected["_pd_high"].to_numpy(dtype=float) - effective = result.effective_pd[funded] - if "sub_grade" in selected.columns: - loan_grade = selected["sub_grade"].astype(str).str[:1] - elif "int_rate_bucket__grade" in selected.columns: - loan_grade = selected["int_rate_bucket__grade"].astype(str).str.rsplit("__").str[-1] - else: - loan_grade = selected["grade"].astype(str) - output = pd.DataFrame( - { - "role": role, - "id": selected["id"].astype(str), - "issue_d": selected["issue_d"], - "grade": loan_grade, - "conformal_group": selected["grade"].astype(str), - "loan_amnt": selected["_loan_amount"].to_numpy(dtype=float), - "int_rate": rates, - "outcome": outcome, - "pd_point": point, - "pd_low": selected["_pd_low"].to_numpy(dtype=float), - "pd_high": high, - "pd_effective": effective, - "allocation": selected_allocation, - "funded_exposure": exposure, - "funded_weight": exposure / total_exposure, - "miscoverage": (outcome > high).astype(int), - "expected_return_contribution": exposure * (rates - point * float(lgd)), - "realized_return_contribution": np.where( - outcome.astype(int) == 1, - -float(lgd) * exposure, - rates * exposure, - ), - } - ) - return output - - -def _contrast_payload(evaluation: pd.DataFrame) -> dict[str, Any]: - output: dict[str, Any] = {} - for period in ("full_oot", "2020+"): - period_rows = evaluation.loc[evaluation["period"].eq(period)].set_index("role") - selected = period_rows.loc["calibration_selected"] - point = period_rows.loc["point_pd_matched_tau"] - incumbent = period_rows.loc["incumbent_linear"] - output[period] = { - "selected_realized_return": float(selected["realized_return"]), - "selected_weighted_outcome": float(selected["weighted_outcome"]), - "selected_markov_threshold": float(selected["markov_loss_threshold"]), - "return_cost_vs_point": float(point["realized_return"] - selected["realized_return"]), - "default_delta_vs_point": float( - selected["weighted_outcome"] - point["weighted_outcome"] - ), - "threshold_delta_vs_point": float( - selected["markov_loss_threshold"] - point["markov_loss_threshold"] - ), - "return_delta_vs_incumbent": float( - selected["realized_return"] - incumbent["realized_return"] - ), - "default_delta_vs_incumbent": float( - selected["weighted_outcome"] - incumbent["weighted_outcome"] - ), - } - return output - - -def _git_commit() -> str: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=ROOT, - capture_output=True, - text=True, - check=False, - ) - return result.stdout.strip() if result.returncode == 0 else "unknown" - - -def run(config_path: Path) -> dict[str, Any]: - config = _load_config(config_path) - run_tag = str(config["run_tag"]) - data_dir, model_dir = _experiment_paths(run_tag) - calibration_holdout, calibration_outcomes, recipe, calibration_metadata = ( - _load_calibration_selection_panel(config) - ) - design = config["design"] - selection_period = str(design.get("selection_period", "")).strip() - audit_period = str(design.get("audit_period", "")).strip() - selection_mask = ( - calibration_holdout["_month"].eq(selection_period) - if selection_period - else pd.Series(True, index=calibration_holdout.index) - ) - selection_panel = calibration_holdout.loc[selection_mask].reset_index(drop=True).copy() - if selection_panel.empty: - raise ValueError(f"Calibration selection period has no rows: {selection_period!r}") - audit_mask = ( - calibration_holdout["_month"].eq(audit_period) - if audit_period - else pd.Series(False, index=calibration_holdout.index) - ) - audit_panel = calibration_holdout.loc[audit_mask].reset_index(drop=True).copy() - if audit_period and audit_panel.empty: - raise ValueError(f"Calibration audit period has no rows: {audit_period!r}") - audit_outcomes = calibration_outcomes.loc[audit_mask.to_numpy()].reset_index(drop=True) - calibration_metadata.update( - { - "selection_period": selection_period or "full_calibration_holdout", - "selection_rows": int(len(selection_panel)), - "audit_period": audit_period or None, - "audit_rows": int(len(audit_panel)), - "outcomes_isolated_until_post_selection_audit": True, - } - ) - grid_config = config["policy_grid"] - candidates = build_linear_policy_grid( - risk_tolerances=[float(value) for value in grid_config["risk_tolerances"]], - gammas=[float(value) for value in grid_config["gammas"]], - uncertainty_aversions=[float(value) for value in grid_config["uncertainty_aversions"]], - ) - selection_results = _run_calibration_grid( - selection_panel, - candidates, - config=config, - output_path=data_dir / "calibration_policy_selection_grid.parquet", - ) - endpoint_cap = _endpoint_budget_cap(config) - selected_row, selection_audit = select_policy_result_ex_ante( - selection_results, - endpoint_budget_cap=endpoint_cap, - budget=float(design["budget"]), - min_budget_utilization=float(design["selection_min_budget_utilization"]), - ) - candidate_lookup = {candidate.candidate_id: candidate for candidate in candidates} - selected = candidate_lookup[str(selected_row["candidate_id"])] - incumbent = _match_candidate(candidates, config["incumbent_policy"]) - cap_stability = endpoint_cap_stability( - selection_results, - selected_candidate_id=selected.candidate_id, - endpoint_budget_cap=endpoint_cap, - budget=float(design["budget"]), - min_budget_utilization=float(design["selection_min_budget_utilization"]), - ) - - audit_grid_path: Path | None = None - audit_evaluation_path: Path | None = None - audit_selected_id: str | None = None - audit_evaluation = pd.DataFrame() - if not audit_panel.empty: - audit_grid_path = data_dir / "calibration_policy_audit_grid.parquet" - audit_results = _run_calibration_grid( - audit_panel, - candidates, - config=config, - output_path=audit_grid_path, - ) - audit_selected_row, _audit_selection = select_policy_result_ex_ante( - audit_results, - endpoint_budget_cap=endpoint_cap, - budget=float(design["budget"]), - min_budget_utilization=float(design["selection_min_budget_utilization"]), - ) - audit_selected_id = str(audit_selected_row["candidate_id"]) - audited_decisions = audit_panel.assign(_outcome=audit_outcomes.to_numpy(dtype=float)) - audit_evaluation = _evaluate_calibration_audit( - audited_decisions, - selected, - incumbent, - config=config, - period=audit_period, - ) - audit_evaluation_path = data_dir / "calibration_policy_holdout_audit.csv" - audit_evaluation.to_csv(audit_evaluation_path, index=False) - - oot_panel = load_policy_panel(config) - evaluation, allocations = _evaluate_fixed_policies( - oot_panel, - selected, - incumbent, - config=config, - ) - evaluation_path = data_dir / "calibration_selected_policy_oot_evaluation.csv" - allocation_path = data_dir / "calibration_selected_policy_full_oot_allocations.parquet" - evaluation.to_csv(evaluation_path, index=False) - allocations.to_parquet(allocation_path, index=False) - payload: dict[str, Any] = { - "schema_version": str(config["schema_version"]), - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": run_tag, - "source_commit": _git_commit(), - "config_path": str(config_path.relative_to(ROOT)), - "config_sha256": hashlib.sha256(config_path.read_bytes()).hexdigest(), - "design": config["design"], - "calibration_metadata": calibration_metadata, - "recipe": { - "partition": recipe.partition, - "partition_probability_source": recipe.partition_probability_source, - "reference_target_alpha": recipe.reference_target_alpha, - "reference_used_alpha": recipe.reference_used_alpha, - }, - "grid_size": int(len(candidates)), - "selector_input_columns": list(_selector_input_frame(selection_panel).columns), - "selector_columns": list(selection_results.columns), - "selector_forbidden_columns_present": sorted( - FORBIDDEN_SELECTOR_COLUMNS.intersection(selection_results.columns) - ), - "selection_audit": selection_audit, - "endpoint_cap_stability": cap_stability, - "calibration_audit": { - "period": audit_period or None, - "outcome_free_selected_candidate_id": audit_selected_id, - "same_policy_selected": bool(audit_selected_id == selected.candidate_id), - "grid_path": ( - None if audit_grid_path is None else str(audit_grid_path.relative_to(ROOT)) - ), - "evaluation_path": ( - None - if audit_evaluation_path is None - else str(audit_evaluation_path.relative_to(ROOT)) - ), - "policy_evaluations": audit_evaluation.to_dict(orient="records"), - "claim_boundary": ( - "Independent post-selection decision audit; not a selected-set coverage theorem." - ), - }, - "selected_policy": selected.to_record(), - "selected_calibration_metrics": selected_row.to_dict(), - "incumbent_policy": incumbent.to_record(), - "evaluation_path": str(evaluation_path.relative_to(ROOT)), - "allocation_path": str(allocation_path.relative_to(ROOT)), - "oot_alignment": dict(oot_panel.attrs), - "contrasts": _contrast_payload(evaluation), - "claim_boundary": str(config["claim_boundary"]), - } - summary_path = model_dir / "calibration_selected_policy_summary.json" - write_json(summary_path, payload) - logger.info("Selected calibration-only policy: {}", selected.candidate_id) - logger.info("Wrote calibration-only policy summary to {}", summary_path) - return payload - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) - args = parser.parse_args() - config_path = args.config if args.config.is_absolute() else ROOT / args.config - payload = run(config_path.resolve()) - print(json.dumps(payload["contrasts"], indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/experiments/run_ijds_credit_risk_controls.py b/scripts/experiments/run_ijds_credit_risk_controls.py new file mode 100644 index 0000000..9c4169a --- /dev/null +++ b/scripts/experiments/run_ijds_credit_risk_controls.py @@ -0,0 +1,32 @@ +"""Run one phase of protocol-locked IJDS credit-risk learner controls.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from pathlib import Path + +from src.ijds_audit.credit_control_protocol import ( + evaluate_credit_controls, + freeze_credit_controls, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("freeze", "evaluate")) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + runner = freeze_credit_controls if args.phase == "freeze" else evaluate_credit_controls + print(runner(config_path=args.config, repo_root=args.repo_root)) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_endpoint_availability_sensitivity.py b/scripts/experiments/run_ijds_endpoint_availability_sensitivity.py new file mode 100644 index 0000000..87b71ba --- /dev/null +++ b/scripts/experiments/run_ijds_endpoint_availability_sensitivity.py @@ -0,0 +1,486 @@ +"""Evaluate frozen IJDS evidence under every declared endpoint-availability lag.""" + +from __future__ import annotations + +import argparse +import json +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import pandas as pd +import yaml +from loguru import logger + +from src.ijds_audit.allocations import policy_family +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.endpoint_sensitivity import ( + direction_census, + endpoint_census, + exact_support_census, + rebuild_archive_outcomes, + summarize_coverage_sensitivity, +) +from src.ijds_audit.evaluation import ( + comparator_envelopes, + evaluate_frozen_portfolios, + indexed_portfolio_contrasts, + temporal_coverage_audit, +) +from src.ijds_audit.protocol import ( + load_outcome_universe, + load_recipes, + verified_freeze_artifact_paths, +) +from src.ijds_challengers.evaluation import ( + build_endpoint_contrasts, + build_metric_directions, + validate_complete_evaluation, + validate_outcome_alignment, + verify_frontier_freeze, +) +from src.ijds_challengers.evaluation_config import load_v2_config +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, +) +from src.utils.isolated_experiment import ( + dataframe_schema, + environment_provenance, + git_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_repo_input, + sha256_file, +) +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet, utc_now_iso + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = ROOT / "configs/experiments/ijds_endpoint_availability_sensitivity_2026-07-14.yaml" +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +_INDEX_ALLOCATION_COLUMNS = [ + "id", + "role", + "policy_label", + "exposure", + "expected_payoff_contribution", + "comparator_rule", + "frontier_cap", +] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the endpoint-sensitivity CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + return parser.parse_args(argv) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Endpoint-sensitivity config must be a mapping.") + if payload.get("protocol_status") != "locked_retrospective_endpoint_assumption_sensitivity": + raise ValueError("Endpoint-sensitivity protocol is not locked.") + endpoint = payload.get("endpoint", {}) + if [int(value) for value in endpoint.get("charged_off_lag_months", [])] != [0, 3, 6, 8, 12]: + raise ValueError("Endpoint lag grid must remain 0/3/6/8/12 months.") + if int(endpoint.get("fully_paid_lag_months", -1)) != 0: + raise ValueError("Fully Paid must retain the declared zero administrative lag.") + boundary = payload.get("claim_boundary", {}) + required_false = { + "preregistered", + "confirmatory", + "prospective", + "outcome_based_selection", + "allocation_refit", + "policy_selection", + "model_selection", + "endpoint_selection", + } + if any(boundary.get(field) is not False for field in required_false): + raise ValueError("Endpoint-sensitivity claim boundary changed.") + return payload + + +def _verified_freeze( + descriptor: Mapping[str, Any], + *, + repo_root: Path, + expected_status: str, +) -> tuple[dict[str, Any], dict[str, Path]]: + path = resolve_repo_input(str(descriptor["path"]), repo_root=repo_root) + if relative_artifact_descriptor(path, repo_root=repo_root) != dict(descriptor): + raise RuntimeError(f"Freeze descriptor changed: {descriptor['path']}.") + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("status") != expected_status: + raise RuntimeError(f"Unexpected freeze status for {descriptor['path']}.") + artifacts = verified_freeze_artifact_paths(payload, repo_root=repo_root) + return payload, artifacts + + +def _primary_loan_facts( + allocations: pd.DataFrame, + outcomes: pd.DataFrame, +) -> pd.DataFrame: + required_allocations = {"id", "role", "period", "contractual_rate"} + missing_allocations = sorted(required_allocations - set(allocations.columns)) + if missing_allocations: + raise KeyError(f"Allocations are missing primary fact columns: {missing_allocations}.") + required_outcomes = { + "id", + "role", + "period", + "snapshot_default", + "snapshot_resolution", + } + missing_outcomes = sorted(required_outcomes - set(outcomes.columns)) + if missing_outcomes: + raise KeyError(f"Outcomes are missing primary fact columns: {missing_outcomes}.") + if bool(outcomes["id"].isna().any()) or bool(outcomes["id"].duplicated().any()): + raise RuntimeError("Endpoint outcomes must contain one nonmissing row per ID.") + + primary = allocations.loc[ + allocations["role"].eq("primary_oot"), + ["id", "role", "period", "contractual_rate"], + ].copy() + if primary.empty or bool(primary["id"].isna().any()): + raise RuntimeError("Primary allocation facts are empty or contain missing IDs.") + conflicts = primary.groupby("id", observed=True, dropna=False)[ + ["role", "period", "contractual_rate"] + ].nunique(dropna=False) + if bool(conflicts.gt(1).any(axis=None)): + raise RuntimeError("Primary allocations contain conflicting per-loan facts.") + primary = primary.drop_duplicates() + if bool(primary["id"].duplicated().any()): + raise RuntimeError("Primary allocation facts do not reduce to one row per ID.") + + endpoint = outcomes.loc[ + outcomes["id"].isin(primary["id"]), + ["id", "role", "period", "snapshot_default", "snapshot_resolution"], + ].rename(columns={"role": "outcome_role", "period": "outcome_period"}) + joined = primary.merge( + endpoint, + on="id", + how="left", + validate="one_to_one", + indicator="outcome_join", + sort=False, + ) + missing = int(joined["outcome_join"].eq("left_only").sum()) + if missing: + raise RuntimeError(f"Exact-frontier funded outcome ID mismatch: missing={missing}.") + role_mismatch = joined["role"].astype("string").ne(joined["outcome_role"].astype("string")) + period_mismatch = ( + joined["period"].astype("string").ne(joined["outcome_period"].astype("string")) + ) + if bool(role_mismatch.fillna(True).any()) or bool(period_mismatch.fillna(True).any()): + raise RuntimeError("Exact-frontier outcome role or period disagrees with allocations.") + if bool(joined["snapshot_resolution"].isna().any()): + raise RuntimeError("Exact-frontier outcome resolution is incomplete.") + return joined[["id", "contractual_rate", "snapshot_default"]] + + +def _window_loan_facts( + base_facts: pd.DataFrame, + scores: pd.DataFrame, + recipe: BinaryOutcomeConformalRecipe, +) -> pd.DataFrame: + required_scores = {"id", "design_split", "pd_catboost_platt"} + missing = sorted(required_scores - set(scores.columns)) + if missing: + raise KeyError(f"Scores are missing endpoint columns: {missing}.") + primary = scores.loc[ + scores["design_split"].eq("primary_oot"), ["id", "pd_catboost_platt"] + ].copy() + if bool(primary["id"].isna().any()) or bool(primary["id"].duplicated().any()): + raise RuntimeError("Primary score census must contain one nonmissing row per ID.") + probability = primary["pd_catboost_platt"].to_numpy(dtype=float) + _, lower, upper = apply_binary_outcome_recipe(probability, recipe) + endpoints = pd.DataFrame( + { + "id": primary["id"].astype("string"), + "conformal_lower": lower, + "conformal_upper": upper, + } + ) + facts = base_facts.merge( + endpoints, + on="id", + how="left", + validate="one_to_one", + indicator="score_join", + sort=False, + ) + missing_scores = int(facts["score_join"].eq("left_only").sum()) + if missing_scores: + raise RuntimeError(f"Exact-frontier score ID mismatch: missing={missing_scores}.") + return facts[ + [ + "id", + "contractual_rate", + "conformal_lower", + "conformal_upper", + "snapshot_default", + ] + ] + + +def _exact_support( + *, + records: pd.DataFrame, + allocations: pd.DataFrame, + support: pd.DataFrame, + scores: pd.DataFrame, + recipes: Mapping[str, Any], + outcomes: pd.DataFrame, + config: Mapping[str, Any], +) -> pd.DataFrame: + frontier_mask = records["comparator_rule"].eq("point_cap_frontier") + named_records = records.loc[~frontier_mask] + named_allocations = allocations.loc[~allocations["comparator_rule"].eq("point_cap_frontier")] + shared_allocations = allocations.loc[allocations["comparator_rule"].eq("point_cap_frontier")] + _, named_joined = evaluate_frozen_portfolios( + named_records, named_allocations, outcomes, config=config + ) + base_facts = _primary_loan_facts(allocations, outcomes) + shared_index_allocations = shared_allocations[_INDEX_ALLOCATION_COLUMNS] + policy_ids = tuple(candidate.candidate_id for candidate in policy_family(config)) + frames: list[pd.DataFrame] = [] + for window_id, group_recipes in recipes["catboost_platt"].items(): + loan_facts = _window_loan_facts(base_facts, scores, group_recipes[5]) + named_window = named_joined.loc[ + named_joined["window_id"].eq(window_id), _INDEX_ALLOCATION_COLUMNS + ] + window_allocations = pd.concat( + [named_window, shared_index_allocations], + ignore_index=True, + copy=False, + ) + frames.append( + indexed_portfolio_contrasts( + window_allocations, + loan_facts=loan_facts, + window_id=str(window_id), + policy_ids=policy_ids, + lgd=float(config["payoff"]["lgd"]), + ) + ) + contrasts = pd.concat(frames, ignore_index=True) + frontier = config["comparators"]["exact_point_cap_frontier"] + return comparator_envelopes( + contrasts, + support, + broad_lower=float(frontier["start"]), + broad_upper=float(frontier["stop"]), + ) + + +def run(*, config_path: Path, repo_root: Path = ROOT) -> Path: + """Run the complete frozen endpoint sensitivity and write immutable artifacts.""" + started = time.perf_counter() + started_at = utc_now_iso() + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + parent = config["parent"] + raw_path = resolve_repo_input(str(parent["raw_path"]), repo_root=root) + if sha256_file(raw_path) != str(parent["raw_sha256"]): + raise RuntimeError("Raw archive hash changed before endpoint sensitivity.") + v4_config = load_v4_config(resolve_repo_input(str(parent["v4_config"]), repo_root=root)) + two_ruler_config = load_v2_config( + resolve_repo_input(str(parent["two_ruler_config"]), repo_root=root) + ) + _, v4_paths = _verified_freeze( + parent["v4_freeze"], + repo_root=root, + expected_status="outcome_free_allocations_frozen_before_archive_outcome_join", + ) + _, credit_paths = _verified_freeze( + parent["credit_freeze"], + repo_root=root, + expected_status="credit_control_scores_frozen_before_primary_oot_outcome_join", + ) + two_ruler = verify_frontier_freeze(two_ruler_config, repo_root=root) + universe = load_outcome_universe(v4_config, raw_path=raw_path) + v4_scores = pd.read_parquet(v4_paths["scores"]) + v4_recipes = load_recipes(v4_paths["recipes"]) + credit_scores = pd.read_parquet(credit_paths["scores"]) + credit_recipes = load_recipes(credit_paths["recipes"]) + credit_fit = pd.read_parquet(credit_paths["fit_audit"]) + two_records = pd.read_parquet(two_ruler.artifacts["solve_records"]) + two_allocations = pd.read_parquet(two_ruler.artifacts["allocations"]) + two_endpoints = pd.read_parquet(two_ruler.artifacts["endpoint_diagnostics"]) + v4_records = pd.read_parquet(v4_paths["solve_records"]) + v4_allocations = pd.read_parquet(v4_paths["allocations"]) + v4_support = pd.read_parquet(v4_paths["comparator_support"]) + + endpoint_frames: list[pd.DataFrame] = [] + coverage_frames: list[pd.DataFrame] = [] + contrast_frames: list[pd.DataFrame] = [] + direction_frames: list[pd.DataFrame] = [] + direction_census_frames: list[pd.DataFrame] = [] + envelope_frames: list[pd.DataFrame] = [] + envelope_census_frames: list[pd.DataFrame] = [] + lags = [int(value) for value in config["endpoint"]["charged_off_lag_months"]] + for lag in lags: + logger.info("Endpoint sensitivity lag {} months", lag) + outcomes = rebuild_archive_outcomes( + universe, + evaluation_cutoff=str(config["endpoint"]["evaluation_cutoff"]), + charged_off_lag_months=lag, + ) + endpoint_frames.append(endpoint_census(outcomes, lag_months=lag)) + coverage = temporal_coverage_audit( + credit_scores, + outcomes, + credit_recipes, + credit_fit, + roles=[str(value) for value in config["endpoint"]["roles"]], + taxonomy_group_counts=[int(config["endpoint"]["canonical_taxonomy_groups"])], + strata=[-1], + ).assign(charged_off_lag_months=lag) + coverage_frames.append(coverage) + outcome_audit = validate_outcome_alignment( + two_allocations, + outcomes, + config=two_ruler_config, + ) + evaluated, joined = evaluate_frozen_portfolios( + two_records, + two_allocations, + outcomes, + config=v4_config, + ) + window, monthly = build_endpoint_contrasts( + joined, + two_endpoints, + config=two_ruler_config, + lgd=float(v4_config["payoff"]["lgd"]), + ) + directions = build_metric_directions(window, config=two_ruler_config) + validate_complete_evaluation( + evaluated, + joined, + window, + monthly, + directions, + config=two_ruler_config, + ) + if outcome_audit.empty: + raise RuntimeError("Endpoint sensitivity produced no outcome audit rows.") + contrast_frames.append(window.assign(charged_off_lag_months=lag)) + direction_frames.append(directions.assign(charged_off_lag_months=lag)) + direction_census_frames.append(direction_census(directions, lag_months=lag)) + envelopes = _exact_support( + records=v4_records, + allocations=v4_allocations, + support=v4_support, + scores=v4_scores, + recipes=v4_recipes, + outcomes=outcomes, + config=v4_config, + ).assign(charged_off_lag_months=lag) + envelope_frames.append(envelopes) + envelope_census_frames.append(exact_support_census(envelopes, lag_months=lag)) + + endpoint_table = pd.concat(endpoint_frames, ignore_index=True) + coverage_table = pd.concat(coverage_frames, ignore_index=True) + coverage_summary = summarize_coverage_sensitivity(coverage_table) + contrast_table = pd.concat(contrast_frames, ignore_index=True) + direction_table = pd.concat(direction_frames, ignore_index=True) + direction_counts = pd.concat(direction_census_frames, ignore_index=True) + envelope_table = pd.concat(envelope_frames, ignore_index=True) + envelope_counts = pd.concat(envelope_census_frames, ignore_index=True) + artifacts = { + "endpoint_census": atomic_write_parquet( + endpoint_table, paths.data_dir / "evaluation/endpoint_census.parquet" + ), + "coverage_cells": atomic_write_parquet( + coverage_table, paths.data_dir / "evaluation/coverage_cells.parquet" + ), + "coverage_summary": atomic_write_parquet( + coverage_summary, paths.data_dir / "evaluation/coverage_summary.parquet" + ), + "two_ruler_window_contrasts": atomic_write_parquet( + contrast_table, paths.data_dir / "evaluation/two_ruler_window_contrasts.parquet" + ), + "two_ruler_directions": atomic_write_parquet( + direction_table, paths.data_dir / "evaluation/two_ruler_directions.parquet" + ), + "two_ruler_direction_census": atomic_write_parquet( + direction_counts, paths.data_dir / "evaluation/two_ruler_direction_census.parquet" + ), + "exact_support_envelopes": atomic_write_parquet( + envelope_table, paths.data_dir / "evaluation/exact_support_envelopes.parquet" + ), + "exact_support_census": atomic_write_parquet( + envelope_counts, paths.data_dir / "evaluation/exact_support_census.parquet" + ), + } + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_retrospective_endpoint_availability_sensitivity", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "elapsed_seconds": float(time.perf_counter() - started), + "claim_boundary": dict(config["claim_boundary"]), + "lags": lags, + "coverage_cells": int(len(coverage_table)), + "two_ruler_direction_cells": int(len(direction_table)), + "exact_support_envelopes": int(len(envelope_table)), + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifacts.items() + }, + "schemas": { + name: dataframe_schema(pd.read_parquet(path)) for name, path in artifacts.items() + }, + "implementation": implementation_provenance( + config_path=resolved_config, + repo_root=root, + relative_paths=( + Path("src/ijds_audit/endpoint_sensitivity.py"), + Path("scripts/experiments/run_ijds_endpoint_availability_sensitivity.py"), + Path("docs/research/ijds_endpoint_availability_sensitivity_protocol_2026-07-14.md"), + ), + ), + "environment": environment_provenance(root), + "git": git_provenance(root), + "selection": { + "lag": None, + "learner": None, + "window": None, + "ruler": None, + "coordinate": None, + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + summary_path = atomic_write_json(paths.model_dir / "endpoint_sensitivity_summary.json", summary) + logger.info("Wrote endpoint sensitivity {}", summary_path) + return summary_path + + +def main(argv: Sequence[str] | None = None) -> None: + """Run the endpoint sensitivity.""" + args = parse_args(argv) + run(config_path=args.config, repo_root=ROOT) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py b/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py deleted file mode 100644 index 5b71902..0000000 --- a/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Recompute the IJDS alpha grid from the frozen conformal recipe.""" - -from __future__ import annotations - -import argparse -import hashlib -import os -import pickle -import sys -from dataclasses import asdict -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from loguru import logger - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.generate_conformal_intervals import ( # noqa: E402 - _build_probability_lookups, - _build_tuning_split, - _load_conformal_inputs, -) -from src.models.conformal import conditional_coverage_by_group # noqa: E402 -from src.models.conformal_alpha_grid import ( # noqa: E402 - ExactAlphaIntervals, - FrozenConformalRecipe, - alpha_interval_columns, - compute_exact_alpha_intervals, -) -from src.utils.script_helpers import resolve_repo_artifact_path, write_json # noqa: E402 - -DEFAULT_CONFIG = ROOT / "configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml" - - -def _utc_now() -> str: - return datetime.now(tz=UTC).isoformat() - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for block in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def _load_config(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise TypeError("Experiment config must contain a mapping.") - return payload - - -def _experiment_paths(run_tag: str) -> tuple[Path, Path]: - data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "conformal" - model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "conformal" - data_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - return data_dir, model_dir - - -def _load_results_payload(path: Path) -> dict[str, Any]: - with path.open("rb") as handle: - payload = pickle.load(handle) - if not isinstance(payload, dict): - raise TypeError("Frozen conformal results must contain a mapping.") - return payload - - -def _coverage_summary( - result: ExactAlphaIntervals, - *, - y_true: np.ndarray, - grades: pd.Series, -) -> dict[str, Any]: - covered = (y_true >= result.low) & (y_true <= result.high) - partition_metrics = conditional_coverage_by_group( - y_true, - np.column_stack([result.low, result.high]), - result.partition_labels, - ) - grade_metrics = conditional_coverage_by_group( - y_true, - np.column_stack([result.low, result.high]), - grades, - ) - return { - "target_alpha": result.target_alpha, - "used_alpha": result.used_alpha, - "target_coverage": 1.0 - result.target_alpha, - "empirical_coverage": float(covered.mean()), - "coverage_gap": float(covered.mean() - (1.0 - result.target_alpha)), - "avg_width": float(np.mean(result.high - result.low)), - "median_width": float(np.median(result.high - result.low)), - "min_partition_coverage": float(partition_metrics["coverage"].min()), - "min_grade_coverage": float(grade_metrics["coverage"].min()), - "high_endpoint_mean": float(result.high.mean()), - "high_endpoint_min": float(result.high.min()), - "high_endpoint_p01": float(np.quantile(result.high, 0.01)), - "high_endpoint_p10": float(np.quantile(result.high, 0.10)), - "high_endpoint_at_one_rate": float(np.mean(result.high >= 1.0 - 1e-12)), - "partition_count": int(result.partition_labels.nunique()), - "group_quantiles": { - str(key): float(value) - for key, value in result.diagnostics.get("group_quantiles", {}).items() - }, - } - - -def _base_grid_frame(source_intervals: pd.DataFrame) -> pd.DataFrame: - preferred = [ - "_row_number", - "id", - "y_true", - "grade", - "loan_amnt", - "temporal_segment", - ] - columns = [column for column in preferred if column in source_intervals.columns] - return source_intervals.loc[:, columns].copy() - - -def _add_alpha_result(frame: pd.DataFrame, result: ExactAlphaIntervals) -> None: - low_column, high_column = alpha_interval_columns(result.target_alpha) - frame[low_column] = result.low - frame[high_column] = result.high - - -def _replay_differences( - result: ExactAlphaIntervals, - source_intervals: pd.DataFrame, -) -> dict[str, float]: - return { - "point_max_abs": float( - np.max(np.abs(result.point - source_intervals["y_pred"].to_numpy(dtype=float))) - ), - "low_max_abs": float( - np.max(np.abs(result.low - source_intervals["pd_low_90"].to_numpy(dtype=float))) - ), - "high_max_abs": float( - np.max(np.abs(result.high - source_intervals["pd_high_90"].to_numpy(dtype=float))) - ), - } - - -def run(config_path: Path) -> dict[str, Any]: - config = _load_config(config_path) - source = config["source"] - design = config["design"] - run_tag = str(config["run_tag"]) - results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) - intervals_path = resolve_repo_artifact_path(source["conformal_intervals_path"], root=ROOT) - os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) - - results_payload = _load_results_payload(results_path) - recipe = FrozenConformalRecipe.from_results_payload(results_payload) - inputs = _load_conformal_inputs( - calibration_fraction=recipe.calibration_fraction, - calibrator_override_path=( - str(results_payload.get("calibrator_override_path", "")).strip() or None - ), - ) - split = _build_tuning_split( - cal_df=inputs.cal_df, - test_df=inputs.test_df, - X_cal=inputs.X_cal, - y_cal=inputs.y_cal, - group_cal_base=inputs.group_cal_base, - y_prob_cal_raw=inputs.y_prob_cal_raw, - tuning_holdout_ratio=recipe.tuning_holdout_ratio, - tuning_random_state=recipe.tuning_random_state, - ) - probability_fit, _probability_tune, probability_test = _build_probability_lookups( - inputs, - split, - ) - source_intervals = pd.read_parquet(intervals_path) - if len(source_intervals) != len(inputs.y_test): - raise ValueError("Source conformal intervals and evaluation rows differ in length.") - - grid = _base_grid_frame(source_intervals) - grid["y_pred"] = np.nan - alpha_summaries: list[dict[str, Any]] = [] - results: dict[float, ExactAlphaIntervals] = {} - for alpha in [float(value) for value in design["alpha_grid"]]: - result = compute_exact_alpha_intervals( - recipe=recipe, - target_alpha=alpha, - y_cal=split.y_cal_fit, - interval_probability_cal=probability_fit["calibrated"], - interval_probability_eval=probability_test["calibrated"], - partition_probability_cal=probability_fit[recipe.partition_probability_source], - partition_probability_eval=probability_test[recipe.partition_probability_source], - base_groups_cal=split.group_cal_fit_base, - base_groups_eval=inputs.group_test_base, - issue_dates_eval=split.issue_test, - ) - results[alpha] = result - if grid["y_pred"].isna().all(): - grid["y_pred"] = result.point - elif not np.array_equal(grid["y_pred"].to_numpy(dtype=float), result.point): - raise AssertionError("Point predictions changed across alpha levels.") - _add_alpha_result(grid, result) - summary = _coverage_summary( - result, - y_true=inputs.y_test.to_numpy(dtype=float), - grades=inputs.group_test_base.reset_index(drop=True), - ) - alpha_summaries.append(summary) - logger.info( - "Exact alpha={:.3f} (used={:.4f}): coverage={:.4f}, width={:.4f}, high=1 rate={:.2%}", - alpha, - result.used_alpha, - summary["empirical_coverage"], - summary["avg_width"], - summary["high_endpoint_at_one_rate"], - ) - - reference_alpha = recipe.reference_target_alpha - reference_result = next( - (result for alpha, result in results.items() if np.isclose(alpha, reference_alpha)), - None, - ) - if reference_result is None: - raise ValueError("Alpha grid must contain the recipe reference target alpha.") - replay = _replay_differences(reference_result, source_intervals) - tolerance = float(design["replay_tolerance"]) - replay["tolerance"] = tolerance - replay["pass"] = bool(max(replay.values()) <= tolerance) - if not replay["pass"]: - raise AssertionError(f"Frozen 90% interval replay drifted: {replay}") - - data_dir, model_dir = _experiment_paths(run_tag) - grid_path = data_dir / "exact_alpha_grid.parquet" - summary_path = model_dir / "exact_alpha_grid_summary.json" - grid.to_parquet(grid_path, index=False) - summary_payload: dict[str, Any] = { - "schema_version": str(config["schema_version"]), - "generated_at_utc": _utc_now(), - "run_tag": run_tag, - "source_commit": _git_commit(), - "config_path": str(config_path.relative_to(ROOT)), - "config_sha256": _sha256(config_path), - "source": { - **source, - "conformal_results_sha256": _sha256(results_path), - "conformal_intervals_sha256": _sha256(intervals_path), - }, - "recipe": asdict(recipe), - "alpha_mapping": str(design["alpha_mapping"]), - "reference_replay": replay, - "alpha_summaries": alpha_summaries, - "grid_path": str(grid_path.relative_to(ROOT)), - "grid_rows": int(len(grid)), - "claim_boundary": str(config["claim_boundary"]), - } - write_json(summary_path, summary_payload) - logger.info("Wrote exact alpha grid to {}", grid_path) - logger.info("Wrote exact alpha summary to {}", summary_path) - return summary_payload - - -def _git_commit() -> str: - import subprocess - - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=ROOT, - capture_output=True, - text=True, - check=False, - ) - return result.stdout.strip() if result.returncode == 0 else "unknown" - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) - args = parser.parse_args() - config_path = args.config if args.config.is_absolute() else ROOT / args.config - run(config_path.resolve()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/experiments/run_ijds_fit_label_completion_sensitivity.py b/scripts/experiments/run_ijds_fit_label_completion_sensitivity.py new file mode 100644 index 0000000..6fc828e --- /dev/null +++ b/scripts/experiments/run_ijds_fit_label_completion_sensitivity.py @@ -0,0 +1,466 @@ +"""Run the two-phase fitting-label completion sensitivity.""" + +from __future__ import annotations + +import argparse +import copy +import json +from collections.abc import Mapping, Sequence +from dataclasses import replace +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.evaluation import temporal_coverage_audit +from src.ijds_audit.fit_label_sensitivity import ( + FIT_LABEL_SCENARIOS, + apply_fit_label_scenario, + summarize_fit_label_coverage, +) +from src.ijds_audit.prediction import fit_primary_scores, fit_window_recipes, prepare_data +from src.ijds_audit.protocol import ( + configured_archive_outcomes, + load_outcome_universe, + load_recipes, + recipe_payload, + verified_freeze_artifact_paths, +) +from src.utils.isolated_experiment import ( + environment_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, +) +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = ROOT / "configs/experiments/ijds_fit_label_completion_sensitivity_2026-07-16.yaml" +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +LEARNER_PREFIX = "fit_label_" + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("freeze", "evaluate")) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Fitting-label completion config must be a mapping.") + sensitivity = payload.get("fit_label_completion", {}) + if tuple(sensitivity.get("scenarios", ())) != FIT_LABEL_SCENARIOS: + raise ValueError("Fitting-label scenarios must remain the complete declared family.") + if sensitivity.get("outcome_based_selection") is not False: + raise ValueError("Fitting-label completion cannot select from evaluation outcomes.") + if int(sensitivity.get("taxonomy_groups", 0)) != 5: + raise ValueError("Fitting-label completion must retain the canonical five-group taxonomy.") + if sensitivity.get("evaluation_strata") != [-1, 2]: + raise ValueError("Fitting-label completion must report overall and phase-stratum cells.") + tolerances = ( + "baseline_score_replay_tolerance", + "baseline_recipe_replay_tolerance", + "baseline_coverage_replay_tolerance", + "baseline_geometry_replay_tolerance", + ) + if any(float(sensitivity.get(name, 0.0)) <= 0.0 for name in tolerances): + raise ValueError("Fitting-label replay tolerances must be positive.") + if sensitivity.get("numerical_recovery_only") is not True: + raise ValueError("Fitting-label v2 must remain a numerical recovery only.") + return payload + + +def _verified_descriptor( + descriptor: Mapping[str, Any], + *, + repo_root: Path, +) -> Path: + path = resolve_repo_input(str(descriptor["path"]), repo_root=repo_root) + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor.get(field): + raise RuntimeError(f"Fitting-label parent artifact mismatch for {field}.") + return path + + +def _run_dirs(config: Mapping[str, Any], root: Path) -> tuple[Path, Path]: + data_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=str(config["run_tag"]), + ) + model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(config["run_tag"]), + ) + return data_dir, model_dir + + +def _recipe_difference( + current: Mapping[str, Mapping[int, Any]], + reference: Mapping[str, Mapping[int, Any]], +) -> float: + differences: list[float] = [] + if set(current) != set(reference): + raise RuntimeError("Observed-only residual windows differ from the parent freeze.") + for window_id in sorted(current): + if set(current[window_id]) != {5} or 5 not in reference[window_id]: + raise RuntimeError(f"Canonical recipe is unavailable for {window_id}.") + left = current[window_id][5] + right = reference[window_id][5] + for field in ("bin_edges", "residual_quantiles"): + differences.extend( + np.abs( + np.asarray(getattr(left, field), dtype=float) + - np.asarray(getattr(right, field), dtype=float) + ).tolist() + ) + for field in ("group_counts", "finite_sample_ranks", "raw_finite_sample_ranks"): + if tuple(getattr(left, field)) != tuple(getattr(right, field)): + raise RuntimeError(f"Observed-only recipe changed {field} for {window_id}.") + return max(differences, default=0.0) + + +def freeze(config_path: Path, *, repo_root: Path) -> Path: + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + parent_config_path = resolve_repo_input(config["parent"]["config"], repo_root=root) + parent = load_v4_config(parent_config_path) + parent_freeze_path = _verified_descriptor( + config["parent"]["outcome_freeze"], + repo_root=root, + ) + parent_freeze = json.loads(parent_freeze_path.read_text(encoding="utf-8")) + parent_artifacts = verified_freeze_artifact_paths(parent_freeze, repo_root=root) + parent_scores = pd.read_parquet(parent_artifacts["scores"]) + parent_recipes = load_recipes(parent_artifacts["recipes"])["catboost_platt"] + + raw_path = resolve_repo_input(parent["source"]["raw_path"], repo_root=root) + data = prepare_data(parent, raw_path=raw_path) + sensitivity_config = copy.deepcopy(parent) + sensitivity_config["conformal"]["diagnostic_group_counts"] = [5] + sensitivity_config["execution"]["threads"] = int(config["execution"]["threads"]) + + score_frame = pd.DataFrame( + { + "id": data.universe["id"].astype("string"), + "issue_d": data.universe["issue_d"], + "design_split": data.universe["design_split"].astype("string"), + } + ) + scenario_audits: list[pd.DataFrame] = [] + fit_audits: list[pd.DataFrame] = [] + windows_by_learner: dict[str, Any] = {} + model_metrics: dict[str, Any] = {} + baseline_score_difference: float | None = None + baseline_recipe_difference: float | None = None + for scenario in FIT_LABEL_SCENARIOS: + scenario_universe, scenario_audit = apply_fit_label_scenario( + data.universe, + scenario=scenario, + ) + scenario_audits.append(scenario_audit) + scenario_data = replace(data, universe=scenario_universe) + fitted = fit_primary_scores(scenario_data, sensitivity_config) + learner_name = f"{LEARNER_PREFIX}{scenario}" + fitted = replace(fitted, name=learner_name) + windows = fit_window_recipes(scenario_data, fitted, sensitivity_config) + windows_by_learner[learner_name] = windows + score_frame[f"pd_{learner_name}"] = fitted.probabilities + fit_audits.extend(window.fit_audit for window in windows.values()) + model_metrics[scenario] = fitted.metrics + + if scenario == "observed_only": + if not parent_scores["id"].astype("string").equals(score_frame["id"]): + raise RuntimeError("Observed-only scores do not align to the parent ID census.") + baseline_score_difference = float( + np.max( + np.abs( + fitted.probabilities + - parent_scores["pd_catboost_platt"].to_numpy(dtype=float) + ) + ) + ) + current_recipes = {window_id: window.recipes for window_id, window in windows.items()} + baseline_recipe_difference = _recipe_difference(current_recipes, parent_recipes) + + score_tolerance = float(config["fit_label_completion"]["baseline_score_replay_tolerance"]) + recipe_tolerance = float(config["fit_label_completion"]["baseline_recipe_replay_tolerance"]) + if baseline_score_difference is None or baseline_score_difference > score_tolerance: + raise RuntimeError( + f"Observed-only score replay drifted by {baseline_score_difference}; " + f"tolerance={score_tolerance}." + ) + if baseline_recipe_difference is None or baseline_recipe_difference > recipe_tolerance: + raise RuntimeError( + f"Observed-only recipe replay drifted by {baseline_recipe_difference}; " + f"tolerance={recipe_tolerance}." + ) + baseline_column = f"pd_{LEARNER_PREFIX}observed_only" + score_shift = { + scenario: float( + np.max( + np.abs( + score_frame[f"pd_{LEARNER_PREFIX}{scenario}"].to_numpy(dtype=float) + - score_frame[baseline_column].to_numpy(dtype=float) + ) + ) + ) + for scenario in FIT_LABEL_SCENARIOS + } + + artifact_paths = { + "scores": atomic_write_parquet(score_frame, paths.data_dir / "outcome_free/scores.parquet"), + "fit_audit": atomic_write_parquet( + pd.concat(fit_audits, ignore_index=True), + paths.data_dir / "outcome_free/fit_audit.parquet", + ), + "scenario_audit": atomic_write_parquet( + pd.concat(scenario_audits, ignore_index=True), + paths.data_dir / "outcome_free/scenario_audit.parquet", + ), + "recipes": atomic_write_json( + paths.model_dir / "outcome_free/residual_recipes.json", + recipe_payload(windows_by_learner), + ), + } + freeze_payload = { + "schema_version": str(config["schema_version"]), + "status": "fit_labels_completed_before_evaluation_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "historical_archive_previously_inspected": True, + "outcome_based_selection": False, + "evaluation_outcome_columns_passed_to_fitting": [], + "scenarios": list(FIT_LABEL_SCENARIOS), + "baseline_replay": { + "score_max_abs_difference": baseline_score_difference, + "recipe_max_abs_difference": baseline_recipe_difference, + "score_tolerance": score_tolerance, + "recipe_tolerance": recipe_tolerance, + "v1_stopped_score_drift": float( + config["fit_label_completion"]["v1_stopped_score_drift"] + ), + }, + "score_shift_max_abs_from_observed_only": score_shift, + "model_metrics": model_metrics, + "parent": { + "config": relative_artifact_descriptor(parent_config_path, repo_root=root), + "outcome_freeze": relative_artifact_descriptor(parent_freeze_path, repo_root=root), + }, + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_paths.items() + }, + "implementation": implementation_provenance( + config_path=resolved_config, + repo_root=root, + relative_paths=[ + Path("src/ijds_audit/fit_label_sensitivity.py"), + Path("src/ijds_audit/prediction.py"), + Path("scripts/experiments/run_ijds_fit_label_completion_sensitivity.py"), + Path("docs/research/ijds_fit_label_completion_sensitivity_protocol_2026-07-16.md"), + ], + ), + "environment": environment_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze_payload) + + +def evaluate(config_path: Path, *, repo_root: Path) -> Path: + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + data_dir, model_dir = _run_dirs(config, root) + freeze_path = model_dir / "protocol_freeze.json" + if not freeze_path.is_file(): + raise FileNotFoundError("Run the fitting-label freeze phase first.") + summary_path = model_dir / "fit_label_completion_summary.json" + evaluation_dir = data_dir / "evaluation" + if summary_path.exists() or evaluation_dir.exists(): + raise FileExistsError("Fitting-label evaluation outputs are immutable.") + freeze_payload = json.loads(freeze_path.read_text(encoding="utf-8")) + for field, expected in { + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "status": "fit_labels_completed_before_evaluation_outcome_join", + }.items(): + if freeze_payload.get(field) != expected: + raise RuntimeError(f"Fitting-label freeze mismatch for {field}.") + if freeze_payload.get("evaluation_outcome_columns_passed_to_fitting") != []: + raise RuntimeError("Fitting-label freeze reports evaluation-outcome leakage.") + artifacts = { + name: _verified_descriptor(descriptor, repo_root=root) + for name, descriptor in freeze_payload["artifacts"].items() + } + + parent_config_path = resolve_repo_input(config["parent"]["config"], repo_root=root) + parent = load_v4_config(parent_config_path) + raw_path = resolve_repo_input(parent["source"]["raw_path"], repo_root=root) + universe = load_outcome_universe(parent, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, parent) + scores = pd.read_parquet(artifacts["scores"]) + recipes = load_recipes(artifacts["recipes"]) + fit_audit = pd.read_parquet(artifacts["fit_audit"]) + coverage = temporal_coverage_audit( + scores, + outcomes, + recipes, + fit_audit, + roles=("primary_oot",), + taxonomy_group_counts=(5,), + strata=tuple(config["fit_label_completion"]["evaluation_strata"]), + ) + coverage["fit_label_scenario"] = coverage["learner"].str.removeprefix(LEARNER_PREFIX) + window_ids = tuple(str(item["id"]) for item in parent["residual_specification"]["windows"]) + nominal = 1.0 - float(parent["conformal"]["alpha"]) + summary_table = summarize_fit_label_coverage( + coverage, + window_ids=window_ids, + nominal_coverage=nominal, + ) + phase = coverage.loc[coverage["conformal_group"].eq(2)].copy() + active_coverage_path = _verified_descriptor( + config["parent"]["active_coverage"], + repo_root=root, + ) + active_coverage = pd.read_parquet(active_coverage_path) + active_reference = active_coverage.loc[ + active_coverage["learner"].eq("catboost_platt") + & active_coverage["taxonomy_groups"].eq(5) + & active_coverage["role"].eq("primary_oot") + & active_coverage["conformal_group"].isin((-1, 2)) + ].copy() + observed_replay = coverage.loc[coverage["fit_label_scenario"].eq("observed_only")].copy() + comparison = observed_replay.merge( + active_reference, + on=["window_id", "taxonomy_groups", "role", "conformal_group"], + how="outer", + validate="one_to_one", + suffixes=("_replay", "_active"), + indicator=True, + ) + if len(comparison) != 16 or not bool(comparison["_merge"].eq("both").all()): + raise RuntimeError("Observed-only coverage replay grid does not match the active grid.") + bound_columns = ("coverage_lower", "coverage_upper", "coverage_resolved") + geometry_columns = ("mean_width", "fit_prevalence", "fit_residual_quantile") + bound_difference = max( + float( + np.max( + np.abs( + comparison[f"{column}_replay"].to_numpy(dtype=float) + - comparison[f"{column}_active"].to_numpy(dtype=float) + ) + ) + ) + for column in bound_columns + ) + geometry_difference = max( + float( + np.max( + np.abs( + comparison[f"{column}_replay"].to_numpy(dtype=float) + - comparison[f"{column}_active"].to_numpy(dtype=float) + ) + ) + ) + for column in geometry_columns + ) + bound_tolerance = float(config["fit_label_completion"]["baseline_coverage_replay_tolerance"]) + geometry_tolerance = float(config["fit_label_completion"]["baseline_geometry_replay_tolerance"]) + if bound_difference > bound_tolerance or geometry_difference > geometry_tolerance: + raise RuntimeError( + "Observed-only coverage replay exceeds the declared recovery tolerance: " + f"bounds={bound_difference}/{bound_tolerance}, " + f"geometry={geometry_difference}/{geometry_tolerance}." + ) + evaluation_paths = { + "coverage": atomic_write_parquet(coverage, evaluation_dir / "temporal_coverage.parquet"), + "summary_table": atomic_write_parquet( + summary_table, + evaluation_dir / "scenario_summary.parquet", + ), + "phase_stratum": atomic_write_parquet( + phase, + evaluation_dir / "phase_stratum.parquet", + ), + } + all_below = bool(summary_table["all_windows_upper_below_nominal"].all()) + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_fit_label_completion_corner_sensitivity", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "historical_archive_previously_inspected": True, + "outcome_based_selection": False, + "scenarios": list(FIT_LABEL_SCENARIOS), + "scope": ( + "Four declared joint completion scenarios for 215 fitting labels unavailable " + "at the information cutoff; nonlinear learner refits mean these are stress " + "corners, not sharp bounds over every label assignment." + ), + "results": { + "coverage_rows": int(len(coverage)), + "overall_cells": int(coverage["conformal_group"].eq(-1).sum()), + "phase_cells": int(coverage["conformal_group"].eq(2).sum()), + "all_scenarios_all_windows_upper_below_nominal": all_below, + "scenario_rows": summary_table.to_dict(orient="records"), + "observed_only_active_replay": { + "coverage_max_abs_difference": bound_difference, + "coverage_tolerance": bound_tolerance, + "geometry_max_abs_difference": geometry_difference, + "geometry_tolerance": geometry_tolerance, + }, + }, + "freeze": relative_artifact_descriptor(freeze_path, repo_root=root), + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in evaluation_paths.items() + }, + "environment": environment_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(summary_path, summary) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + path = ( + freeze(args.config, repo_root=args.repo_root) + if args.phase == "freeze" + else evaluate(args.config, repo_root=args.repo_root) + ) + print(path) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_label_lag_sensitivity.py b/scripts/experiments/run_ijds_label_lag_sensitivity.py new file mode 100644 index 0000000..e9bd217 --- /dev/null +++ b/scripts/experiments/run_ijds_label_lag_sensitivity.py @@ -0,0 +1,140 @@ +"""Run the predeclared label-lag sensitivity on the frozen V4 score.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import pandas as pd +import yaml + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.lag_sensitivity import build_label_lag_phase_sensitivity +from src.ijds_audit.protocol import ( + load_outcome_universe, + load_recipes, + verified_freeze_artifact_paths, +) +from src.utils.isolated_experiment import ( + implementation_provenance, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_repo_input, + validate_run_tag, + write_csv_atomic, +) +from src.utils.pipeline_runtime import atomic_write_json + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = ROOT / "configs/experiments/ijds_label_lag_sensitivity_2026-07-14.yaml" +ALLOWED_OUTPUT_ROOT = Path("reports/crpto/sensitivity") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Label-lag sensitivity config must be a mapping.") + sensitivity = payload.get("lag_sensitivity", {}) + if sensitivity.get("outcome_based_selection") is not False: + raise ValueError("Label-lag sensitivity cannot select from outcomes.") + if [int(value) for value in sensitivity.get("charged_off_lag_months", [])] != [ + 0, + 3, + 6, + 8, + 12, + ]: + raise ValueError("Label-lag sensitivity grid must remain 0/3/6/8/12 months.") + return payload + + +def _output_dir(config: dict[str, Any], root: Path) -> Path: + allowed = (root / ALLOWED_OUTPUT_ROOT).resolve() + output = (allowed / validate_run_tag(str(config["run_tag"]))).resolve() + output.relative_to(allowed) + if output.exists(): + raise FileExistsError(f"Sensitivity output already exists: {output}") + output.mkdir(parents=True) + return output + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + root = args.repo_root.resolve() + config_path = resolve_repo_input(args.config, repo_root=root) + config = _load_config(config_path) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + parent_path = resolve_repo_input(config["parent"]["config"], repo_root=root) + parent = load_v4_config(parent_path) + freeze_path = resolve_repo_input(config["parent"]["protocol_freeze"]["path"], repo_root=root) + freeze_descriptor = relative_artifact_descriptor(freeze_path, repo_root=root) + for field in ("path", "bytes", "sha256"): + if freeze_descriptor[field] != config["parent"]["protocol_freeze"][field]: + raise RuntimeError(f"Parent freeze mismatch for {field}.") + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + artifacts = verified_freeze_artifact_paths(freeze, repo_root=root) + raw_path = resolve_repo_input(parent["source"]["raw_path"], repo_root=root) + universe = load_outcome_universe(parent, raw_path=raw_path) + scores = pd.read_parquet(artifacts["scores"]) + if not scores["id"].astype("string").equals(universe["id"].astype("string")): + raise RuntimeError("Frozen scores do not align to the design universe.") + recipes = load_recipes(artifacts["recipes"])["catboost_platt"] + table = build_label_lag_phase_sensitivity( + universe, + scores["pd_catboost_platt"].to_numpy(dtype=float), + recipes, + {**parent, "lag_sensitivity": config["lag_sensitivity"]}, + lag_months=[int(value) for value in config["lag_sensitivity"]["charged_off_lag_months"]], + ) + output = _output_dir(config, root) + table_path = write_csv_atomic(table, output / "label_lag_phase_sensitivity.csv") + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_frozen_score_label_lag_sensitivity", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "historical_archive_previously_inspected": True, + "outcome_based_selection": False, + "source_freeze": freeze_descriptor, + "results": { + "rows": int(len(table)), + "lags": sorted(table["charged_off_lag_months"].unique().tolist()), + "windows": int(table["window_id"].nunique()), + "minimum_monthly_retention_by_lag": { + str(int(lag)): float(frame["minimum_monthly_retention"].iloc[0]) + for lag, frame in table.groupby("charged_off_lag_months", sort=True) + }, + "retention_stop_pass_by_lag": { + str(int(lag)): bool(frame["passes_locked_retention"].all()) + for lag, frame in table.groupby("charged_off_lag_months", sort=True) + }, + }, + "artifact": relative_artifact_descriptor(table_path, repo_root=root), + "implementation": implementation_provenance( + config_path=config_path, + repo_root=root, + relative_paths=[ + Path("src/ijds_audit/lag_sensitivity.py"), + Path("scripts/experiments/run_ijds_label_lag_sensitivity.py"), + Path("docs/research/ijds_label_lag_sensitivity_protocol_2026-07-14.md"), + ], + ), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + print(atomic_write_json(output / "evidence.json", summary)) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_missingness_sensitivity.py b/scripts/experiments/run_ijds_missingness_sensitivity.py new file mode 100644 index 0000000..f488f06 --- /dev/null +++ b/scripts/experiments/run_ijds_missingness_sensitivity.py @@ -0,0 +1,36 @@ +"""Run one phase of the protocol-locked IJDS missingness sensitivity.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from pathlib import Path + +from src.ijds_audit.missingness_sensitivity import ( + evaluate_missingness_sensitivity, + freeze_missingness_sensitivity, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("freeze", "evaluate")) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + runner = ( + freeze_missingness_sensitivity + if args.phase == "freeze" + else evaluate_missingness_sensitivity + ) + print(runner(config_path=args.config, repo_root=args.repo_root)) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_normalized_objective_frontier.py b/scripts/experiments/run_ijds_normalized_objective_frontier.py new file mode 100644 index 0000000..1c5c6cf --- /dev/null +++ b/scripts/experiments/run_ijds_normalized_objective_frontier.py @@ -0,0 +1,351 @@ +"""Run the locked outcome-free IJDS normalized/objective frontier V1c.""" + +from __future__ import annotations + +import argparse +import sys +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from src.ijds_audit.config import load_v4_config # noqa: E402 +from src.ijds_audit.protocol import load_recipes # noqa: E402 +from src.ijds_challengers.archive import ( # noqa: E402 + load_outcome_free_decision_base, + verified_parent_artifacts, +) +from src.ijds_challengers.config import load_frontier_config # noqa: E402 +from src.ijds_challengers.normalized_frontier import ( # noqa: E402 + FrontierBuild, + build_outcome_free_frontiers, +) +from src.utils.isolated_experiment import ( # noqa: E402 + OutputPaths, + dataframe_schema, + environment_provenance, + git_provenance, + implementation_provenance, + package_version, + prepare_output_paths as prepare_isolated_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_repo_input, +) +from src.utils.pipeline_runtime import ( # noqa: E402 + atomic_write_json, + atomic_write_parquet, + utc_now_iso, +) + +DEFAULT_CONFIG_PATH = ( + ROOT / "configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml" +) +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +IMPLEMENTATION_PATHS = ( + Path("docs/research/ijds_normalized_objective_frontier_protocol_2026-07-12.md"), + Path("docs/research/ijds_normalized_objective_frontier_v1_stop_2026-07-13.md"), + Path("docs/research/ijds_normalized_objective_frontier_v1b_protocol_2026-07-13.md"), + Path("docs/research/ijds_normalized_objective_frontier_v1b_stop_2026-07-13.md"), + Path("docs/research/ijds_normalized_objective_frontier_v1c_protocol_2026-07-13.md"), + Path("docs/research/ijds_decision_method_applicability_2026-07-12.md"), + Path("scripts/experiments/run_ijds_normalized_objective_frontier.py"), + Path("src/evaluation/standardized_credit_payoff.py"), + Path("src/ijds_audit/config.py"), + Path("src/ijds_audit/portfolio.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/ijds_challengers/__init__.py"), + Path("src/ijds_challengers/archive.py"), + Path("src/ijds_challengers/config.py"), + Path("src/ijds_challengers/frontier.py"), + Path("src/ijds_challengers/normalized_frontier.py"), + Path("src/models/binary_conformal_guardrail.py"), + Path("src/utils/isolated_experiment.py"), + Path("src/utils/pipeline_runtime.py"), + Path("tests/test_ijds_normalized_objective_frontier.py"), + Path("pyproject.toml"), + Path("uv.lock"), +) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the outcome-free V1 CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH) + return parser.parse_args(argv) + + +def prepare_output_paths( + config: Mapping[str, Any], + *, + repo_root: Path = ROOT, +) -> OutputPaths: + """Create fresh V1 output directories contained in the IJDS experiment roots.""" + return prepare_isolated_output_paths( + dict(config), + repo_root=repo_root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + + +def _summary( + build: FrontierBuild, + *, + config: Mapping[str, Any], + parent_freeze: Mapping[str, Any], + protocol_commit: str, +) -> dict[str, Any]: + records = build.solve_records + endpoints = build.endpoint_diagnostics + optimum = build.objective_optimum_diagnostics + order = build.order_sensitivity + validation = build.independent_validation + degeneracy_tolerance = float(config["solver"]["endpoint_pair_degeneracy_tolerance"]) + ruler_counts = { + str(ruler): { + "comparisons": int(len(frame)), + "nonidentical_comparisons": int( + (frame["normalized_exposure_distance"] > degeneracy_tolerance).sum() + ), + "maximum_exposure_distance": float(frame["normalized_exposure_distance"].max()), + } + for ruler, frame in endpoints.groupby("ruler", observed=True, sort=True) + } + degenerate_rulers = sorted( + ruler for ruler, values in ruler_counts.items() if values["nonidentical_comparisons"] == 0 + ) + status = ( + "stopped_outcome_free_endpoint_degeneracy" + if degenerate_rulers + else "outcome_free_frontiers_frozen_before_archive_outcome_join" + ) + return { + "schema_version": str(config["schema_version"]), + "status": status, + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "lineage": dict(config["lineage"]), + "parent_run_tag": str(config["parent"]["run_tag"]), + "parent_protocol_freeze_sha256": str(config["parent"]["protocol_freeze"]["sha256"]), + "parent_status": parent_freeze.get("status"), + "counts": { + "solve_records": int(len(records)), + "funded_rows": int(len(build.allocations)), + "endpoint_comparisons": int(len(endpoints)), + "objective_optimum_diagnostics": int(len(optimum)), + "order_reruns": int(len(order)), + "independent_solver_cells": int(len(validation)), + "windows": int(records["window_id"].nunique()), + "roles": int(records["role"].nunique()), + "periods": int(records[["role", "period"]].drop_duplicates().shape[0]), + "gammas": int(records["gamma"].nunique()), + "coordinates": int(records["frontier_coordinate"].nunique()), + "rulers": int(records["frontier_ruler"].nunique()), + }, + "frontier_ranges": { + "minimum_score_range": float(records["score_range"].min()), + "maximum_score_range": float(records["score_range"].max()), + "minimum_common_objective_range_dollars": float( + (records["unconstrained_objective"] - records["common_objective_lower"]).min() + ), + "maximum_common_objective_range_dollars": float( + (records["unconstrained_objective"] - records["common_objective_lower"]).max() + ), + }, + "objective_optimum_stability": { + "minimum_absolute_nonbasic_reduced_cost": float( + optimum["minimum_absolute_nonbasic_reduced_cost"].min() + ), + "minimum_scaled_nonbasic_reduced_cost": float( + optimum["minimum_scaled_nonbasic_reduced_cost"].min() + ), + "near_zero_nonbasic_reduced_costs": int( + optimum["near_zero_nonbasic_reduced_costs"].sum() + ), + "primal_degenerate_menus": int(optimum["basis_primal_degenerate"].sum()), + "maximum_reversed_id_exposure_distance": float( + optimum["reversed_id_exposure_distance"].max() + ), + "maximum_reversed_id_objective_difference_dollars": float( + optimum["reversed_id_objective_difference"].abs().max() + ), + }, + "numerical_reconciliation": { + "maximum_budget_residual_dollars": float(records["budget_residual"].abs().max()), + "maximum_constraint_slack_absolute": float(records["constraint_slack"].abs().max()), + "maximum_order_exposure_distance": float(order["normalized_exposure_distance"].max()), + "maximum_order_objective_difference_dollars": float( + order["objective_difference"].abs().max() + ), + "maximum_glop_objective_rate_difference": float( + validation["objective_rate_difference"].abs().max() + ), + "maximum_glop_weighted_score_difference": float( + validation["weighted_score_difference"].abs().max() + ), + }, + "endpoint_comparison_by_ruler": ruler_counts, + "degenerate_rulers": degenerate_rulers, + "outcome_columns_passed": [], + "policy_selection": None, + "policy_winner": None, + "conformal_guarantee_repair": False, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + + +def run_outcome_free( + *, + config_path: Path, + repo_root: Path = ROOT, +) -> Path: + """Verify sources, solve the complete census, and atomically freeze V1c.""" + started = time.perf_counter() + started_at = utc_now_iso() + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_frontier_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + parent_paths, parent_freeze = verified_parent_artifacts(config, repo_root=root) + parent_config_path = resolve_repo_input(str(config["parent"]["config"]), repo_root=root) + parent_config = load_v4_config(parent_config_path) + raw_path = resolve_repo_input(str(config["source_ingest"]["raw_path"]), repo_root=root) + base = load_outcome_free_decision_base( + scores_path=parent_paths["scores"], + raw_path=raw_path, + config=config, + ) + recipes = load_recipes(parent_paths["recipes"]) + build = build_outcome_free_frontiers( + base, + recipes, + config=config, + parent_config=parent_config, + ) + summary = _summary( + build, + config=config, + parent_freeze=parent_freeze, + protocol_commit=protocol_commit, + ) + paths = prepare_output_paths(config, repo_root=root) + frontier_dir = paths.data_dir / "frontier" + output = config["output"] + artifact_files = { + "solve_records": atomic_write_parquet( + build.solve_records, + frontier_dir / str(output["solve_records"]), + ), + "allocations": atomic_write_parquet( + build.allocations, + frontier_dir / str(output["allocations"]), + ), + "endpoint_diagnostics": atomic_write_parquet( + build.endpoint_diagnostics, + frontier_dir / str(output["endpoint_diagnostics"]), + ), + "objective_optimum_diagnostics": atomic_write_parquet( + build.objective_optimum_diagnostics, + frontier_dir / str(output["objective_optimum_diagnostics"]), + ), + "order_sensitivity": atomic_write_parquet( + build.order_sensitivity, + frontier_dir / str(output["order_sensitivity"]), + ), + "independent_validation": atomic_write_parquet( + build.independent_validation, + frontier_dir / str(output["independent_validation"]), + ), + } + summary_path = atomic_write_json( + paths.model_dir / str(output["deterministic_summary"]), + summary, + ) + receipt_path = atomic_write_json( + paths.model_dir / str(output["execution_receipt"]), + { + "schema_version": str(config["schema_version"]), + "status": summary["status"], + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "elapsed_seconds": float(time.perf_counter() - started), + "outcome_columns_passed": [], + "protected_stages_run": [], + "protected_artifacts_written": [], + }, + ) + implementation = implementation_provenance( + config_path=resolved_config, + relative_paths=IMPLEMENTATION_PATHS, + repo_root=root, + ) + environment = environment_provenance(root) + environment["packages"]["ortools"] = package_version("ortools") + freeze = { + "schema_version": str(config["schema_version"]), + "status": summary["status"], + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "lineage": dict(config["lineage"]), + "parent": { + "run_tag": str(config["parent"]["run_tag"]), + "protocol_freeze": dict(config["parent"]["protocol_freeze"]), + }, + "outcome_columns_passed_to_frontier": [], + "policy_selection": None, + "window_selection": None, + "ruler_selection": None, + "implementation_provenance": implementation, + "environment": environment, + "git": git_provenance(root), + "schemas": { + "solve_records": dataframe_schema(build.solve_records), + "allocations": dataframe_schema(build.allocations), + "endpoint_diagnostics": dataframe_schema(build.endpoint_diagnostics), + "objective_optimum_diagnostics": dataframe_schema(build.objective_optimum_diagnostics), + "order_sensitivity": dataframe_schema(build.order_sensitivity), + "independent_validation": dataframe_schema(build.independent_validation), + }, + "outcome_free_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_files.items() + }, + "summary": relative_artifact_descriptor(summary_path, repo_root=root), + "execution_receipt": relative_artifact_descriptor(receipt_path, repo_root=root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + freeze_path = atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze) + logger.info( + "Outcome-free frontier {}: {} records, {} funded rows in {:.1f}s", + summary["status"], + len(build.solve_records), + len(build.allocations), + time.perf_counter() - started, + ) + return freeze_path + + +def main(argv: Sequence[str] | None = None) -> None: + """Run the locked V1 outcome-free phase.""" + args = parse_args(argv) + freeze = run_outcome_free(config_path=args.config, repo_root=ROOT) + logger.info("Wrote {}", freeze) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_normalized_objective_frontier_v2.py b/scripts/experiments/run_ijds_normalized_objective_frontier_v2.py new file mode 100644 index 0000000..a958942 --- /dev/null +++ b/scripts/experiments/run_ijds_normalized_objective_frontier_v2.py @@ -0,0 +1,393 @@ +"""Evaluate the hash-verified V1c two-ruler frontier after one outcome join.""" + +from __future__ import annotations + +import argparse +import sys +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import pandas as pd +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from src.ijds_audit.config import load_v4_config # noqa: E402 +from src.ijds_audit.endpoint_recovery import reconcile_from_json_reference # noqa: E402 +from src.ijds_audit.evaluation import evaluate_frozen_portfolios # noqa: E402 +from src.ijds_audit.protocol import ( # noqa: E402 + configured_archive_outcomes, + load_outcome_universe, +) +from src.ijds_challengers.evaluation import ( # noqa: E402 + FrozenFrontier, + build_endpoint_contrasts, + build_metric_directions, + validate_complete_evaluation, + validate_outcome_alignment, + verify_frontier_freeze, +) +from src.ijds_challengers.evaluation_config import load_v2_config # noqa: E402 +from src.utils.isolated_experiment import ( # noqa: E402 + OutputPaths, + dataframe_schema, + environment_provenance, + git_provenance, + implementation_provenance, + prepare_output_paths as prepare_isolated_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, + sha256_file, +) +from src.utils.pipeline_runtime import ( # noqa: E402 + atomic_write_json, + atomic_write_parquet, + utc_now_iso, +) + +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +IMPLEMENTATION_PATHS = ( + Path("docs/research/ijds_normalized_objective_frontier_v2_protocol_2026-07-13.md"), + Path("docs/research/ijds_normalized_objective_frontier_v1c_results_2026-07-13.md"), + Path("scripts/experiments/run_ijds_normalized_objective_frontier_v2.py"), + Path("src/evaluation/maturity_safe_portfolio.py"), + Path("src/evaluation/policy_contrast_bounds.py"), + Path("src/evaluation/standardized_credit_payoff.py"), + Path("src/ijds_audit/evaluation.py"), + Path("src/ijds_audit/endpoint_recovery.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/ijds_challengers/evaluation.py"), + Path("src/ijds_challengers/evaluation_config.py"), + Path("src/utils/isolated_experiment.py"), + Path("src/utils/pipeline_runtime.py"), + Path("tests/test_ijds_normalized_objective_frontier_v2.py"), + Path("pyproject.toml"), + Path("uv.lock"), +) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse one explicit hash-verified evaluation configuration.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + return parser.parse_args(argv) + + +def prepare_output_paths( + config: Mapping[str, Any], + *, + repo_root: Path = ROOT, +) -> OutputPaths: + """Create fresh V2 output directories inside the IJDS experiment roots.""" + return prepare_isolated_output_paths( + dict(config), + repo_root=repo_root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + + +def preflight_output_paths( + config: Mapping[str, Any], + *, + repo_root: Path = ROOT, +) -> OutputPaths: + """Reject an occupied run tag before reading any archive outcome.""" + output = config["output"] + run_tag = str(config["run_tag"]) + paths = OutputPaths( + data_dir=resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=run_tag, + ), + model_dir=resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=run_tag, + ), + ) + existing = [path for path in (paths.data_dir, paths.model_dir) if path.exists()] + if existing: + rendered = ", ".join(str(path) for path in existing) + raise FileExistsError( + f"Experiment output already exists ({rendered}); choose a fresh run tag." + ) + return paths + + +def _direction_summary(directions: pd.DataFrame) -> dict[str, Any]: + result: dict[str, Any] = {} + for metric, frame in directions.groupby("metric", observed=True, sort=True): + counts = frame["direction"].value_counts().sort_index().to_dict() + observed = {str(value) for value in frame["direction"]} + universal = ( + next(iter(observed)) + if len(observed) == 1 and observed.isdisjoint({"crosses_zero", "exact_zero"}) + else None + ) + result[str(metric)] = { + "cells": int(len(frame)), + "direction_counts": {str(key): int(value) for key, value in counts.items()}, + "universal_nonzero_direction": universal, + "eligible_for_separate_rolling_origin_challenger": bool(universal is not None), + } + return result + + +def _summary( + *, + config: Mapping[str, Any], + protocol_commit: str, + frontier: FrozenFrontier, + evaluated: pd.DataFrame, + joined: pd.DataFrame, + window_contrasts: pd.DataFrame, + monthly_contrasts: pd.DataFrame, + directions: pd.DataFrame, + outcome_audit: pd.DataFrame, +) -> dict[str, Any]: + return { + "schema_version": str(config["schema_version"]), + "status": "verified_post_freeze_outcome_evaluation_complete", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "source_frontier": { + "run_tag": str(config["source_frontier"]["run_tag"]), + "freeze": dict(config["source_frontier"]["freeze"]), + "source_counts": dict(frontier.summary["counts"]), + }, + "counts": { + "evaluated_portfolios": int(len(evaluated)), + "joined_funded_rows": int(len(joined)), + "window_endpoint_contrasts": int(len(window_contrasts)), + "monthly_endpoint_contrasts": int(len(monthly_contrasts)), + "metric_direction_cells": int(len(directions)), + "outcome_audit_rows": int(len(outcome_audit)), + }, + "outcomes": { + "joined_columns": list(config["outcomes"]["joined_columns"]), + "candidate_unresolved_by_role": { + str(role): int(frame["unresolved_rows"].sum()) + for role, frame in outcome_audit.groupby("role", observed=True, sort=True) + }, + "outcome_refit": False, + "outcome_resolution": False, + "outcome_selection": False, + }, + "structural_activity": { + "nonidentical_months": int(window_contrasts["nonidentical_months"].sum()), + "minimum_nonidentical_months_per_window_cell": int( + window_contrasts["nonidentical_months"].min() + ), + "maximum_nonidentical_months_per_window_cell": int( + window_contrasts["nonidentical_months"].max() + ), + }, + "metric_directions": _direction_summary(directions), + "policy_selection": None, + "window_selection": None, + "ruler_selection": None, + "coordinate_selection": None, + "gamma_selection": None, + "policy_winner": None, + "causal_interpretation": False, + "conformal_guarantee_repair": False, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + + +def run_evaluation(*, config_path: Path, repo_root: Path = ROOT) -> Path: + """Verify V1c before loading outcomes, then evaluate the complete fixed grid.""" + started = time.perf_counter() + started_at = utc_now_iso() + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_v2_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + preflight_output_paths(config, repo_root=root) + + frontier = verify_frontier_freeze(config, repo_root=root) + records = pd.read_parquet(frontier.artifacts["solve_records"]) + allocations = pd.read_parquet(frontier.artifacts["allocations"]) + endpoints = pd.read_parquet(frontier.artifacts["endpoint_diagnostics"]) + + parent_config_path = resolve_repo_input(str(config["parent"]["config"]), repo_root=root) + parent_config = load_v4_config(parent_config_path) + raw_path = resolve_repo_input(str(config["parent"]["raw_path"]), repo_root=root) + if sha256_file(raw_path) != str(config["parent"]["raw_sha256"]): + raise RuntimeError("The locked raw archive hash changed before the V2 outcome join.") + universe = load_outcome_universe(parent_config, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, parent_config) + outcome_audit = validate_outcome_alignment(allocations, outcomes, config=config) + + evaluated, joined = evaluate_frozen_portfolios( + records, + allocations, + outcomes, + config=parent_config, + ) + window_contrasts, monthly_contrasts = build_endpoint_contrasts( + joined, + endpoints, + config=config, + lgd=float(parent_config["payoff"]["lgd"]), + ) + directions = build_metric_directions(window_contrasts, config=config) + validate_complete_evaluation( + evaluated, + joined, + window_contrasts, + monthly_contrasts, + directions, + config=config, + ) + endpoint_recovery = config.get("endpoint_reason_recovery") + recovery_audit = None + if endpoint_recovery: + recovery_audit = reconcile_from_json_reference( + { + "evaluated_portfolios": evaluated, + "window_endpoint_contrasts": window_contrasts, + "monthly_endpoint_contrasts": monthly_contrasts, + "metric_direction_census": directions, + }, + reference_json=endpoint_recovery["reference_json"], + artifact_section=str(endpoint_recovery["artifact_section"]), + repo_root=root, + float_atol=float(endpoint_recovery.get("float_atol", 0.0)), + float_rtol=float(endpoint_recovery.get("float_rtol", 0.0)), + ) + summary = _summary( + config=config, + protocol_commit=protocol_commit, + frontier=frontier, + evaluated=evaluated, + joined=joined, + window_contrasts=window_contrasts, + monthly_contrasts=monthly_contrasts, + directions=directions, + outcome_audit=outcome_audit, + ) + summary["endpoint_reason_recovery"] = recovery_audit + + paths = prepare_output_paths(config, repo_root=root) + evaluation_dir = paths.data_dir / "evaluation" + output = config["output"] + artifact_files = { + "evaluated_portfolios": atomic_write_parquet( + evaluated, evaluation_dir / str(output["evaluated_portfolios"]) + ), + "joined_funded_allocations": atomic_write_parquet( + joined, evaluation_dir / str(output["joined_funded_allocations"]) + ), + "window_endpoint_contrasts": atomic_write_parquet( + window_contrasts, evaluation_dir / str(output["window_endpoint_contrasts"]) + ), + "monthly_endpoint_contrasts": atomic_write_parquet( + monthly_contrasts, evaluation_dir / str(output["monthly_endpoint_contrasts"]) + ), + "metric_direction_census": atomic_write_parquet( + directions, evaluation_dir / str(output["metric_direction_census"]) + ), + "outcome_join_audit": atomic_write_parquet( + outcome_audit, evaluation_dir / str(output["outcome_join_audit"]) + ), + } + summary_path = atomic_write_json( + paths.model_dir / str(output["deterministic_summary"]), summary + ) + receipt_path = atomic_write_json( + paths.model_dir / str(output["execution_receipt"]), + { + "schema_version": str(config["schema_version"]), + "status": summary["status"], + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "elapsed_seconds": float(time.perf_counter() - started), + "source_frontier_freeze_sha256": str(config["source_frontier"]["freeze"]["sha256"]), + "protected_stages_run": [], + "protected_artifacts_written": [], + }, + ) + manifest = { + "schema_version": str(config["schema_version"]), + "status": summary["status"], + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "source_frontier_freeze": dict(config["source_frontier"]["freeze"]), + "source_artifacts": { + name: dict(frontier.freeze["outcome_free_artifacts"][name]) + for name in sorted(frontier.artifacts) + }, + "outcome_columns_joined_after_freeze": list(config["outcomes"]["joined_columns"]), + "schemas": { + "evaluated_portfolios": dataframe_schema(evaluated), + "joined_funded_allocations": dataframe_schema(joined), + "window_endpoint_contrasts": dataframe_schema(window_contrasts), + "monthly_endpoint_contrasts": dataframe_schema(monthly_contrasts), + "metric_direction_census": dataframe_schema(directions), + "outcome_join_audit": dataframe_schema(outcome_audit), + }, + "evaluation_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_files.items() + }, + "summary": relative_artifact_descriptor(summary_path, repo_root=root), + "execution_receipt": relative_artifact_descriptor(receipt_path, repo_root=root), + "implementation_provenance": implementation_provenance( + config_path=resolved_config, + relative_paths=( + *IMPLEMENTATION_PATHS, + *[Path(value) for value in config.get("protocol_lineage_files", [])], + ), + repo_root=root, + ), + "environment": environment_provenance(root), + "git": git_provenance(root), + "selection": { + "policy": None, + "window": None, + "ruler": None, + "coordinate": None, + "gamma": None, + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + manifest_path = atomic_write_json( + paths.model_dir / str(output["evaluation_manifest"]), manifest + ) + logger.info( + "Frontier evaluation complete: {} portfolios, {} window contrasts in {:.1f}s", + len(evaluated), + len(window_contrasts), + time.perf_counter() - started, + ) + return manifest_path + + +def main(argv: Sequence[str] | None = None) -> None: + """Run the locked post-freeze evaluation.""" + args = parse_args(argv) + manifest = run_evaluation(config_path=args.config, repo_root=ROOT) + logger.info("Wrote {}", manifest) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_policy_support_tie_audit.py b/scripts/experiments/run_ijds_policy_support_tie_audit.py new file mode 100644 index 0000000..8876355 --- /dev/null +++ b/scripts/experiments/run_ijds_policy_support_tie_audit.py @@ -0,0 +1,798 @@ +"""Run the locked outcome-free IJDS policy-support and solver-tie audit.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from src.evaluation.standardized_credit_payoff import ( # noqa: E402 + contractual_rate_decimal, + expected_objective_coefficients, +) +from src.ijds_audit.config import load_v4_config # noqa: E402 +from src.ijds_audit.policy_support import ( # noqa: E402 + build_cap_census, + classify_cap, + point_basis_diagnostics, +) +from src.ijds_audit.portfolio import PointPortfolioSession # noqa: E402 +from src.ijds_audit.protocol import load_recipes, verified_freeze_artifact_paths # noqa: E402 +from src.models.binary_conformal_guardrail import apply_binary_outcome_recipe # noqa: E402 +from src.utils.isolated_experiment import ( # noqa: E402 + OutputPaths, + dataframe_schema, + environment_provenance, + git_provenance, + implementation_provenance, + prepare_output_paths as prepare_isolated_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_repo_input, + sha256_file, +) +from src.utils.pipeline_runtime import ( # noqa: E402 + atomic_write_json, + atomic_write_parquet, + utc_now_iso, +) + +DEFAULT_CONFIG_PATH = ROOT / "configs/experiments/ijds_policy_support_tie_audit_2026-07-12.yaml" +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +IMPLEMENTATION_PATHS = ( + Path("docs/research/ijds_policy_support_tie_audit_protocol_2026-07-12.md"), + Path("scripts/experiments/run_ijds_policy_support_tie_audit.py"), + Path("src/evaluation/standardized_credit_payoff.py"), + Path("src/ijds_audit/config.py"), + Path("src/ijds_audit/policy_support.py"), + Path("src/ijds_audit/portfolio.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/models/binary_conformal_guardrail.py"), + Path("src/utils/isolated_experiment.py"), + Path("src/utils/pipeline_runtime.py"), + Path("tests/test_ijds_policy_support_tie_audit.py"), + Path("pyproject.toml"), + Path("uv.lock"), +) +OUTCOME_TOKENS = ("status", "outcome", "default", "pymnt", "realized", "miscoverage") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the audit CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH) + return parser.parse_args(argv) + + +def load_config(path: Path) -> dict[str, Any]: + """Load and validate the locked structural-audit contract.""" + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Policy-support audit config must be a YAML mapping.") + required = { + "protocol_status", + "protocol_tag", + "run_tag", + "parent", + "source_ingest", + "family_audit", + "comparator_support", + "tie_audit", + "claim_boundary", + "output", + } + missing = sorted(required.difference(payload)) + if missing: + raise ValueError(f"Policy-support audit config is missing sections: {missing}.") + if payload["protocol_status"] != "locked_outcome_free_structural_audit_before_execution": + raise ValueError("Policy-support audit protocol is not locked.") + family = payload["family_audit"] + if [float(value) for value in family["risk_tolerances"]] != [0.15, 0.17, 0.19]: + raise ValueError("The inherited risk-tolerance census changed.") + if [float(value) for value in family["gamma_grid"]] != [0.0, 0.25, 0.5, 0.75, 1.0]: + raise ValueError("The semantic gamma endpoint census changed.") + if payload["claim_boundary"].get("outcome_columns_passed") != []: + raise ValueError("The policy-support audit cannot accept outcome columns.") + if payload["claim_boundary"].get("no_policy_promotion") is not True: + raise ValueError("Policy promotion must remain forbidden.") + if payload["output"].get("immutability") != "hard_no_overwrite_choose_fresh_run_tag": + raise ValueError("Policy-support outputs must remain immutable.") + return cast(dict[str, Any], payload) + + +def prepare_output_paths(config: Mapping[str, Any], *, repo_root: Path = ROOT) -> OutputPaths: + """Create fresh, contained audit output directories.""" + return prepare_isolated_output_paths( + dict(config), + repo_root=repo_root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + + +def _json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError(f"Expected a JSON object at {path}.") + return payload + + +def _verified_parent_paths( + config: Mapping[str, Any], *, repo_root: Path +) -> tuple[dict[str, Path], dict[str, Any]]: + parent = config["parent"] + descriptor = parent["protocol_freeze"] + freeze_path = (repo_root / str(descriptor["path"])).resolve() + actual = relative_artifact_descriptor(freeze_path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Parent protocol freeze mismatch for {field}.") + freeze = _json(freeze_path) + expected = { + "run_tag": str(parent["run_tag"]), + "protocol_tag": str(parent["protocol_tag"]), + "protocol_commit": str(parent["protocol_commit"]), + "status": "outcome_free_allocations_frozen_before_archive_outcome_join", + } + for field, value in expected.items(): + if freeze.get(field) != value: + raise RuntimeError(f"Parent freeze field mismatch: {field}.") + if freeze.get("outcome_columns_passed_to_policy_or_comparator") != []: + raise RuntimeError("Parent outcome-free freeze reports an outcome column.") + return verified_freeze_artifact_paths(freeze, repo_root=repo_root), freeze + + +def _load_decision_base( + *, + scores_path: Path, + raw_path: Path, + config: Mapping[str, Any], +) -> pd.DataFrame: + source = config["source_ingest"] + if sha256_file(raw_path) != str(source["raw_sha256"]): + raise RuntimeError("Raw archive hash does not match the locked audit source.") + scores = pd.read_parquet(scores_path) + roles = {str(value) for value in config["family_audit"]["roles"]} + score_frame = scores.loc[scores["design_split"].isin(roles)].copy() + score_frame["id"] = score_frame["id"].astype("string") + if bool(score_frame["id"].duplicated().any()): + raise RuntimeError("Frozen decision scores contain duplicate IDs.") + target_ids = set(score_frame["id"].astype(str)) + allowed = [str(value) for value in source["allowed_raw_columns"]] + if any(token in column.casefold() for column in allowed for token in OUTCOME_TOKENS): + raise ValueError("Raw-column allowlist contains an outcome-like name.") + pieces: list[pd.DataFrame] = [] + for chunk in pd.read_csv( + raw_path, + usecols=lambda column: str(column) in allowed, + dtype={"id": "string"}, + chunksize=int(source["chunksize"]), + low_memory=False, + ): + selected = chunk.loc[chunk["id"].astype(str).isin(target_ids)] + if not selected.empty: + pieces.append(selected) + raw = pd.concat(pieces, ignore_index=True) + raw["id"] = raw["id"].astype("string") + if bool(raw["id"].duplicated().any()) or set(raw["id"].astype(str)) != target_ids: + raise RuntimeError("Raw decision fields do not align one-to-one with frozen scores.") + frame = score_frame.merge(raw, on="id", how="left", validate="one_to_one") + frame = frame.rename(columns={"pd_catboost_platt": "pd_point"}) + frame["issue_d"] = pd.to_datetime(frame["issue_d"]) + frame["loan_amnt"] = pd.to_numeric(frame["loan_amnt"], errors="raise").astype(float) + frame["purpose"] = frame["purpose"].astype("string").fillna("unknown") + frame["contractual_rate"] = contractual_rate_decimal(frame["int_rate"]) + frame = frame.drop(columns=["int_rate", "pd_numeric_logistic_platt"]) + forbidden = [ + column + for column in frame.columns + if any(token in str(column).casefold() for token in OUTCOME_TOKENS) + ] + if forbidden: + raise RuntimeError(f"Decision base contains forbidden columns: {forbidden}.") + counts = frame["design_split"].value_counts().to_dict() + expected = {"policy_development": 94_885, "primary_oot": 376_890} + if counts != expected: + raise RuntimeError(f"Decision role census changed: {counts}.") + return frame.sort_values(["issue_d", "id"], kind="mergesort").reset_index(drop=True) + + +def _portfolio_session( + month: pd.DataFrame, + *, + score: np.ndarray, + objective: np.ndarray, + parent_config: Mapping[str, Any], + threads: int, +) -> PointPortfolioSession: + return PointPortfolioSession( + month, + point_score=score, + objective_rate=objective, + budget=float(parent_config["policy"]["budget"]), + purpose_cap=float(parent_config["policy"]["max_concentration_by_purpose"]), + time_limit=int(parent_config["execution"]["solver_time_limit_seconds"]), + threads=int(threads), + ) + + +def _period_frames(base: pd.DataFrame, role: str) -> tuple[tuple[str, pd.DataFrame], ...]: + frame = base.loc[base["design_split"].eq(role)].copy() + periods = frame["issue_d"].dt.to_period("M") + return tuple( + (str(period), frame.loc[periods.eq(period)].copy()) for period in sorted(periods.unique()) + ) + + +def _run_family_audit( + base: pd.DataFrame, + recipes: Mapping[str, Any], + parent_records: pd.DataFrame, + *, + config: Mapping[str, Any], + parent_config: Mapping[str, Any], +) -> tuple[pd.DataFrame, dict[str, float]]: + family = config["family_audit"] + tolerances = [float(value) for value in family["risk_tolerances"]] + gammas = [float(value) for value in family["gamma_grid"]] + cap_tolerance = float(family["cap_classification_tolerance"]) + threads = int(config["tie_audit"]["threads"]) + budget = float(parent_config["policy"]["budget"]) + point = base["pd_point"].to_numpy(dtype=float) + rows: list[dict[str, Any]] = [] + windows = recipes["catboost_platt"] + for window_index, (window_id, group_recipes) in enumerate(sorted(windows.items()), start=1): + logger.info("Family domain: window {}/8 {}", window_index, window_id) + _, _, upper = apply_binary_outcome_recipe(point, group_recipes[5]) + for gamma in gammas: + effective = point + gamma * (upper - point) + scored = base.assign(pd_effective=effective, conformal_upper=upper) + for role in ("policy_development", "primary_oot"): + for period, month in _period_frames(scored, role): + q = month["pd_effective"].to_numpy(dtype=float) + p = month["pd_point"].to_numpy(dtype=float) + rates = month["contractual_rate"].to_numpy(dtype=float) + objective = expected_objective_coefficients( + p, rates, lgd=float(parent_config["payoff"]["lgd"]) + ) + minimum_session = _portfolio_session( + month, + score=q, + objective=-q, + parent_config=parent_config, + threads=threads, + ) + minimum = minimum_session.solve(1.0) + session = _portfolio_session( + month, + score=q, + objective=objective, + parent_config=parent_config, + threads=threads, + ) + unconstrained = session.solve(1.0) + q_min = float(minimum.weighted_point_score) + q_obj = float(unconstrained.weighted_point_score) + score_range = q_obj - q_min + if score_range < -cap_tolerance: + raise RuntimeError("Effective-score feasible range has reversed endpoints.") + for tau in tolerances: + classification = classify_cap( + tau, + minimum_feasible_score=q_min, + unconstrained_objective_score=q_obj, + tolerance=cap_tolerance, + ) + solution = None if classification == "infeasible" else session.solve(tau) + realized_score = ( + float("nan") + if solution is None + else float(solution.weighted_point_score) + ) + rows.append( + { + "window_id": window_id, + "role": role, + "period": period, + "gamma": gamma, + "risk_tolerance": tau, + "candidate_rows": int(len(month)), + "minimum_feasible_score": q_min, + "unconstrained_objective_score": q_obj, + "decision_score_range": score_range, + "cap_classification": classification, + "weighted_effective_score": realized_score, + "cap_slack": ( + float("nan") if solution is None else tau - realized_score + ), + "cap_binding": bool( + solution is not None + and abs(tau - realized_score) <= cap_tolerance + ), + "total_allocated": ( + float("nan") + if solution is None + else float(solution.total_allocated) + ), + "budget_residual": ( + float("nan") + if solution is None + else float(solution.total_allocated - budget) + ), + "expected_objective": ( + float("nan") + if solution is None + else float(solution.objective_value) + ), + "effective_minus_point_mean": float(np.mean(q - p)), + "upper_saturated_share": float( + np.mean( + month["conformal_upper"].to_numpy(dtype=float) + >= 1.0 - 1e-12 + ) + ), + } + ) + result = pd.DataFrame(rows) + expected_rows = 8 * (11 + 15) * 5 * 3 + if len(result) != expected_rows: + raise RuntimeError(f"Policy-family audit produced {len(result)} rows, not {expected_rows}.") + + parent = parent_records.loc[ + parent_records["comparator_rule"].eq("guardrail") + & parent_records["role"].isin(family["roles"]), + [ + "window_id", + "role", + "period", + "gamma", + "risk_tolerance", + "weighted_pd_effective", + "expected_objective", + ], + ].rename( + columns={ + "weighted_pd_effective": "parent_weighted_effective_score", + "expected_objective": "parent_expected_objective", + } + ) + merged = result.merge( + parent, + on=["window_id", "role", "period", "gamma", "risk_tolerance"], + how="left", + validate="one_to_one", + ) + interior = merged["gamma"].isin(family["inherited_interior_gammas"]) + if int(interior.sum()) != 1_872 or bool( + merged.loc[interior, "parent_expected_objective"].isna().any() + ): + raise RuntimeError("Inherited policy-family reconstruction lost a frozen cell.") + merged["parent_score_difference"] = ( + merged["weighted_effective_score"] - merged["parent_weighted_effective_score"] + ) + merged["parent_objective_difference"] = ( + merged["expected_objective"] - merged["parent_expected_objective"] + ) + score_difference = float(merged.loc[interior, "parent_score_difference"].abs().max()) + objective_difference = float(merged.loc[interior, "parent_objective_difference"].abs().max()) + if score_difference > float(family["reconstruction_score_tolerance"]): + raise RuntimeError(f"Frozen funded-score reconstruction drifted by {score_difference}.") + if objective_difference > float(family["reconstruction_objective_tolerance"]): + raise RuntimeError(f"Frozen objective reconstruction drifted by {objective_difference}.") + return merged, { + "maximum_absolute_parent_score_difference": score_difference, + "maximum_absolute_parent_objective_difference": objective_difference, + } + + +def _run_point_cap_audit( + base: pd.DataFrame, + solve_records: pd.DataFrame, + comparator_support: pd.DataFrame, + frontier: pd.DataFrame, + *, + config: Mapping[str, Any], + parent_config: Mapping[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame]: + tie = config["tie_audit"] + support = config["comparator_support"] + primary = _period_frames(base, "primary_oot") + periods = [period for period, _ in primary] + broad_values = [float(value) for value in support["broad_stress"]] + if len(broad_values) != 2: + raise ValueError("Broad comparator stress support requires two endpoints.") + census = build_cap_census( + solve_records, + comparator_support, + frontier, + periods=periods, + broad_support=(broad_values[0], broad_values[1]), + tolerance=float(support["cap_deduplication_tolerance"]), + ) + rows: list[dict[str, Any]] = [] + sensitivity_inputs: dict[str, list[dict[str, Any]]] = {} + for period_index, (period, month) in enumerate(primary, start=1): + logger.info("Point-cap basis census: month {}/15 {}", period_index, period) + point = month["pd_point"].to_numpy(dtype=float) + rates = month["contractual_rate"].to_numpy(dtype=float) + objective = expected_objective_coefficients( + point, rates, lgd=float(parent_config["payoff"]["lgd"]) + ) + minimum_session = _portfolio_session( + month, + score=point, + objective=-point, + parent_config=parent_config, + threads=int(tie["threads"]), + ) + point_minimum = minimum_session.solve(1.0).weighted_point_score + session = _portfolio_session( + month, + score=point, + objective=objective, + parent_config=parent_config, + threads=int(tie["threads"]), + ) + point_objective = session.solve(1.0).weighted_point_score + period_caps = census.loc[census["period"].eq(period)] + for cap_row in period_caps.itertuples(index=False): + cap = float(cap_row.point_cap) + classification = classify_cap( + cap, + minimum_feasible_score=float(point_minimum), + unconstrained_objective_score=float(point_objective), + tolerance=float(config["family_audit"]["cap_classification_tolerance"]), + ) + if classification == "infeasible": + raise RuntimeError(f"Declared point support is infeasible in {period}: {cap}.") + solution = session.solve(cap) + diagnostics = point_basis_diagnostics( + session, + solution, + dual_tolerance=float(tie["dual_tolerance"]), + primal_tolerance=float(tie["primal_tolerance"]), + ) + if abs(float(diagnostics["objective_reconciliation_error"])) > float( + tie["objective_reconciliation_tolerance"] + ): + raise RuntimeError(f"Point objective failed reconciliation at {period}/{cap}.") + row = { + **cap_row._asdict(), + "cap_classification": classification, + "minimum_feasible_point_score": float(point_minimum), + "unconstrained_objective_point_score": float(point_objective), + "weighted_point_score": float(solution.weighted_point_score), + "point_cap_slack": float(cap - solution.weighted_point_score), + "cap_binding": bool( + abs(cap - solution.weighted_point_score) + <= float(config["family_audit"]["cap_classification_tolerance"]) + ), + "expected_objective": float(solution.objective_value), + "basis_cap_lower": float(solution.basis_cap_lower), + "basis_cap_upper": float(solution.basis_cap_upper), + **diagnostics, + } + rows.append(row) + if bool(diagnostics["basis_primal_degenerate"]) or int( + diagnostics["near_zero_nonbasic_reduced_costs"] + ): + sensitivity_inputs.setdefault(period, []).append( + { + "point_cap": cap, + "base_exposure": solution.exposure.copy(), + "base_objective": float(solution.objective_value), + "base_weighted_point": float(solution.weighted_point_score), + } + ) + diagnostics_frame = pd.DataFrame(rows) + if len(diagnostics_frame) != len(census): + raise RuntimeError("Point-cap diagnostics lost a census row.") + + sensitivity_rows: list[dict[str, Any]] = [] + primary_by_period = dict(primary) + for period, items in sorted(sensitivity_inputs.items()): + month = primary_by_period[period] + original_ids = month["id"].astype("string").to_numpy() + reversed_month = month.sort_values("id", ascending=False, kind="mergesort").copy() + reversed_point = reversed_month["pd_point"].to_numpy(dtype=float) + reversed_objective = expected_objective_coefficients( + reversed_point, + reversed_month["contractual_rate"].to_numpy(dtype=float), + lgd=float(parent_config["payoff"]["lgd"]), + ) + reversed_session = _portfolio_session( + reversed_month, + score=reversed_point, + objective=reversed_objective, + parent_config=parent_config, + threads=int(tie["threads"]), + ) + for item in items: + reversed_solution = reversed_session.solve(float(item["point_cap"])) + reversed_exposure = pd.Series( + reversed_solution.exposure, + index=reversed_month["id"].astype("string"), + ).reindex(original_ids) + if bool(reversed_exposure.isna().any()): + raise RuntimeError("Reordered tie audit lost a candidate ID.") + base_exposure = np.asarray(item["base_exposure"], dtype=float) + distance = float( + np.abs(base_exposure - reversed_exposure.to_numpy(dtype=float)).sum() + / (base_exposure.sum() + reversed_solution.exposure.sum()) + ) + objective_difference = float( + reversed_solution.objective_value - float(item["base_objective"]) + ) + if abs(objective_difference) > float(tie["objective_reconciliation_tolerance"]): + raise RuntimeError( + f"Column-order objective instability at {period}/{item['point_cap']}." + ) + sensitivity_rows.append( + { + "period": period, + "point_cap": float(item["point_cap"]), + "allocation_distance": distance, + "objective_difference": objective_difference, + "weighted_point_difference": float( + reversed_solution.weighted_point_score - float(item["base_weighted_point"]) + ), + "tie_sensitive": bool(distance > float(tie["order_sensitivity_tolerance"])), + } + ) + sensitivity = pd.DataFrame(sensitivity_rows).reindex( + columns=[ + "period", + "point_cap", + "allocation_distance", + "objective_difference", + "weighted_point_difference", + "tie_sensitive", + ], + ) + return diagnostics_frame, sensitivity + + +def _records(frame: pd.DataFrame) -> list[dict[str, Any]]: + payload = frame.to_json(orient="records", double_precision=15) + if payload is None: + raise RuntimeError("Pandas did not serialize policy-support records.") + return json.loads(payload) + + +def _compact_summary( + family: pd.DataFrame, + reconstruction: Mapping[str, float], + diagnostics: pd.DataFrame, + sensitivity: pd.DataFrame, +) -> dict[str, Any]: + family_count_series = family.groupby( + ["role", "gamma", "risk_tolerance", "cap_classification"], + observed=True, + sort=True, + ).size() + family_counts = family_count_series.to_frame(name="cells").reset_index() + source_columns = [column for column in diagnostics if column.startswith("is_")] + support_rows = [] + for column in source_columns: + selected = diagnostics.loc[diagnostics[column]] + support_rows.append( + { + "source": column.removeprefix("is_"), + "cap_month_rows": int(len(selected)), + "decision_active": int(selected["cap_classification"].eq("decision_active").sum()), + "objective_slack": int(selected["cap_classification"].eq("objective_slack").sum()), + "near_zero_bases": int(selected["near_zero_nonbasic_reduced_costs"].gt(0).sum()), + "primal_degenerate_bases": int(selected["basis_primal_degenerate"].sum()), + } + ) + inherited = family["gamma"].isin([0.25, 0.5, 0.75]) + gamma_one = family["gamma"].eq(1.0) + return { + "family": { + "rows": int(len(family)), + "inherited_rows": int(inherited.sum()), + "inherited_infeasible": int( + family.loc[inherited, "cap_classification"].eq("infeasible").sum() + ), + "inherited_decision_active": int( + family.loc[inherited, "cap_classification"].eq("decision_active").sum() + ), + "gamma_one_infeasible": int( + family.loc[gamma_one, "cap_classification"].eq("infeasible").sum() + ), + "gamma_one_decision_active": int( + family.loc[gamma_one, "cap_classification"].eq("decision_active").sum() + ), + "gamma_one_objective_slack": int( + family.loc[gamma_one, "cap_classification"].eq("objective_slack").sum() + ), + "classification_counts": _records(family_counts), + **dict(reconstruction), + }, + "point_cap_census": { + "rows": int(len(diagnostics)), + "periods": int(diagnostics["period"].nunique()), + "named_unique_cap_months": int( + diagnostics[ + [ + "is_named_c0", + "is_named_c1", + "is_named_c2", + ] + ] + .any(axis=1) + .sum() + ), + "minimum_absolute_nonbasic_reduced_cost": float( + diagnostics["minimum_absolute_nonbasic_reduced_cost"].min() + ), + "near_zero_bases": int(diagnostics["near_zero_nonbasic_reduced_costs"].gt(0).sum()), + "primal_degenerate_bases": int(diagnostics["basis_primal_degenerate"].sum()), + "maximum_dual_sign_violation": float(diagnostics["maximum_dual_sign_violation"].max()), + "maximum_objective_reconciliation_error": float( + diagnostics["objective_reconciliation_error"].abs().max() + ), + "support_sources": support_rows, + }, + "order_sensitivity": { + "triggered_rows": int(len(sensitivity)), + "tie_sensitive_rows": int( + sensitivity["tie_sensitive"].sum() if not sensitivity.empty else 0 + ), + "maximum_allocation_distance": float( + sensitivity["allocation_distance"].max() if not sensitivity.empty else 0.0 + ), + "maximum_absolute_objective_difference": float( + sensitivity["objective_difference"].abs().max() if not sensitivity.empty else 0.0 + ), + }, + } + + +def run_audit(*, config_path: Path, repo_root: Path = ROOT) -> Path: + """Execute the tagged audit and return its deterministic summary path.""" + started_at = utc_now_iso() + started_counter = time.perf_counter() + config_path = resolve_repo_input(config_path, repo_root=repo_root) + config = load_config(config_path) + protocol_commit = require_clean_tagged_head(repo_root, str(config["protocol_tag"])) + initial_git = git_provenance(repo_root) + implementation_start = implementation_provenance( + config_path=config_path, + relative_paths=IMPLEMENTATION_PATHS, + repo_root=repo_root, + ) + parent_paths, parent_freeze = _verified_parent_paths(config, repo_root=repo_root) + parent_config_path = resolve_repo_input(config["parent"]["config"], repo_root=repo_root) + parent_config = load_v4_config(parent_config_path) + raw_path = resolve_repo_input(config["source_ingest"]["raw_path"], repo_root=repo_root) + base = _load_decision_base( + scores_path=parent_paths["scores"], + raw_path=raw_path, + config=config, + ) + recipes = load_recipes(parent_paths["recipes"]) + solve_records = pd.read_parquet(parent_paths["solve_records"]) + comparator_support = pd.read_parquet(parent_paths["comparator_support"]) + frontier = pd.read_parquet(parent_paths["frontier_breakpoints"]) + paths = prepare_output_paths(config, repo_root=repo_root) + protocol_freeze = atomic_write_json( + paths.model_dir / "protocol_freeze.json", + { + "schema_version": str(config["schema_version"]), + "status": "outcome_free_policy_support_tie_audit_frozen", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "claim_boundary": dict(config["claim_boundary"]), + "outcome_columns_passed": [], + "parent_protocol_freeze": config["parent"]["protocol_freeze"], + "parent_outcome_free_artifacts": parent_freeze["outcome_free_artifacts"], + "raw_source": relative_artifact_descriptor(raw_path, repo_root=repo_root), + "implementation_provenance": implementation_start, + "protected_stages_run": [], + "protected_artifacts_written": [], + }, + ) + + family, reconstruction = _run_family_audit( + base, + recipes, + solve_records, + config=config, + parent_config=parent_config, + ) + diagnostics, sensitivity = _run_point_cap_audit( + base, + solve_records, + comparator_support, + frontier, + config=config, + parent_config=parent_config, + ) + output = config["output"] + frames = { + str(output["family_feasibility"]): family, + str(output["point_cap_diagnostics"]): diagnostics, + str(output["order_sensitivity"]): sensitivity, + } + written: dict[Path, pd.DataFrame] = {} + for filename, frame in frames.items(): + path = atomic_write_parquet(frame, paths.data_dir / filename, index=False) + written[path] = frame + implementation_end = implementation_provenance( + config_path=config_path, + relative_paths=IMPLEMENTATION_PATHS, + repo_root=repo_root, + ) + if implementation_end != implementation_start: + raise RuntimeError("Policy-support implementation changed during execution.") + artifacts = { + descriptor["path"]: descriptor + for descriptor in [ + relative_artifact_descriptor(protocol_freeze, repo_root=repo_root), + *(relative_artifact_descriptor(path, repo_root=repo_root) for path in written), + ] + } + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "hypothesis": str(config["hypothesis"]), + "claim_boundary": ( + "Outcome-free structural evidence only. No empirical metric, direction, " + "policy promotion, universal comparator support, or selected-set claim." + ), + "outcome_columns_passed": [], + "results": _compact_summary(family, reconstruction, diagnostics, sensitivity), + "parent_protocol_freeze": config["parent"]["protocol_freeze"], + "raw_source": relative_artifact_descriptor(raw_path, repo_root=repo_root), + "implementation_provenance": implementation_start, + "artifacts": artifacts, + "schemas": { + relative_artifact_descriptor(path, repo_root=repo_root)["path"]: dataframe_schema(frame) + for path, frame in written.items() + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + summary_path = atomic_write_json(paths.model_dir / str(output["deterministic_result"]), summary) + receipt = { + "run_tag": str(config["run_tag"]), + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "runtime_seconds": float(time.perf_counter() - started_counter), + "initial_git": initial_git, + "final_git": git_provenance(repo_root), + "environment": environment_provenance(repo_root), + "deterministic_summary": relative_artifact_descriptor(summary_path, repo_root=repo_root), + } + atomic_write_json(paths.model_dir / str(output["execution_receipt"]), receipt) + logger.info("Policy-support and tie audit complete: {}", summary_path) + return summary_path + + +def main(argv: Sequence[str] | None = None) -> None: + """Run the CLI entry point.""" + args = parse_args(argv) + run_audit(config_path=args.config) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_portfolio_structure_sensitivity.py b/scripts/experiments/run_ijds_portfolio_structure_sensitivity.py new file mode 100644 index 0000000..b2e8064 --- /dev/null +++ b/scripts/experiments/run_ijds_portfolio_structure_sensitivity.py @@ -0,0 +1,908 @@ +"""Freeze and evaluate the complete IJDS portfolio-structure sensitivity grid.""" + +from __future__ import annotations + +import argparse +import copy +import json +import multiprocessing as mp +import time +from collections.abc import Mapping, Sequence +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.endpoint_sensitivity import rebuild_archive_outcomes +from src.ijds_audit.evaluation import evaluate_frozen_portfolios +from src.ijds_audit.protocol import load_outcome_universe, load_recipes +from src.ijds_audit.structural_checkpoint import ( + StructuralShardInspection, + hardlink_structural_shard, + inspect_structural_shard, +) +from src.ijds_audit.structural_sensitivity import ( + allocation_activity, + declared_scenarios, + scenario_result_summary, +) +from src.ijds_challengers.archive import ( + load_outcome_free_decision_base, + verified_parent_artifacts, +) +from src.ijds_challengers.config import load_frontier_config +from src.ijds_challengers.evaluation import build_endpoint_contrasts, build_metric_directions +from src.ijds_challengers.evaluation_config import load_v2_config +from src.ijds_challengers.normalized_frontier import build_outcome_free_frontiers +from src.utils.isolated_experiment import ( + OutputPaths, + dataframe_schema, + environment_provenance, + git_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_git_tag, + resolve_isolated_run_dir, + resolve_repo_input, + sha256_file, +) +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet, utc_now_iso + +ROOT = Path(__file__).resolve().parents[2] +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +ARTIFACT_NAMES = ( + "solve_records", + "allocations", + "endpoint_diagnostics", + "minimum_endpoint_diagnostics", + "objective_optimum_diagnostics", + "order_sensitivity", + "independent_validation", +) +_FREEZE_WORKER_STATE: dict[str, Any] | None = None + +STATUS_V1 = "locked_retrospective_outcome_free_structural_sensitivity" +STATUS_V2 = "locked_retrospective_outcome_free_structural_sensitivity_v2" +STATUS_V3 = "locked_retrospective_outcome_free_structural_sensitivity_v3_parallel_execution" +STATUS_V4 = "locked_retrospective_outcome_free_structural_sensitivity_v4_interruption_recovery" +STATUS_V5 = "locked_retrospective_outcome_free_structural_sensitivity_v5_retry_ladder" +STATUS_V6 = "locked_retrospective_outcome_free_structural_sensitivity_v6_order_tolerance" +ALLOWED_STATUSES = {STATUS_V1, STATUS_V2, STATUS_V3, STATUS_V4, STATUS_V5, STATUS_V6} + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the structural-sensitivity CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--phase", choices=("freeze", "evaluate"), required=True) + return parser.parse_args(argv) + + +def _require_retry_contract( + payload: Mapping[str, Any], + *, + version: str, + slacks: Sequence[float], + scope: str, +) -> None: + numerics = payload.get("numerics", {}) + configured = numerics.get("minimum_endpoint_retry_slacks") + if configured is None: + configured = [numerics.get("minimum_endpoint_retry_slack", 0.0)] + if [float(value) for value in configured] != list(slacks): + raise ValueError(f"{version} minimum-endpoint retry contract changed.") + if numerics.get("retry_scope") != scope: + raise ValueError(f"{version} minimum-endpoint retry scope changed.") + + +def _require_recovery_contract( + payload: Mapping[str, Any], + scenarios: Sequence[Mapping[str, Any]], + *, + version: str, + workers: int, + recovered: int, + missing_ids: Sequence[str] | None = None, +) -> None: + execution = payload.get("execution", {}) + recovery = execution.get("recovery", {}) + missing = recovery.get("expected_missing_scenario_ids", []) + expected_missing = len(scenarios) - recovered + if int(execution.get("freeze_workers", 0)) != workers: + raise ValueError(f"{version} freeze-worker contract changed.") + if not isinstance(missing, list) or len(missing) != expected_missing: + raise ValueError(f"{version} recovery complement size changed.") + declared_ids = {str(item["scenario_id"]) for item in scenarios} + actual_missing = {str(value) for value in missing} + if not actual_missing.issubset(declared_ids): + raise ValueError(f"{version} recovery scenario identities changed.") + if missing_ids is not None and actual_missing != set(missing_ids): + raise ValueError(f"{version} locked missing scenario changed.") + if int(recovery.get("expected_recovered_scenarios", -1)) != recovered: + raise ValueError(f"{version} recovered-scenario count changed.") + + +def _require_v6_numerics(payload: Mapping[str, Any]) -> None: + numerics = payload.get("numerics", {}) + if float(numerics.get("order_exposure_distance_tolerance", -1.0)) != 1.0e-8: + raise ValueError("V6 order-exposure tolerance must remain 1e-8.") + if float(numerics.get("order_objective_tolerance_dollars", -1.0)) != 1.0e-5: + raise ValueError("V6 must retain the parent 1e-5 objective tolerance.") + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Structural-sensitivity config must be a mapping.") + status = payload.get("protocol_status") + if status not in ALLOWED_STATUSES: + raise ValueError("Structural-sensitivity protocol is not locked.") + scenarios = declared_scenarios(payload) + if int(payload["structural_grid"].get("scenarios", -1)) != len(scenarios): + raise ValueError("Declared structural scenario count changed.") + boundary = payload.get("claim_boundary", {}) + required_false = { + "preregistered", + "confirmatory", + "prospective", + "outcome_based_scenario_selection", + "model_refit", + "conformal_refit", + "ruler_selection", + "scenario_selection", + "policy_winner", + } + if any(boundary.get(field) is not False for field in required_false): + raise ValueError("Structural-sensitivity claim boundary changed.") + if status in {STATUS_V2, STATUS_V3, STATUS_V4}: + _require_retry_contract( + payload, + version="V2--V4", + slacks=(1.0e-12,), + scope="known_exact_minimum_boundary_failure_only", + ) + if status == STATUS_V3 and int(payload.get("execution", {}).get("freeze_workers", 0)) != 10: + raise ValueError("V3 must retain ten deterministic freeze workers.") + if status == STATUS_V4: + _require_recovery_contract(payload, scenarios, version="V4", workers=7, recovered=29) + if status in {STATUS_V5, STATUS_V6}: + _require_retry_contract( + payload, + version="V5--V6", + slacks=(1.0e-12, 1.0e-10), + scope="closed_known_boundary_status_ladder", + ) + if status == STATUS_V5: + _require_recovery_contract(payload, scenarios, version="V5", workers=3, recovered=33) + if status == STATUS_V6: + _require_recovery_contract( + payload, + scenarios, + version="V6", + workers=1, + recovered=35, + missing_ids=("b0500k_p020_l025",), + ) + _require_v6_numerics(payload) + return payload + + +def _minimum_endpoint_retry_slacks(config: Mapping[str, Any]) -> tuple[float, ...]: + numerics = config.get("numerics", {}) + configured = numerics.get("minimum_endpoint_retry_slacks") + if configured is not None: + return tuple(float(value) for value in configured) + single = float(numerics.get("minimum_endpoint_retry_slack", 0.0)) + return () if single == 0.0 else (single,) + + +def _protocol_documents(config: Mapping[str, Any]) -> tuple[Path, ...]: + documents = config.get( + "protocol_documents", + [ + config.get( + "protocol_document", + "docs/research/ijds_portfolio_structure_sensitivity_protocol_2026-07-14.md", + ) + ], + ) + if not isinstance(documents, list) or not documents: + raise TypeError("Structural protocol documents must be a nonempty list.") + return tuple(Path(str(document)) for document in documents) + + +def _run_paths(config: Mapping[str, Any], *, repo_root: Path) -> OutputPaths: + output = config["output"] + run_tag = str(config["run_tag"]) + return OutputPaths( + data_dir=resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=run_tag, + ), + model_dir=resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=run_tag, + ), + ) + + +def _frontier_config(base: Mapping[str, Any], structural: Mapping[str, Any]) -> dict[str, Any]: + config = copy.deepcopy(dict(base)) + grid = structural["structural_grid"] + config["frontier"]["roles"] = [str(value) for value in grid["roles"]] + config["frontier"]["gamma_grid"] = [float(value) for value in grid["state_gamma_grid"]] + config["frontier"]["reported_gamma_grid"] = [ + float(value) for value in grid["reported_gamma_grid"] + ] + config["frontier"]["coordinate_grid"] = [float(value) for value in grid["coordinates"]] + config["frontier"]["normalized_score"]["minimum_endpoint_retry_slacks"] = list( + _minimum_endpoint_retry_slacks(structural) + ) + numerics = structural.get("numerics", {}) + if "order_exposure_distance_tolerance" in numerics: + config["solver"]["order_exposure_distance_tolerance"] = float( + numerics["order_exposure_distance_tolerance"] + ) + if "order_objective_tolerance_dollars" in numerics: + config["solver"]["order_objective_tolerance_dollars"] = float( + numerics["order_objective_tolerance_dollars"] + ) + return config + + +def _scenario_parent(base: Mapping[str, Any], scenario: Mapping[str, Any]) -> dict[str, Any]: + config = copy.deepcopy(dict(base)) + config["policy"]["budget"] = float(scenario["budget"]) + config["policy"]["max_concentration_by_purpose"] = float(scenario["purpose_cap"]) + config["payoff"]["lgd"] = float(scenario["lgd"]) + return config + + +def _tag(frame: pd.DataFrame, scenario: Mapping[str, Any]) -> pd.DataFrame: + return frame.assign( + scenario_id=str(scenario["scenario_id"]), + scenario_budget=float(scenario["budget"]), + scenario_purpose_cap=float(scenario["purpose_cap"]), + scenario_lgd=float(scenario["lgd"]), + scenario_is_baseline=bool(scenario["is_baseline"]), + ) + + +def _write_build( + build: Any, + *, + data_dir: Path, + scenario: Mapping[str, Any], + repo_root: Path, +) -> dict[str, Any]: + scenario_dir = data_dir / "scenarios" / str(scenario["scenario_id"]) / "frontier" + frames = { + "solve_records": build.solve_records, + "allocations": build.allocations, + "endpoint_diagnostics": build.endpoint_diagnostics, + "minimum_endpoint_diagnostics": build.minimum_endpoint_diagnostics, + "objective_optimum_diagnostics": build.objective_optimum_diagnostics, + "order_sensitivity": build.order_sensitivity, + "independent_validation": build.independent_validation, + } + descriptors: dict[str, Any] = {} + for name, frame in frames.items(): + path = atomic_write_parquet(_tag(frame, scenario), scenario_dir / f"{name}.parquet") + descriptors[name] = relative_artifact_descriptor(path, repo_root=repo_root) + return descriptors + + +def _initialize_freeze_worker( + config_path: str, + repo_root: str, + decision_base_path: str, +) -> None: + """Load immutable outcome-free inputs once in each spawned worker.""" + global _FREEZE_WORKER_STATE + root = Path(repo_root).resolve() + config = _load_config(Path(config_path)) + parent = config["parent"] + source_frontier_config = load_frontier_config( + resolve_repo_input(str(parent["frontier_config"]), repo_root=root) + ) + parent_paths, _ = verified_parent_artifacts(source_frontier_config, repo_root=root) + v4_config = load_v4_config(resolve_repo_input(str(parent["v4_config"]), repo_root=root)) + _FREEZE_WORKER_STATE = { + "root": root, + "config": config, + "v4_config": v4_config, + "build_config": _frontier_config(source_frontier_config, config), + "base": pd.read_parquet(Path(decision_base_path)), + "recipes": load_recipes(parent_paths["recipes"]), + "paths": _run_paths(config, repo_root=root), + } + + +def _freeze_scenario_worker(scenario: Mapping[str, Any]) -> dict[str, Any]: + """Build one complete scenario in a directory no other worker can write.""" + state = _FREEZE_WORKER_STATE + if state is None: + raise RuntimeError("Structural freeze worker was not initialized.") + build = build_outcome_free_frontiers( + state["base"], + state["recipes"], + config=state["build_config"], + parent_config=_scenario_parent(state["v4_config"], scenario), + ) + diagnostics = build.minimum_endpoint_diagnostics + artifacts = _write_build( + build, + data_dir=state["paths"].data_dir, + scenario=scenario, + repo_root=state["root"], + ) + return { + "scenario_id": str(scenario["scenario_id"]), + "artifacts": artifacts, + "counts": { + **dict(scenario), + "solve_records": int(len(build.solve_records)), + "funded_rows": int(len(build.allocations)), + "endpoint_cells": int(len(build.endpoint_diagnostics)), + "minimum_endpoint_cells": int(len(diagnostics)), + "minimum_endpoint_retries": int(diagnostics["minimum_endpoint_retried"].sum()), + "maximum_minimum_endpoint_retry_slack": float( + diagnostics["minimum_endpoint_retry_slack"].max() + ), + "maximum_minimum_cap_residual": float(diagnostics["minimum_cap_residual"].abs().max()), + "outcome_columns_passed": [], + }, + } + + +def _inspection_counts( + inspection: StructuralShardInspection, + *, + scenario: Mapping[str, Any], +) -> dict[str, Any]: + return { + **dict(scenario), + "solve_records": int(inspection.rows["solve_records"]), + "funded_rows": int(inspection.rows["allocations"]), + "endpoint_cells": int(inspection.rows["endpoint_diagnostics"]), + "minimum_endpoint_cells": int(inspection.rows["minimum_endpoint_diagnostics"]), + "minimum_endpoint_retries": int(inspection.minimum_endpoint_retries), + "maximum_minimum_endpoint_retry_slack": float(inspection.maximum_retry_slack), + "maximum_minimum_cap_residual": float(inspection.maximum_cap_residual), + "outcome_columns_passed": [], + } + + +def _recover_interrupted_scenarios( + config: Mapping[str, Any], + *, + paths: OutputPaths, + repo_root: Path, +) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: + recovery = config.get("execution", {}).get("recovery") + if not isinstance(recovery, dict): + return {}, [], {"enabled": False} + source_tag = str(recovery["source_run_tag"]) + source_protocol_tag = str(recovery["source_protocol_tag"]) + source_protocol_commit = str(recovery["source_protocol_commit"]) + if resolve_git_tag(repo_root, source_protocol_tag) != source_protocol_commit: + raise RuntimeError( + "Structural recovery source tag no longer resolves to its locked commit." + ) + source_root = resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(config["output"]["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=source_tag, + ) + scenarios = declared_scenarios(config) + lookup = {str(item["scenario_id"]): item for item in scenarios} + missing = {str(value) for value in recovery["expected_missing_scenario_ids"]} + recovered = sorted(set(lookup).difference(missing)) + if len(recovered) != int(recovery["expected_recovered_scenarios"]): + raise RuntimeError("Structural recovery complement has the wrong scenario count.") + physical_scenarios = { + path.name for path in (source_root / "scenarios").iterdir() if path.is_dir() + } + if physical_scenarios != set(recovered): + raise RuntimeError( + "Structural recovery source directories differ from the locked complement." + ) + retry_slacks = _minimum_endpoint_retry_slacks(config) + cap_tolerance = float(config["numerics"]["cap_residual_tolerance"]) + artifacts: dict[str, Any] = {} + counts: list[dict[str, Any]] = [] + source_descriptors: dict[str, Any] = {} + for scenario_id in recovered: + inspection = inspect_structural_shard( + source_root / "scenarios" / scenario_id, + scenario_id=scenario_id, + retry_slacks=retry_slacks, + cap_residual_tolerance=cap_tolerance, + ) + linked = hardlink_structural_shard( + inspection, + destination_root=paths.data_dir / "scenarios" / scenario_id, + ) + artifacts[scenario_id] = {} + counts.append(_inspection_counts(inspection, scenario=lookup[scenario_id])) + source_descriptors[scenario_id] = {} + for name, source_path in inspection.paths.items(): + descriptor = relative_artifact_descriptor(source_path, repo_root=repo_root) + destination_path = linked[name].resolve().relative_to(repo_root.resolve()).as_posix() + artifacts[scenario_id][name] = { + "path": destination_path, + "bytes": descriptor["bytes"], + "sha256": descriptor["sha256"], + } + source_descriptors[scenario_id][name] = { + "original_path": descriptor["path"], + "bytes": descriptor["bytes"], + "sha256": descriptor["sha256"], + } + return ( + artifacts, + counts, + { + "enabled": True, + "source_run_tag": source_tag, + "source_protocol_tag": source_protocol_tag, + "source_protocol_commit": source_protocol_commit, + "recovered_scenarios": len(recovered), + "recomputed_scenarios": len(missing), + "missing_scenario_ids": sorted(missing), + "source_artifacts": source_descriptors, + "copy_method": "ntfs_hardlink_after_full_shard_validation", + }, + ) + + +def freeze(*, config_path: Path, repo_root: Path = ROOT) -> Path: + """Build and hash-freeze every structural allocation before outcomes.""" + started = time.perf_counter() + started_at = utc_now_iso() + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + parent = config["parent"] + raw_path = resolve_repo_input(str(parent["raw_path"]), repo_root=root) + if sha256_file(raw_path) != str(parent["raw_sha256"]): + raise RuntimeError("Raw archive hash changed before structural freeze.") + source_frontier_config = load_frontier_config( + resolve_repo_input(str(parent["frontier_config"]), repo_root=root) + ) + parent_paths, parent_freeze = verified_parent_artifacts(source_frontier_config, repo_root=root) + load_v4_config(resolve_repo_input(str(parent["v4_config"]), repo_root=root)) + decision_base = load_outcome_free_decision_base( + scores_path=parent_paths["scores"], + raw_path=raw_path, + config=source_frontier_config, + ) + decision_base_path = atomic_write_parquet( + decision_base, + paths.data_dir / "frontier/outcome_free_decision_base.parquet", + ) + del decision_base + scenarios = declared_scenarios(config) + scenario_lookup = {str(item["scenario_id"]): item for item in scenarios} + scenario_artifacts, scenario_counts, recovery_audit = _recover_interrupted_scenarios( + config, + paths=paths, + repo_root=root, + ) + pending = [ + scenario for scenario in scenarios if str(scenario["scenario_id"]) not in scenario_artifacts + ] + workers = int(config.get("execution", {}).get("freeze_workers", 1)) + logger.info( + "Structural freeze recovered {} scenarios and dispatching {} to {} workers", + len(scenario_artifacts), + len(pending), + workers, + ) + with ProcessPoolExecutor( + max_workers=workers, + mp_context=mp.get_context("spawn"), + initializer=_initialize_freeze_worker, + initargs=(str(resolved_config), str(root), str(decision_base_path)), + ) as executor: + futures = { + executor.submit(_freeze_scenario_worker, scenario): str(scenario["scenario_id"]) + for scenario in pending + } + for index, future in enumerate(as_completed(futures), start=1): + scenario_id = futures[future] + result = future.result() + if result["scenario_id"] != scenario_id: + raise RuntimeError("Structural worker returned the wrong scenario identity.") + inspection = inspect_structural_shard( + paths.data_dir / "scenarios" / scenario_id, + scenario_id=scenario_id, + retry_slacks=_minimum_endpoint_retry_slacks(config), + cap_residual_tolerance=float(config["numerics"]["cap_residual_tolerance"]), + ) + scenario_artifacts[scenario_id] = { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in inspection.paths.items() + } + if scenario_artifacts[scenario_id] != result["artifacts"]: + raise RuntimeError(f"Worker descriptors failed shard inspection for {scenario_id}.") + verified_counts = _inspection_counts( + inspection, + scenario=scenario_lookup[scenario_id], + ) + if verified_counts != result["counts"]: + raise RuntimeError(f"Worker counts failed shard inspection for {scenario_id}.") + scenario_counts.append(verified_counts) + logger.info( + "Structural freeze completed new scenario {}/{}: {}", + index, + len(pending), + scenario_id, + ) + expected_scenario_ids = set(scenario_lookup) + if set(scenario_artifacts) != expected_scenario_ids: + missing_ids = sorted(expected_scenario_ids.difference(scenario_artifacts)) + raise RuntimeError(f"Structural freeze is missing scenarios: {missing_ids}.") + scenario_artifacts = dict(sorted(scenario_artifacts.items())) + scenario_counts.sort(key=lambda item: str(item["scenario_id"])) + counts_path = atomic_write_parquet( + pd.DataFrame(scenario_counts), paths.data_dir / "frontier/scenario_counts.parquet" + ) + freeze_payload = { + "schema_version": str(config["schema_version"]), + "status": "outcome_free_structural_grid_frozen_before_endpoint_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "elapsed_seconds": float(time.perf_counter() - started), + "claim_boundary": dict(config["claim_boundary"]), + "numerics": dict(config.get("numerics", {})), + "execution": dict(config.get("execution", {})), + "recovery": recovery_audit, + "source_frontier_freeze": { + "status": parent_freeze["status"], + "run_tag": parent_freeze["run_tag"], + "sha256": source_frontier_config["parent"]["protocol_freeze"]["sha256"], + }, + "scenario_count": len(scenario_artifacts), + "outcome_columns_passed_to_frontier": [], + "outcome_free_decision_base": relative_artifact_descriptor( + decision_base_path, repo_root=root + ), + "scenario_artifacts": scenario_artifacts, + "scenario_counts": relative_artifact_descriptor(counts_path, repo_root=root), + "implementation": implementation_provenance( + config_path=resolved_config, + repo_root=root, + relative_paths=( + Path("src/ijds_audit/structural_sensitivity.py"), + Path("src/ijds_audit/structural_checkpoint.py"), + Path("src/ijds_challengers/normalized_frontier.py"), + Path("scripts/experiments/run_ijds_portfolio_structure_sensitivity.py"), + *_protocol_documents(config), + ), + ), + "environment": environment_provenance(root), + "git": git_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + freeze_path = atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze_payload) + logger.info("Structural grid frozen at {}", freeze_path) + return freeze_path + + +def _verified_structural_freeze( + config: Mapping[str, Any], *, repo_root: Path +) -> tuple[OutputPaths, dict[str, Any]]: + paths = _run_paths(config, repo_root=repo_root) + freeze_path = paths.model_dir / "protocol_freeze.json" + payload = json.loads(freeze_path.read_text(encoding="utf-8")) + expected = { + "status": "outcome_free_structural_grid_frozen_before_endpoint_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + } + for field, value in expected.items(): + if payload.get(field) != value: + raise RuntimeError(f"Structural freeze mismatch for {field}.") + if payload.get("outcome_columns_passed_to_frontier") != []: + raise RuntimeError("Structural freeze reports outcome leakage.") + if int(payload.get("scenario_count", -1)) != len(declared_scenarios(config)): + raise RuntimeError("Structural freeze scenario census changed.") + if payload.get("execution") != dict(config.get("execution", {})): + raise RuntimeError("Structural freeze execution contract changed.") + recovery = payload.get("recovery", {}) + expected_recovery = config.get("execution", {}).get("recovery") + if expected_recovery is not None and ( + recovery.get("enabled") is not True + or int(recovery.get("recovered_scenarios", -1)) + != int(expected_recovery["expected_recovered_scenarios"]) + or sorted(recovery.get("missing_scenario_ids", [])) + != sorted(expected_recovery["expected_missing_scenario_ids"]) + ): + raise RuntimeError("Structural recovery audit changed.") + decision_base = payload.get("outcome_free_decision_base") + if not isinstance(decision_base, dict): + raise RuntimeError("Structural freeze lacks its outcome-free decision base.") + decision_base_path = resolve_repo_input(str(decision_base["path"]), repo_root=repo_root) + if relative_artifact_descriptor(decision_base_path, repo_root=repo_root) != decision_base: + raise RuntimeError("Structural outcome-free decision base changed.") + for artifacts in payload["scenario_artifacts"].values(): + if set(artifacts) != set(ARTIFACT_NAMES): + raise RuntimeError("Structural scenario artifact inventory changed.") + for descriptor in artifacts.values(): + path = resolve_repo_input(str(descriptor["path"]), repo_root=repo_root) + if relative_artifact_descriptor(path, repo_root=repo_root) != descriptor: + raise RuntimeError(f"Structural artifact changed: {descriptor['path']}.") + return paths, payload + + +def _evaluation_contract(base: Mapping[str, Any]) -> dict[str, Any]: + config = copy.deepcopy(dict(base)) + config["evaluation"]["evaluated_roles"] = ["primary_oot"] + config["evaluation"]["expected_solve_records"] = 8 * 15 * 2 * 3 * 2 + config["evaluation"]["expected_window_contrasts"] = 8 * 3 * 2 + config["evaluation"]["expected_monthly_contrasts"] = 8 * 15 * 3 * 2 + config["evaluation"]["expected_metric_directions"] = 8 * 3 * 2 * 3 + config["evaluation"]["expected_candidate_counts"] = {"primary_oot": 376_890} + return config + + +def _baseline_reconciliation( + baseline: pd.DataFrame, + *, + active_run_tag: str, + repo_root: Path, +) -> dict[str, float]: + active_path = ( + repo_root + / "data/processed/experiments/ijds_audit" + / active_run_tag + / "evaluation/window_endpoint_contrasts.parquet" + ) + active = pd.read_parquet(active_path) + keys = ["window_id", "ruler", "coordinate"] + left = baseline.sort_values(keys).reset_index(drop=True) + right = active.sort_values(keys).reset_index(drop=True) + if not left[keys].equals(right[keys]): + raise RuntimeError("Structural baseline keys differ from the active two-ruler result.") + columns = [ + "realized_payoff_difference_lower", + "realized_payoff_difference_upper", + "weighted_default_difference_lower", + "weighted_default_difference_upper", + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ] + differences = { + column: float( + np.max(np.abs(left[column].to_numpy(dtype=float) - right[column].to_numpy(dtype=float))) + ) + for column in columns + } + if max(differences.values()) > 1e-8: + raise RuntimeError(f"Structural baseline failed active reconciliation: {differences}.") + return differences + + +def evaluate(*, config_path: Path, repo_root: Path = ROOT) -> Path: + """Verify the structural freeze, join one endpoint, and report every scenario.""" + started = time.perf_counter() + started_at = utc_now_iso() + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = _load_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths, frozen = _verified_structural_freeze(config, repo_root=root) + summary_path = paths.model_dir / "structural_sensitivity_summary.json" + if summary_path.exists() or (paths.data_dir / "evaluation").exists(): + raise FileExistsError("Structural evaluation already exists; choose a fresh run tag.") + parent = config["parent"] + raw_path = resolve_repo_input(str(parent["raw_path"]), repo_root=root) + if sha256_file(raw_path) != str(parent["raw_sha256"]): + raise RuntimeError("Raw archive hash changed before structural evaluation.") + v4_config = load_v4_config(resolve_repo_input(str(parent["v4_config"]), repo_root=root)) + active_evaluation = load_v2_config( + resolve_repo_input(str(parent["evaluation_config"]), repo_root=root) + ) + evaluation_config = _evaluation_contract(active_evaluation) + universe = load_outcome_universe(v4_config, raw_path=raw_path) + outcomes = rebuild_archive_outcomes( + universe, + evaluation_cutoff=str(config["endpoint"]["evaluation_cutoff"]), + charged_off_lag_months=int(config["endpoint"]["charged_off_lag_months"]), + ) + scenario_results: list[dict[str, Any]] = [] + activity_rows: list[dict[str, Any]] = [] + contrast_frames: list[pd.DataFrame] = [] + direction_frames: list[pd.DataFrame] = [] + evaluation_artifacts: dict[str, Any] = {} + scenarios = declared_scenarios(config) + scenario_lookup = {str(item["scenario_id"]): item for item in scenarios} + for index, scenario_id in enumerate(sorted(frozen["scenario_artifacts"]), start=1): + scenario = scenario_lookup[scenario_id] + logger.info("Structural evaluation scenario {}/36: {}", index, scenario_id) + descriptors = frozen["scenario_artifacts"][scenario_id] + records = pd.read_parquet( + resolve_repo_input(descriptors["solve_records"]["path"], repo_root=root) + ) + allocations = pd.read_parquet( + resolve_repo_input(descriptors["allocations"]["path"], repo_root=root) + ) + endpoints = pd.read_parquet( + resolve_repo_input(descriptors["endpoint_diagnostics"]["path"], repo_root=root) + ) + scenario_parent = _scenario_parent(v4_config, scenario) + evaluated, joined = evaluate_frozen_portfolios( + records, + allocations, + outcomes, + config=scenario_parent, + ) + window, monthly = build_endpoint_contrasts( + joined, + endpoints, + config=evaluation_config, + lgd=float(scenario["lgd"]), + ) + directions = build_metric_directions(window, config=evaluation_config) + expected = { + "evaluated": 8 * 15 * 2 * 3 * 2, + "window": 8 * 3 * 2, + "monthly": 8 * 15 * 3 * 2, + "directions": 8 * 3 * 2 * 3, + } + actual = { + "evaluated": len(evaluated), + "window": len(window), + "monthly": len(monthly), + "directions": len(directions), + } + if actual != expected: + raise RuntimeError(f"Incomplete structural evaluation for {scenario_id}: {actual}.") + activity = allocation_activity( + records, + allocations, + scenario=scenario, + allocation_tolerance=float( + load_frontier_config( + resolve_repo_input(str(parent["frontier_config"]), repo_root=root) + )["solver"]["allocation_tolerance"] + ), + ) + activity_rows.append(activity) + scenario_results.append( + { + **scenario_result_summary(window, directions, scenario=scenario), + **{ + f"activity_{key}": value + for key, value in activity.items() + if key not in scenario + }, + } + ) + tagged_window = _tag(window, scenario) + tagged_directions = _tag(directions, scenario) + contrast_frames.append(tagged_window) + direction_frames.append(tagged_directions) + scenario_dir = paths.data_dir / "evaluation/scenarios" / scenario_id + window_path = atomic_write_parquet(tagged_window, scenario_dir / "window_contrasts.parquet") + direction_path = atomic_write_parquet( + tagged_directions, scenario_dir / "metric_directions.parquet" + ) + evaluation_artifacts[scenario_id] = { + "window_contrasts": relative_artifact_descriptor(window_path, repo_root=root), + "metric_directions": relative_artifact_descriptor(direction_path, repo_root=root), + } + summary_table = pd.DataFrame(scenario_results).sort_values("scenario_id").reset_index(drop=True) + activity_table = pd.DataFrame(activity_rows).sort_values("scenario_id").reset_index(drop=True) + contrast_table = pd.concat(contrast_frames, ignore_index=True) + direction_table = pd.concat(direction_frames, ignore_index=True) + consolidated = { + "scenario_summary": atomic_write_parquet( + summary_table, paths.data_dir / "evaluation/scenario_summary.parquet" + ), + "allocation_activity": atomic_write_parquet( + activity_table, paths.data_dir / "evaluation/allocation_activity.parquet" + ), + "window_contrasts": atomic_write_parquet( + contrast_table, paths.data_dir / "evaluation/window_contrasts.parquet" + ), + "metric_directions": atomic_write_parquet( + direction_table, paths.data_dir / "evaluation/metric_directions.parquet" + ), + } + baseline = contrast_table.loc[contrast_table["scenario_is_baseline"]] + reconciliation = _baseline_reconciliation( + baseline, + active_run_tag=str(active_evaluation["run_tag"]), + repo_root=root, + ) + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_post_freeze_structural_sensitivity_evaluation", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "elapsed_seconds": float(time.perf_counter() - started), + "claim_boundary": dict(config["claim_boundary"]), + "scenario_count": int(len(summary_table)), + "baseline_reconciliation_maxima": reconciliation, + "outcome_columns_joined_after_freeze": ["snapshot_default", "snapshot_resolution"], + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in consolidated.items() + }, + "schemas": { + "scenario_summary": dataframe_schema(summary_table), + "allocation_activity": dataframe_schema(activity_table), + "window_contrasts": dataframe_schema(contrast_table), + "metric_directions": dataframe_schema(direction_table), + }, + "selection": {"scenario": None, "budget": None, "purpose_cap": None, "lgd": None}, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + written = atomic_write_json(summary_path, summary) + atomic_write_json( + paths.model_dir / "verified_evaluation_manifest.json", + { + **summary, + "source_freeze": relative_artifact_descriptor( + paths.model_dir / "protocol_freeze.json", repo_root=root + ), + "scenario_artifacts": evaluation_artifacts, + "implementation": implementation_provenance( + config_path=resolved_config, + repo_root=root, + relative_paths=( + Path("src/ijds_audit/structural_sensitivity.py"), + Path("src/ijds_audit/structural_checkpoint.py"), + Path("scripts/experiments/run_ijds_portfolio_structure_sensitivity.py"), + *_protocol_documents(config), + ), + ), + "environment": environment_provenance(root), + "git": git_provenance(root), + }, + ) + logger.info("Wrote structural sensitivity {}", written) + return written + + +def main(argv: Sequence[str] | None = None) -> None: + """Run one structural-sensitivity phase.""" + args = parse_args(argv) + if args.phase == "freeze": + freeze(config_path=args.config, repo_root=ROOT) + else: + evaluate(config_path=args.config, repo_root=ROOT) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_ijds_raw_data_audit.py b/scripts/experiments/run_ijds_raw_data_audit.py new file mode 100644 index 0000000..dafd7aa --- /dev/null +++ b/scripts/experiments/run_ijds_raw_data_audit.py @@ -0,0 +1,172 @@ +"""Build the full-archive data and feature-contract audit for IJDS.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import yaml + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.raw_data_audit import audit_raw_dataset +from src.utils.isolated_experiment import ( + git_provenance, + relative_artifact_descriptor, + resolve_repo_input, + sha256_file, + validate_run_tag, + write_csv_atomic, +) +from src.utils.pipeline_runtime import atomic_write_json + +ROOT = Path(__file__).resolve().parents[2] +ALLOWED_OUTPUT_ROOT = Path("reports/crpto/data_audit") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=ROOT) + return parser.parse_args(argv) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Raw-data audit config must be a YAML mapping.") + if payload.get("status") != "retrospective_full_archive_data_contract_audit": + raise ValueError("Unexpected raw-data audit status.") + if payload.get("rules", {}).get("no_model_or_policy_selection") is not True: + raise ValueError("The raw-data audit cannot select a model or policy.") + return payload + + +def _output_dir(config: dict[str, Any], repo_root: Path) -> Path: + run_tag = validate_run_tag(str(config["run_tag"])) + configured = (repo_root / str(config["output"]["root"])).resolve() + allowed = (repo_root / ALLOWED_OUTPUT_ROOT).resolve() + if configured != allowed: + raise ValueError(f"Output root {configured} is not allowlisted as {allowed}.") + output = (allowed / run_tag).resolve() + output.relative_to(allowed) + if output.exists(): + raise FileExistsError(f"Audit output already exists: {output}") + output.mkdir(parents=True) + return output + + +def _dvc_md5(raw_path: Path) -> str: + pointer_path = raw_path.with_suffix(raw_path.suffix + ".dvc") + payload = yaml.safe_load(pointer_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not isinstance(payload.get("outs"), list): + raise TypeError(f"Invalid DVC pointer: {pointer_path}") + outputs = payload["outs"] + if len(outputs) != 1 or not isinstance(outputs[0], dict): + raise ValueError(f"Expected one DVC output in {pointer_path}") + digest = str(outputs[0].get("md5", "")) + if len(digest) != 32 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError(f"Invalid DVC MD5 in {pointer_path}") + return digest + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + root = args.repo_root.resolve() + config_path = resolve_repo_input(args.config, repo_root=root) + config = _load_config(config_path) + base_path = resolve_repo_input(config["base_protocol"], repo_root=root) + base = load_v4_config(base_path) + raw_path = resolve_repo_input(base["source"]["raw_path"], repo_root=root) + output = _output_dir(config, root) + audit = audit_raw_dataset(raw_path, base, rules=config["rules"]) + + written = { + "archive_inventory": write_csv_atomic( + audit.archive_inventory, output / "archive_inventory.csv" + ), + "status_inventory": write_csv_atomic( + audit.status_inventory, output / "status_inventory.csv" + ), + "cohort_inventory": write_csv_atomic(audit.inventory, output / "cohort_inventory.csv"), + "feature_coverage": write_csv_atomic( + audit.feature_coverage, output / "raw_feature_coverage.csv" + ), + "feature_contract": write_csv_atomic( + audit.feature_contract, output / "raw_feature_contract.csv" + ), + "amount_alignment": write_csv_atomic( + audit.amount_alignment, output / "loan_amount_alignment.csv" + ), + "cutoff_label_availability": write_csv_atomic( + audit.cutoff_label_availability, output / "cutoff_label_availability.csv" + ), + } + primary = audit.amount_alignment.loc[audit.amount_alignment["cohort"].eq("primary_oot")].iloc[0] + late_count = int(audit.feature_contract["late_feature"].sum()) + eligible_count = int(audit.feature_contract["eligible_for_current_temporal_model"].sum()) + exception_count = int(audit.feature_contract["coverage_exception"].notna().sum()) + sensitivity_count = int(audit.feature_contract["requires_sensitivity"].sum()) + archive = audit.archive_inventory.iloc[0] + evidence = { + "schema_version": str(config["schema_version"]), + "status": "complete_full_archive_data_contract_audit", + "run_tag": str(config["run_tag"]), + "base_protocol": relative_artifact_descriptor(base_path, repo_root=root), + "config": relative_artifact_descriptor(config_path, repo_root=root), + "raw_source": { + **relative_artifact_descriptor(raw_path, repo_root=root), + "dvc_md5": _dvc_md5(raw_path), + }, + "results": { + "raw_rows": int(archive["raw_rows"]), + "valid_loan_rows": int(archive["valid_loan_rows"]), + "distinct_nonblank_ids": int(archive["distinct_nonblank_ids"]), + "term36_rows_all_dates": int(archive["term36_rows"]), + "term60_rows_all_dates": int(archive["term60_rows"]), + "raw_schema_columns": int(len(audit.feature_contract)), + "last_payment_date_max": str(archive["last_payment_date_max"]), + "last_credit_pull_date_max": str(archive["last_credit_pull_date_max"]), + "last_payment_rows_after_cutoff": int(archive["last_payment_rows_after_cutoff"]), + "last_credit_pull_rows_after_cutoff": int( + archive["last_credit_pull_rows_after_cutoff"] + ), + "term36_active_design_rows": int( + audit.inventory.loc[ + audit.inventory["cohort"].isin( + [ + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ] + ), + "rows", + ].sum() + ), + "eligible_raw_features": eligible_count, + "declared_coverage_exceptions": exception_count, + "coverage_exceptions_requiring_sensitivity": sensitivity_count, + "late_schema_features": late_count, + "primary_oot_partial_funding_share": float(primary["partial_share"]), + "primary_oot_funded_ratio": float(primary["funded_ratio"]), + "primary_oot_total_requested_minus_funded": float(primary["total_gap"]), + }, + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in written.items() + }, + "implementation_sha256": sha256_file(root / "src/ijds_audit/raw_data_audit.py"), + "git": git_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + evidence_path = atomic_write_json(output / "evidence.json", evidence) + print(evidence_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/experiments/run_tabpfn_tabprep_full.py b/scripts/experiments/run_tabpfn_tabprep_full.py deleted file mode 100644 index d3787a5..0000000 --- a/scripts/experiments/run_tabpfn_tabprep_full.py +++ /dev/null @@ -1,514 +0,0 @@ -"""Run a full-data TabPFN-3 + TabPrep challenger. - -The script is intentionally isolated from DVC and champion artifacts. It fails -before heavy CRPTO data work if TabPFN-3 model access has not been authorized. -""" - -from __future__ import annotations - -import argparse -import gc -import importlib -import json -import os -import sys -import time -from collections.abc import Iterable, Mapping, Sequence -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import pyarrow.parquet as pq -import yaml -from loguru import logger -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score - -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from src.evaluation.metrics import classification_metrics # noqa: E402 -from src.features.feature_config_io import load_feature_config # noqa: E402 -from src.features.feature_engineering import TARGET # noqa: E402 -from src.features.tabprep_challenger import ( # noqa: E402 - TABPREP_VARIANTS, - TabPrepChallengerTransformer, - TabPrepVariantConfig, - resolve_tabprep_categorical_features, - resolve_tabprep_input_features, - validate_no_forbidden_features, -) -from src.utils.pipeline_runtime import ( # noqa: E402 - atomic_write_json, - atomic_write_parquet, - atomic_write_pickle, -) - -ALLOWED_OUTPUT_PREFIXES = { - Path("data/processed/experiments/tabpfn_tabprep"), - Path("models/experiments/tabpfn_tabprep"), - Path("reports/crpto/experiments/tabpfn_tabprep"), -} -PROTECTED_PATHS = { - Path("models/pd_canonical.cbm"), - Path("models/pd_canonical_calibrator.pkl"), - Path("models/conformal_policy_status.json"), - Path("data/processed/conformal_intervals_mondrian.parquet"), - Path("EXTRACTION_MANIFEST.json"), -} - - -def parse_args() -> argparse.Namespace: - """Parse CLI arguments.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--config", - default="configs/experiments/tabpfn_tabprep_full.yaml", - help="Path to the TabPFN + TabPrep experiment config.", - ) - parser.add_argument( - "--access-check-only", - action="store_true", - help="Only verify TabPFN-3 model access and CUDA setup.", - ) - parser.add_argument( - "--force-outside-limits", - action="store_true", - help="Proceed even though full CRPTO exceeds public TabPFN-3 recommended limits.", - ) - return parser.parse_args() - - -def main() -> None: - """Run the full-data experiment or fail with a concrete blocking reason.""" - args = parse_args() - config = _load_config(Path(args.config)) - _check_tabpfn_access(config) - if args.access_check_only: - logger.info("TabPFN access check passed.") - return - - _preflight_dataset_limits(config=config, force=args.force_outside_limits) - run_tag = str(config["run_tag"]) - data_dir = Path(config["output"]["data_dir"]) / run_tag - model_dir = Path(config["output"]["model_dir"]) / run_tag - report_dir = Path(config["output"]["report_dir"]) / run_tag - _validate_output_roots([data_dir, model_dir, report_dir]) - - started = time.perf_counter() - train = pd.read_parquet(config["data"]["train_path"]) - feature_config = load_feature_config( - yaml_path=Path(config["data"]["feature_config_path"]), - prefer="yaml", - ) - base_features = resolve_tabprep_input_features( - train, - feature_config=feature_config, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - validate_no_forbidden_features( - base_features, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - categorical_features = resolve_tabprep_categorical_features( - base_features, - feature_config=feature_config, - ) - categorical_maps = _fit_category_maps(train, categorical_features) - max_features = int(config["tabprep"]["max_tabpfn_features"]) - variant = _resolve_tabprep_variant_for_tabpfn( - str(config["tabprep"]["variant"]), - max_generated=max(0, max_features - len(base_features)), - ) - transformer = TabPrepChallengerTransformer( - variant=variant, - input_features=base_features, - categorical_features=categorical_features, - target=TARGET, - random_state=42, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - logger.info("Fitting TabPrep {} on full train rows={:,}", variant.name, len(train)) - generated_train = transformer.fit_transform( - train, - train[TARGET].astype(int), - issue_dates=train.get("issue_d"), - ) - generated_features = transformer.generated_features_[ - : max(0, max_features - len(base_features)) - ] - model_features = [*base_features, *generated_features] - generated_train = generated_train[generated_features] - x_train = _combine_features( - train, - generated_train, - model_features, - categorical_maps=categorical_maps, - ) - y_train = train[TARGET].astype(int).to_numpy() - train_rows = len(x_train) - encoded_categorical_indices = [ - idx for idx, feature in enumerate(model_features) if feature in categorical_features - ] - use_native_categoricals = bool(config["tabpfn"].get("use_native_categorical_features", False)) - categorical_indices = encoded_categorical_indices if use_native_categoricals else [] - - atomic_write_parquet( - transformer.feature_manifest(), report_dir / "generated_feature_manifest.parquet" - ) - atomic_write_json( - report_dir / "pre_tabpfn_matrix_status.json", - { - "run_tag": run_tag, - "rows": { - "train": int(train_rows), - "calibration": None, - "test": None, - }, - "n_base_features": len(base_features), - "n_generated_features": len(generated_features), - "n_model_features": len(model_features), - "n_encoded_categorical_features": len(encoded_categorical_indices), - "n_native_categorical_features": len(categorical_indices), - "use_native_categorical_features": use_native_categoricals, - "matrix_dtype": str(x_train.dtype), - "matrix_gb": { - "train": _array_gb(x_train), - "calibration": None, - "test": None, - }, - }, - ) - - del train, generated_train - gc.collect() - - tabpfn_classifier = _tabpfn_classifier_class() - clf = tabpfn_classifier( - n_estimators=int(config["tabpfn"]["n_estimators"]), - categorical_features_indices=categorical_indices, - device=str(config["tabpfn"]["device"]), - ignore_pretraining_limits=bool(config["tabpfn"]["ignore_pretraining_limits"]), - inference_precision=config["tabpfn"]["inference_precision"], - fit_mode=str(config["tabpfn"]["fit_mode"]), - memory_saving_mode=config["tabpfn"]["memory_saving_mode"], - keep_cache_on_device=bool(config["tabpfn"]["keep_cache_on_device"]), - n_preprocessing_jobs=int(config["tabpfn"]["n_preprocessing_jobs"]), - inference_config=dict(config["tabpfn"].get("inference_config") or {}), - show_progress_bar=bool(config["tabpfn"]["show_progress_bar"]), - random_state=42, - ) - logger.info( - "Fitting TabPFN on full matrix rows={:,}, features={:,}, native_categorical={}", - len(x_train), - len(model_features), - len(categorical_indices), - ) - clf.fit(x_train, y_train) - del x_train, y_train - gc.collect() - - chunk_rows = int(config["tabpfn"]["prediction_chunk_rows"]) - calibration = pd.read_parquet(config["data"]["calibration_path"]) - generated_cal = transformer.transform(calibration)[generated_features] - x_cal = _combine_features( - calibration, - generated_cal, - model_features, - categorical_maps=categorical_maps, - ) - y_cal = calibration[TARGET].astype(int).to_numpy() - cal_rows = len(x_cal) - raw_cal = _predict_proba_chunks(clf, x_cal, chunk_rows=chunk_rows) - calibrator = _fit_platt(y_cal, raw_cal) - del calibration, generated_cal, x_cal, y_cal, raw_cal - gc.collect() - - test = pd.read_parquet(config["data"]["test_path"]) - generated_test = transformer.transform(test)[generated_features] - x_test = _combine_features( - test, - generated_test, - model_features, - categorical_maps=categorical_maps, - ) - y_test = test[TARGET].astype(int).to_numpy() - test_rows = len(x_test) - raw_test = _predict_proba_chunks(clf, x_test, chunk_rows=chunk_rows) - calibrated_test = calibrator.predict_proba(raw_test.reshape(-1, 1))[:, 1] - - metrics = classification_metrics(y_test, calibrated_test) - raw_metrics = { - "auc_roc": float(roc_auc_score(y_test, raw_test)), - "brier_score": float(brier_score_loss(y_test, raw_test)), - "log_loss": float(log_loss(y_test, raw_test)), - } - predictions = _prediction_frame(test, raw_prob=raw_test, calibrated_prob=calibrated_test) - atomic_write_parquet(predictions, data_dir / "test_predictions.parquet") - classifier_path = None - if bool(config["output"].get("save_classifier_pickle", False)): - classifier_path = model_dir / "tabpfn_classifier.pkl" - atomic_write_pickle(classifier_path, clf) - atomic_write_pickle(model_dir / "tabpfn_platt_calibrator.pkl", calibrator) - status = { - "run_tag": run_tag, - "elapsed_seconds": time.perf_counter() - started, - "rows": { - "train": train_rows, - "calibration": cal_rows, - "test": test_rows, - }, - "n_base_features": len(base_features), - "n_generated_features": len(generated_features), - "n_model_features": len(model_features), - "categorical_features": categorical_features, - "encoded_categorical_indices": encoded_categorical_indices, - "native_categorical_indices": categorical_indices, - "use_native_categorical_features": use_native_categoricals, - "raw_test_metrics": raw_metrics, - "calibrated_test_metrics": metrics, - "classifier_path": str(classifier_path) if classifier_path is not None else None, - "tabpfn_config": config["tabpfn"], - "tabprep_summary": transformer.state_summary(), - } - atomic_write_json(model_dir / "tabpfn_tabprep_status.json", status) - logger.info( - "TabPFN + TabPrep done: calibrated AUC={:.6f}, Brier={:.6f}", - metrics["auc_roc"], - metrics["brier_score"], - ) - - -def _load_config(path: Path) -> dict[str, Any]: - config = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(config, dict): - raise TypeError(f"Config must be a mapping: {path}") - _validate_output_roots(_output_path_values(config["output"])) - return config - - -def _check_tabpfn_access(config: Mapping[str, Any]) -> None: - """Fail before loading CRPTO data if TabPFN-3 weights are not accessible.""" - from sklearn.datasets import make_classification - - tabpfn_classifier = _tabpfn_classifier_class() - x, y = make_classification(n_samples=80, n_features=8, random_state=42) - clf = tabpfn_classifier( - n_estimators=1, - device=str(config["tabpfn"]["device"]), - ignore_pretraining_limits=True, - fit_mode="low_memory", - memory_saving_mode=True, - keep_cache_on_device=False, - show_progress_bar=False, - random_state=42, - ) - try: - clf.fit(x[:40], y[:40]) - clf.predict_proba(x[40:45]) - except Exception as exc: - token_set = bool(os.getenv("TABPFN_TOKEN")) - raise RuntimeError( - "TabPFN-3 model access failed before CRPTO data loading. " - f"TABPFN_TOKEN set={token_set}. Accept the PriorLabs license and set " - "TABPFN_TOKEN, then rerun this script." - ) from exc - - -def _tabpfn_classifier_class() -> Any: - """Return the optional TabPFN classifier class with a readable error.""" - try: - module = importlib.import_module("tabpfn") - except ImportError as exc: - raise RuntimeError( - "TabPFN + TabPrep is an isolated challenger and requires the optional " - "TabPFN package plus PriorLabs model access. Install/authorize TabPFN " - "before running this experiment." - ) from exc - return module.TabPFNClassifier - - -def _preflight_dataset_limits(*, config: Mapping[str, Any], force: bool) -> None: - """Check full-data shape against public TabPFN-3 limits.""" - parquet_file = pq.ParquetFile(config["data"]["train_path"]) - train_rows = int(parquet_file.metadata.num_rows) - schema_columns = list(parquet_file.schema_arrow.names) - schema_frame = pd.DataFrame( - {str(column): pd.Series(dtype="object") for column in schema_columns} - ) - feature_config = load_feature_config( - yaml_path=Path(config["data"]["feature_config_path"]), - prefer="yaml", - ) - base_feature_count = len( - resolve_tabprep_input_features( - schema_frame, - feature_config=feature_config, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - ) - max_features = int(config["tabprep"]["max_tabpfn_features"]) - cells = train_rows * max_features - status = { - "train_rows": train_rows, - "base_feature_count_head_resolved": base_feature_count, - "max_tabpfn_features": max_features, - "train_cells": cells, - "public_recommended_rows_at_200_features": 1_000_000, - "public_recommended_rows_at_2000_features": 100_000, - "outside_public_recommended_limits": train_rows > 1_000_000 or max_features > 2_000, - } - logger.info("TabPFN full-data preflight: {}", json.dumps(status, sort_keys=True)) - if status["outside_public_recommended_limits"] and not force: - raise RuntimeError( - "Full CRPTO + TabPrep is outside public TabPFN-3 limits. " - "Rerun with --force-outside-limits to attempt it anyway." - ) - - -def _resolve_tabprep_variant_for_tabpfn(name: str, *, max_generated: int) -> TabPrepVariantConfig: - base = TABPREP_VARIANTS[name] - if max_generated >= base.max_generated_features: - return base - scale = max_generated / max(base.max_generated_features, 1) - return TabPrepVariantConfig( - name=f"{base.name}_tabpfn_cap_{max_generated}", - max_generated_features=max_generated, - arithmetic_features=max(0, int(round(base.arithmetic_features * scale))), - groupby_features=max(0, int(round(base.groupby_features * scale))), - target_encoding_features=max(0, int(round(base.target_encoding_features * scale))), - interaction_encoding_features=max( - 0, int(round(base.interaction_encoding_features * scale)) - ), - rsfc_features=max(0, int(round(base.rsfc_features * scale))), - max_numeric_base_features=base.max_numeric_base_features, - max_groupby_numeric_features=base.max_groupby_numeric_features, - max_categorical_base_features=base.max_categorical_base_features, - max_scoring_rows=base.max_scoring_rows, - n_oof_folds=base.n_oof_folds, - smoothing=base.smoothing, - min_group_support=base.min_group_support, - rsfc_candidate_multiplier=base.rsfc_candidate_multiplier, - ) - - -def _combine_features( - frame: pd.DataFrame, - generated: pd.DataFrame, - model_features: Sequence[str], - *, - categorical_maps: Mapping[str, Mapping[str, int]], -) -> np.ndarray: - matrix = np.empty((len(frame), len(model_features)), dtype=np.float32) - generated_columns = set(generated.columns) - for idx, feature in enumerate(model_features): - if feature in generated_columns: - values = generated[feature] - elif feature in categorical_maps: - values = _apply_category_map(frame[feature], categorical_maps[feature]) - else: - values = pd.to_numeric(frame[feature], errors="coerce") - matrix[:, idx] = pd.Series(values).to_numpy(dtype=np.float32, na_value=np.nan) - return matrix - - -def _fit_category_maps( - frame: pd.DataFrame, - categorical_features: Sequence[str], -) -> dict[str, dict[str, int]]: - maps: dict[str, dict[str, int]] = {} - for feature in categorical_features: - if feature not in frame.columns: - continue - values = _normalized_category_series(frame[feature]) - categories = pd.Index(values.dropna().unique()).sort_values() - maps[feature] = {str(category): idx for idx, category in enumerate(categories)} - return maps - - -def _apply_category_map( - series: pd.Series, - category_map: Mapping[str, int], -) -> pd.Series: - values = _normalized_category_series(series) - return values.map(category_map).fillna(-1).astype("float32") - - -def _normalized_category_series(series: pd.Series) -> pd.Series: - return series.astype("string").fillna("__MISSING__") - - -def _array_gb(array: np.ndarray) -> float: - return float(array.nbytes / (1024**3)) - - -def _predict_proba_chunks(model: Any, frame: np.ndarray, *, chunk_rows: int) -> np.ndarray: - chunks: list[np.ndarray] = [] - for start in range(0, len(frame), chunk_rows): - stop = min(start + chunk_rows, len(frame)) - logger.info("Predicting TabPFN rows {:,}-{:,} / {:,}", start, stop, len(frame)) - chunks.append(model.predict_proba(frame[start:stop])[:, 1].astype("float32")) - return np.concatenate(chunks) - - -def _fit_platt(y_true: np.ndarray, raw_prob: np.ndarray) -> LogisticRegression: - calibrator = LogisticRegression(max_iter=1000) - calibrator.fit(raw_prob.reshape(-1, 1), y_true) - return calibrator - - -def _prediction_frame( - frame: pd.DataFrame, - *, - raw_prob: np.ndarray, - calibrated_prob: np.ndarray, -) -> pd.DataFrame: - cols = [col for col in ["id", "issue_d", TARGET] if col in frame.columns] - out = frame[cols].copy() - out["pd_raw_tabpfn_tabprep"] = raw_prob - out["pd_calibrated_tabpfn_tabprep"] = calibrated_prob - return out - - -def _validate_output_roots(paths: Iterable[Any]) -> None: - for raw_path in paths: - _assert_isolated_output(Path(raw_path)) - - -def _output_path_values(output_cfg: Mapping[str, Any]) -> list[Any]: - return [value for key, value in output_cfg.items() if key.endswith(("_dir", "_path", "_root"))] - - -def _assert_isolated_output(path: Path) -> None: - normalized = _normalize_repo_path(path) - if normalized in PROTECTED_PATHS: - raise ValueError(f"Refusing to write protected artifact: {path}") - if not any(_is_relative_to(normalized, allowed) for allowed in ALLOWED_OUTPUT_PREFIXES): - raise ValueError( - "TabPFN + TabPrep outputs must stay under " - f"{sorted(str(root) for root in ALLOWED_OUTPUT_PREFIXES)}; got {path}" - ) - - -def _normalize_repo_path(path: Path) -> Path: - path = Path(path) - if path.is_absolute(): - try: - return Path(path.relative_to(Path.cwd()).as_posix()) - except ValueError: - return path - return Path(path.as_posix()) - - -def _is_relative_to(path: Path | str, parent: Path) -> bool: - try: - Path(path).relative_to(parent) - return True - except ValueError: - return False - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/run_tabprep_challenger.py b/scripts/experiments/run_tabprep_challenger.py deleted file mode 100644 index c410649..0000000 --- a/scripts/experiments/run_tabprep_challenger.py +++ /dev/null @@ -1,806 +0,0 @@ -"""Run isolated TabPrep-inspired PD challengers for CRPTO. - -This script never writes canonical champion artifacts. All models, predictions, -feature manifests and audit files are forced under experiment-only directories. -""" - -from __future__ import annotations - -import argparse -from collections.abc import Iterable, Mapping, Sequence -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from loguru import logger -from sklearn.isotonic import IsotonicRegression -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score - -from scripts.train_pd_model import _apply_calibrator, _prepare_catboost_frame -from src.evaluation.fairness import fairness_report -from src.evaluation.metrics import classification_metrics -from src.features.feature_config_io import load_feature_config, save_feature_config -from src.features.feature_engineering import TARGET -from src.features.tabprep_challenger import ( - TabPrepChallengerTransformer, - resolve_tabprep_categorical_features, - resolve_tabprep_input_features, - validate_no_forbidden_features, -) -from src.models.pd_model import ( - resolve_monotonic_constraints, - temporal_train_val_split, - train_catboost_default, -) -from src.models.venn_abers import VennAbersScoreCalibrator -from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet, atomic_write_pickle - -PROTECTED_PATHS = { - Path("models/pd_canonical.cbm"), - Path("models/pd_canonical_calibrator.pkl"), - Path("models/final_project_promotion.json"), - Path("models/conformal_policy_status.json"), - Path("data/processed/conformal_intervals_mondrian.parquet"), - Path("EXTRACTION_MANIFEST.json"), -} -PROTECTED_PREFIXES = { - Path( - "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734" - ), -} -ALLOWED_OUTPUT_PREFIXES = { - Path("data/processed/experiments"), - Path("models/experiments"), - Path("reports/crpto/experiments"), -} - - -def parse_args() -> argparse.Namespace: - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--config", - default="configs/experiments/tabprep_challenger.yaml", - help="Path to the isolated TabPrep challenger YAML config.", - ) - parser.add_argument( - "--variant", - default=None, - help="Variant to run: safe_500, balanced_1500, full_3000, or all.", - ) - parser.add_argument( - "--seed", - type=int, - default=None, - help="Optional single seed override. Defaults to the config seed list.", - ) - parser.add_argument( - "--sample-rows", - type=int, - default=None, - help="Optional sample rows per split for smoke runs. Use 0 with --full-data.", - ) - parser.add_argument( - "--full-data", - action="store_true", - help="Disable sampling even when the config has sample_rows set.", - ) - parser.add_argument( - "--persist-transformed", - action="store_true", - help="Persist transformed train/validation/calibration/test matrices.", - ) - parser.add_argument( - "--skip-baseline", - action="store_true", - help="Skip the no-generated-features baseline control.", - ) - return parser.parse_args() - - -def main() -> None: - """Run the configured TabPrep challenger experiment.""" - args = parse_args() - config = _load_config(Path(args.config)) - run_tag = str(config.get("run_tag", "tabprep-challenger-2026-06-16")) - variants = _resolve_variants(config, args.variant) - seeds = [int(args.seed)] if args.seed is not None else [int(seed) for seed in config["seeds"]] - sample_rows = 0 if args.full_data else _resolve_sample_rows(config, args.sample_rows) - persist_transformed = bool( - args.persist_transformed or config["tabprep"].get("persist_transformed") - ) - - splits = _load_splits(config) - if sample_rows > 0: - splits = { - name: _sample_split(frame, sample_rows=sample_rows, seed=seeds[0]) - for name, frame in splits.items() - } - - feature_config = _load_feature_config(config) - base_features = resolve_tabprep_input_features( - splits["train"], - feature_config=feature_config, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - validate_no_forbidden_features( - base_features, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - categorical_features = resolve_tabprep_categorical_features( - base_features, - feature_config=feature_config, - ) - - train_fit, train_val = temporal_train_val_split( - splits["train"], - val_fraction=float(config["validation"]["val_fraction"]), - date_col=str(config["validation"]["date_col"]), - ) - - summary: dict[str, Any] = { - "run_tag": run_tag, - "sample_rows": sample_rows, - "variants": variants, - "seeds": seeds, - "base_feature_count": len(base_features), - "categorical_feature_count": len(categorical_features), - "results": [], - } - - if not args.skip_baseline: - for seed in seeds: - summary["results"].append( - _run_model_case( - case_name="baseline_control", - run_tag=run_tag, - seed=seed, - config=config, - train_fit=train_fit, - train_val=train_val, - calibration=splits["calibration"], - test=splits["test"], - features=base_features, - categorical_features=categorical_features, - generated_train_fit=None, - generated_train_val=None, - generated_calibration=None, - generated_test=None, - transformer=None, - persist_transformed=persist_transformed, - ) - ) - - for variant in variants: - for seed in seeds: - transformer = TabPrepChallengerTransformer( - variant=variant, - input_features=base_features, - categorical_features=categorical_features, - target=TARGET, - random_state=seed, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - generated_train_fit = transformer.fit_transform( - train_fit, - train_fit[TARGET].astype(int), - issue_dates=train_fit.get(str(config["validation"]["date_col"])), - ) - generated_train_val = transformer.transform(train_val) - generated_calibration = transformer.transform(splits["calibration"]) - generated_test = transformer.transform(splits["test"]) - case_result = _run_model_case( - case_name=variant, - run_tag=run_tag, - seed=seed, - config=config, - train_fit=train_fit, - train_val=train_val, - calibration=splits["calibration"], - test=splits["test"], - features=base_features, - categorical_features=categorical_features, - generated_train_fit=generated_train_fit, - generated_train_val=generated_train_val, - generated_calibration=generated_calibration, - generated_test=generated_test, - transformer=transformer, - persist_transformed=persist_transformed, - ) - summary["results"].append(case_result) - - output_dir = _case_dir( - Path(config["output"]["report_dir"]), - run_tag=run_tag, - case_name="summary", - seed=seeds[0], - ) - atomic_write_json(output_dir / "tabprep_challenger_summary.json", summary) - logger.info("Wrote TabPrep challenger summary to {}", output_dir) - - -def _load_config(path: Path) -> dict[str, Any]: - config = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(config, dict): - raise TypeError(f"Config must be a mapping: {path}") - config.setdefault("run_tag", "tabprep-challenger-2026-06-16") - config.setdefault("seeds", [42, 52, 62]) - config.setdefault("sample_rows", 0) - config.setdefault("tabprep", {}) - config.setdefault("validation", {}) - config.setdefault("calibration", {}) - config.setdefault("output", {}) - config["tabprep"].setdefault("variants", ["safe_500", "balanced_1500", "full_3000"]) - config["tabprep"].setdefault("extra_blacklist", []) - config["tabprep"].setdefault("persist_transformed", False) - config["tabprep"].setdefault("suspicious_auc_threshold", 0.90) - config["validation"].setdefault("val_fraction", 0.15) - config["validation"].setdefault("date_col", "issue_d") - config["calibration"].setdefault("method", "auto") - config["calibration"].setdefault("candidates", ["platt", "isotonic", "venn_abers"]) - config["output"].setdefault("data_dir", "data/processed/experiments/tabprep") - config["output"].setdefault("model_dir", "models/experiments/tabprep") - config["output"].setdefault("report_dir", "reports/crpto/experiments/tabprep") - config.setdefault("model", {}) - config["model"].setdefault("params", {}) - _validate_output_roots(config["output"].values()) - return config - - -def _resolve_variants(config: Mapping[str, Any], variant_override: str | None) -> list[str]: - if variant_override and variant_override != "all": - return [variant_override] - return [str(variant) for variant in config["tabprep"]["variants"]] - - -def _resolve_sample_rows(config: Mapping[str, Any], cli_value: int | None) -> int: - if cli_value is not None: - return max(0, int(cli_value)) - return max(0, int(config.get("sample_rows", 0))) - - -def _load_splits(config: Mapping[str, Any]) -> dict[str, pd.DataFrame]: - data_cfg = config["data"] - return { - "train": pd.read_parquet(data_cfg["train_path"]), - "calibration": pd.read_parquet(data_cfg["calibration_path"]), - "test": pd.read_parquet(data_cfg["test_path"]), - } - - -def _load_feature_config(config: Mapping[str, Any]) -> dict[str, Any]: - path = Path(config["data"].get("feature_config_path", "data/processed/feature_config.yml")) - return load_feature_config(yaml_path=path, prefer="yaml") - - -def _sample_split(frame: pd.DataFrame, *, sample_rows: int, seed: int) -> pd.DataFrame: - if sample_rows <= 0 or len(frame) <= sample_rows: - return frame.reset_index(drop=True) - sampled = frame.sample(n=sample_rows, random_state=seed) - if "issue_d" in sampled.columns: - sampled = sampled.sort_values("issue_d", kind="mergesort") - return sampled.reset_index(drop=True) - - -def _run_model_case( - *, - case_name: str, - run_tag: str, - seed: int, - config: Mapping[str, Any], - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - calibration: pd.DataFrame, - test: pd.DataFrame, - features: Sequence[str], - categorical_features: Sequence[str], - generated_train_fit: pd.DataFrame | None, - generated_train_val: pd.DataFrame | None, - generated_calibration: pd.DataFrame | None, - generated_test: pd.DataFrame | None, - transformer: TabPrepChallengerTransformer | None, - persist_transformed: bool, -) -> dict[str, Any]: - model_dir = _case_dir( - Path(config["output"]["model_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - data_dir = _case_dir( - Path(config["output"]["data_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - report_dir = _case_dir( - Path(config["output"]["report_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - _validate_output_roots([model_dir, data_dir, report_dir]) - - generated_features = ( - list(generated_train_fit.columns) if generated_train_fit is not None else [] - ) - model_features = [*features, *generated_features] - cat_features = [feature for feature in categorical_features if feature in model_features] - - x_train = _combine_features(train_fit, features, generated_train_fit) - x_val = _combine_features(train_val, features, generated_train_val) - x_cal = _combine_features(calibration, features, generated_calibration) - x_test = _combine_features(test, features, generated_test) - - x_train_cb = _prepare_catboost_frame(x_train, model_features, cat_features) - x_val_cb = _prepare_catboost_frame(x_val, model_features, cat_features) - x_cal_cb = _prepare_catboost_frame(x_cal, model_features, cat_features) - x_test_cb = _prepare_catboost_frame(x_test, model_features, cat_features) - - params = _model_params(config, model_features=model_features, seed=seed) - model, train_metrics = train_catboost_default( - x_train_cb, - train_fit[TARGET].astype(int), - x_val_cb, - train_val[TARGET].astype(int), - X_test=x_test_cb, - y_test=test[TARGET].astype(int), - cat_features=cat_features, - params=params, - ) - - raw_cal = model.predict_proba(x_cal_cb)[:, 1] - raw_test = model.predict_proba(x_test_cb)[:, 1] - selected_calibrator, calibration_report = _select_calibrator( - method=str(config["calibration"]["method"]), - candidates=[str(item) for item in config["calibration"]["candidates"]], - y_cal=calibration[TARGET].astype(int).to_numpy(), - raw_cal=raw_cal, - ) - calibrated_test = _apply_calibrator(selected_calibrator, raw_test) - calibrated_cal = _apply_calibrator(selected_calibrator, raw_cal) - - test_metrics = classification_metrics( - test[TARGET].astype(int).to_numpy(), - np.asarray(calibrated_test, dtype=float), - ) - cal_metrics = classification_metrics( - calibration[TARGET].astype(int).to_numpy(), - np.asarray(calibrated_cal, dtype=float), - ) - fairness = _build_fairness_report( - test, - y_prob=np.asarray(calibrated_test, dtype=float), - config=config, - ) - audit = _build_audit_report( - case_name=case_name, - train_generated=generated_train_fit, - test_generated=generated_test, - y_train=train_fit[TARGET].astype(int), - model=model, - x_test=x_test_cb, - y_test=test[TARGET].astype(int), - cat_features=cat_features, - config=config, - ) - - model_path = model_dir / "pd_tabprep_challenger.cbm" - calibrator_path = model_dir / "pd_tabprep_calibrator.pkl" - transformer_path = model_dir / "tabprep_transformer.pkl" - prediction_path = data_dir / "test_predictions.parquet" - status_path = model_dir / "tabprep_training_status.json" - - _assert_isolated_output(model_path) - _assert_isolated_output(calibrator_path) - _assert_isolated_output(transformer_path) - _assert_isolated_output(prediction_path) - _assert_isolated_output(status_path) - - model_dir.mkdir(parents=True, exist_ok=True) - model.save_model(str(model_path)) - atomic_write_pickle(calibrator_path, selected_calibrator) - if transformer is not None: - atomic_write_pickle(transformer_path, transformer) - manifest = transformer.feature_manifest() - atomic_write_parquet(manifest, report_dir / "generated_feature_manifest.parquet") - atomic_write_json( - report_dir / "tabprep_transformer_summary.json", transformer.state_summary() - ) - _write_experiment_feature_config( - report_dir / "feature_config.yml", - base_features=features, - generated_features=generated_features, - categorical_features=cat_features, - ) - - predictions = _prediction_frame( - test, - raw_prob=raw_test, - calibrated_prob=np.asarray(calibrated_test, dtype=float), - case_name=case_name, - seed=seed, - ) - atomic_write_parquet(predictions, prediction_path) - if not fairness.empty: - atomic_write_parquet(fairness, report_dir / "fairness_report.parquet") - atomic_write_json(report_dir / "audit_report.json", audit) - - if persist_transformed: - _persist_transformed_frames( - data_dir=data_dir, - train_fit=x_train_cb, - train_val=x_val_cb, - calibration=x_cal_cb, - test=x_test_cb, - ) - - status = { - "run_tag": run_tag, - "case_name": case_name, - "seed": seed, - "model_path": str(model_path), - "calibrator_path": str(calibrator_path), - "transformer_path": str(transformer_path) if transformer is not None else None, - "prediction_path": str(prediction_path), - "n_base_features": len(features), - "n_generated_features": len(generated_features), - "n_model_features": len(model_features), - "categorical_features": cat_features, - "training_metrics": train_metrics, - "test_metrics": test_metrics, - "calibration_metrics": cal_metrics, - "calibration_selection": calibration_report, - "audit_path": str(report_dir / "audit_report.json"), - } - atomic_write_json(status_path, status) - logger.info( - "{} seed {} done: test AUC={:.4f}, Brier={:.4f}, generated_features={}", - case_name, - seed, - test_metrics["auc_roc"], - test_metrics["brier_score"], - len(generated_features), - ) - return status - - -def _combine_features( - frame: pd.DataFrame, - base_features: Sequence[str], - generated: pd.DataFrame | None, -) -> pd.DataFrame: - base = frame[[feature for feature in base_features if feature in frame.columns]].copy() - base = base.reset_index(drop=True) - if generated is None: - return base - return pd.concat([base, generated.reset_index(drop=True)], axis=1) - - -def _model_params( - config: Mapping[str, Any], - *, - model_features: Sequence[str], - seed: int, -) -> dict[str, Any]: - params = dict(config["model"].get("params", {}) or {}) - raw_constraints = params.pop("monotone_constraints", None) - constraint_map = _parse_monotone_constraints(raw_constraints) - constraint_map.update(config.get("tabprep", {}).get("monotonic_constraints", {}) or {}) - constraints = resolve_monotonic_constraints( - list(model_features), - constraints_config={str(key): int(value) for key, value in constraint_map.items()}, - ) - if constraints is not None: - params["monotone_constraints"] = f"({constraints})" - feature_weights = params.get("feature_weights") - if isinstance(feature_weights, dict): - weights_by_feature = { - str(feature): float(weight) for feature, weight in feature_weights.items() - } - if any(feature in weights_by_feature for feature in model_features): - params["feature_weights"] = [ - weights_by_feature.get(str(feature), 1.0) for feature in model_features - ] - else: - params.pop("feature_weights", None) - params["random_seed"] = int(seed) - params["allow_writing_files"] = False - params.setdefault("verbose", 100) - return params - - -def _parse_monotone_constraints(raw: Any) -> dict[str, int]: - if isinstance(raw, dict): - return {str(key): int(value) for key, value in raw.items()} - if not isinstance(raw, str) or not raw.strip(): - return {} - parsed: dict[str, int] = {} - for part in raw.split(","): - if ":" not in part: - continue - feature, value = part.split(":", 1) - parsed[feature.strip()] = int(value.strip()) - return parsed - - -def _select_calibrator( - *, - method: str, - candidates: Sequence[str], - y_cal: np.ndarray, - raw_cal: np.ndarray, -) -> tuple[Any, dict[str, Any]]: - method = method.lower() - if method != "auto": - return _fit_calibrator(method, y_cal, raw_cal), { - "selected_method": method, - "reason": "fixed", - } - if len(y_cal) < 50 or len(np.unique(y_cal)) < 2: - model = _fit_calibrator("platt", y_cal, raw_cal) - return model, {"selected_method": "platt", "reason": "small_calibration_split"} - - split = max(10, int(round(len(y_cal) * 0.70))) - rows: list[dict[str, Any]] = [] - for candidate in candidates: - try: - fitted = _fit_calibrator(candidate, y_cal[:split], raw_cal[:split]) - pred = _apply_calibrator(fitted, raw_cal[split:]) - rows.append( - { - "method": candidate, - "brier_score": float(brier_score_loss(y_cal[split:], pred)), - "log_loss": float(log_loss(y_cal[split:], pred)), - "auc_roc": float(roc_auc_score(y_cal[split:], pred)), - } - ) - except Exception as exc: # pragma: no cover - defensive diagnostic path - rows.append({"method": candidate, "error": str(exc)}) - feasible = [row for row in rows if "error" not in row] - if not feasible: - model = _fit_calibrator("platt", y_cal, raw_cal) - return model, { - "selected_method": "platt", - "reason": "all_candidates_failed", - "candidate_reports": rows, - } - feasible.sort(key=lambda row: (row["brier_score"], row["log_loss"], -row["auc_roc"])) - selected = str(feasible[0]["method"]) - return _fit_calibrator(selected, y_cal, raw_cal), { - "selected_method": selected, - "reason": "holdout_brier_logloss_auc", - "candidate_reports": rows, - } - - -def _fit_calibrator(method: str, y_true: np.ndarray, raw_prob: np.ndarray) -> Any: - method = method.lower() - if method == "platt": - model = LogisticRegression(max_iter=1000) - model.fit(raw_prob.reshape(-1, 1), y_true) - return model - if method == "isotonic": - model = IsotonicRegression(y_min=0, y_max=1, out_of_bounds="clip") - model.fit(raw_prob, y_true) - return model - if method == "venn_abers": - model = VennAbersScoreCalibrator() - model.fit(raw_prob, y_true) - return model - if method == "beta": - from src.models.calibration import calibrate_beta - - return calibrate_beta(y_true, raw_prob) - if method == "temperature": - from src.models.calibration import TemperatureScalingCalibrator - - return TemperatureScalingCalibrator().fit(raw_prob, y_true) - if method == "quadratic_logit": - from src.models.calibration import QuadraticLogitCalibrator - - return QuadraticLogitCalibrator().fit(raw_prob, y_true) - raise ValueError(f"Unsupported calibration method: {method}") - - -def _prediction_frame( - frame: pd.DataFrame, - *, - raw_prob: np.ndarray, - calibrated_prob: np.ndarray, - case_name: str, - seed: int, -) -> pd.DataFrame: - cols = [col for col in ["id", "issue_d", TARGET] if col in frame.columns] - out = frame[cols].copy() - out["pd_raw"] = raw_prob - out["pd_calibrated"] = calibrated_prob - out["case_name"] = case_name - out["seed"] = int(seed) - return out - - -def _build_fairness_report( - test: pd.DataFrame, - *, - y_prob: np.ndarray, - config: Mapping[str, Any], -) -> pd.DataFrame: - columns = config.get("evaluation", {}).get( - "fairness_columns", - ["grade", "sub_grade", "term", "purpose", "home_ownership"], - ) - groups = {col: test[col].astype(str).to_numpy() for col in columns if col in test.columns} - if not groups: - return pd.DataFrame() - return fairness_report( - y_true=test[TARGET].astype(int).to_numpy(), - y_pred_proba=y_prob, - groups_dict=groups, - threshold=float(config.get("evaluation", {}).get("fairness_threshold", 0.50)), - ) - - -def _build_audit_report( - *, - case_name: str, - train_generated: pd.DataFrame | None, - test_generated: pd.DataFrame | None, - y_train: pd.Series, - model: Any, - x_test: pd.DataFrame, - y_test: pd.Series, - cat_features: Sequence[str], - config: Mapping[str, Any], -) -> dict[str, Any]: - suspicious_threshold = float(config.get("tabprep", {}).get("suspicious_auc_threshold", 0.90)) - report: dict[str, Any] = { - "case_name": case_name, - "suspicious_auc_threshold": suspicious_threshold, - "suspicious_single_feature_auc": [], - "generated_feature_drift": [], - "top_feature_importance": [], - } - if train_generated is not None and test_generated is not None: - report["suspicious_single_feature_auc"] = _single_feature_auc_audit( - train_generated, - y_train, - threshold=suspicious_threshold, - ) - report["generated_feature_drift"] = _generated_drift_audit(train_generated, test_generated) - try: - from catboost import Pool - - importance = model.get_feature_importance( - Pool(x_test, y_test.astype(int), cat_features=list(cat_features)), - prettified=True, - ) - if isinstance(importance, pd.DataFrame): - report["top_feature_importance"] = importance.head(50).to_dict(orient="records") - except Exception as exc: # pragma: no cover - non-critical audit path - report["feature_importance_error"] = str(exc) - return report - - -def _single_feature_auc_audit( - generated: pd.DataFrame, - y: pd.Series, - *, - threshold: float, -) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - y_values = y.astype(int).to_numpy() - if len(np.unique(y_values)) < 2: - return rows - for feature in generated.columns: - values = pd.to_numeric(generated[feature], errors="coerce").replace( - [np.inf, -np.inf], np.nan - ) - fill = values.median() - if not np.isfinite(fill): - fill = 0.0 - values = values.fillna(float(fill)).to_numpy(dtype=float) - if len(np.unique(values)) < 2: - continue - auc = float(roc_auc_score(y_values, values)) - directional_auc = max(auc, 1.0 - auc) - if directional_auc >= threshold: - rows.append({"feature": feature, "auc": directional_auc}) - rows.sort(key=lambda row: -float(row["auc"])) - return rows - - -def _generated_drift_audit( - train_generated: pd.DataFrame, test_generated: pd.DataFrame -) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for feature in train_generated.columns: - train_values = pd.to_numeric(train_generated[feature], errors="coerce") - test_values = pd.to_numeric(test_generated[feature], errors="coerce") - rows.append( - { - "feature": feature, - "coverage_train": float(train_values.notna().mean()), - "coverage_test": float(test_values.notna().mean()), - "mean_train": float(train_values.mean()) if train_values.notna().any() else None, - "mean_test": float(test_values.mean()) if test_values.notna().any() else None, - "std_train": float(train_values.std()) if train_values.notna().any() else None, - "std_test": float(test_values.std()) if test_values.notna().any() else None, - } - ) - return rows - - -def _write_experiment_feature_config( - path: Path, - *, - base_features: Sequence[str], - generated_features: Sequence[str], - categorical_features: Sequence[str], -) -> None: - cfg = { - "CATBOOST_FEATURES": [*base_features, *generated_features], - "CATEGORICAL_FEATURES": list(categorical_features), - "TABPREP_BASE_FEATURES": list(base_features), - "TABPREP_GENERATED_FEATURES": list(generated_features), - "schema_version": "tabprep-challenger-2026-06-16", - } - _assert_isolated_output(path) - save_feature_config(cfg, yaml_path=path) - - -def _persist_transformed_frames( - *, - data_dir: Path, - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - calibration: pd.DataFrame, - test: pd.DataFrame, -) -> None: - for name, frame in { - "train_fit_tabprep.parquet": train_fit, - "train_val_tabprep.parquet": train_val, - "calibration_tabprep.parquet": calibration, - "test_tabprep.parquet": test, - }.items(): - path = data_dir / name - _assert_isolated_output(path) - atomic_write_parquet(frame, path) - - -def _case_dir(root: Path, *, run_tag: str, case_name: str, seed: int) -> Path: - return root / run_tag / case_name / f"seed_{seed}" - - -def _validate_output_roots(paths: Iterable[Any]) -> None: - for raw_path in paths: - _assert_isolated_output(Path(raw_path)) - - -def _assert_isolated_output(path: Path) -> None: - normalized = _normalize_repo_path(path) - if normalized in PROTECTED_PATHS: - raise ValueError(f"Refusing to write protected CRPTO artifact: {path}") - if any(_is_relative_to(normalized, protected) for protected in PROTECTED_PREFIXES): - raise ValueError(f"Refusing to write below protected CRPTO artifact directory: {path}") - if not any(_is_relative_to(normalized, allowed) for allowed in ALLOWED_OUTPUT_PREFIXES): - raise ValueError( - "TabPrep challenger outputs must stay under " - f"{sorted(str(root) for root in ALLOWED_OUTPUT_PREFIXES)}; got {path}" - ) - - -def _normalize_repo_path(path: Path) -> Path: - path = Path(path) - if path.is_absolute(): - try: - return Path(path.relative_to(Path.cwd()).as_posix()) - except ValueError: - return path - return Path(path.as_posix()) - - -def _is_relative_to(path: Path | str, parent: Path) -> bool: - try: - Path(path).relative_to(parent) - return True - except ValueError: - return False - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/run_tabprep_feature_selection_catboost.py b/scripts/experiments/run_tabprep_feature_selection_catboost.py deleted file mode 100644 index 8a1212a..0000000 --- a/scripts/experiments/run_tabprep_feature_selection_catboost.py +++ /dev/null @@ -1,1105 +0,0 @@ -"""Run full-data CatBoost experiments with selected TabPrep features. - -This runner is experimental-only. It reads frozen CRPTO feature splits, creates -TabPrep features once, ranks generated features using a prior selector model, -and trains several compact CatBoost challengers under isolated output roots. -""" - -from __future__ import annotations - -import argparse -import gc -import sys -import time -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from catboost import CatBoostClassifier, Pool -from loguru import logger -from sklearn.metrics import roc_auc_score - -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -from scripts.experiments.run_tabprep_challenger import ( # noqa: E402 - _apply_calibrator, - _build_fairness_report, - _case_dir, - _combine_features, - _model_params, - _prediction_frame, - _prepare_catboost_frame, - _resolve_sample_rows, - _sample_split, - _select_calibrator, - _validate_output_roots, -) -from src.evaluation.metrics import classification_metrics # noqa: E402 -from src.features.feature_config_io import load_feature_config # noqa: E402 -from src.features.feature_engineering import TARGET # noqa: E402 -from src.features.tabprep_challenger import ( # noqa: E402 - TabPrepChallengerTransformer, - resolve_tabprep_categorical_features, - resolve_tabprep_input_features, - validate_no_forbidden_features, -) -from src.models.pd_model import temporal_train_val_split, train_catboost_default # noqa: E402 -from src.utils.pipeline_runtime import ( # noqa: E402 - atomic_write_json, - atomic_write_parquet, - atomic_write_pickle, - write_runtime_checkpoint, - write_runtime_status, -) - -DEFAULT_CASES = [ - "core42", - "core42_woe", - "pool93_woe", - "pool93_top50", - "pool93_top100", - "pool93_business80", - "core42_business80", -] -STABLE_CORE_EXCLUDES = {"rev_utilization", "high_util_pct"} -BUSINESS_SOURCE_DENY_TOKENS = {"__is_missing"} - - -def parse_args() -> argparse.Namespace: - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--config", - default="configs/experiments/tabprep_catboost_full.yaml", - help="Base experiment config.", - ) - parser.add_argument( - "--run-tag", - default="tabprep-feature-selection-2026-06-17", - help="Isolated run tag for selected-feature experiments.", - ) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument( - "--tabprep-seed", - type=int, - default=None, - help=( - "Seed for TabPrep feature generation. Defaults to --seed; set to 42 " - "when testing CatBoost seed stability on the same generated feature space." - ), - ) - parser.add_argument("--variant", default="balanced_1500") - parser.add_argument( - "--selector-model", - default=( - "models/experiments/tabprep/tabprep-catboost-full-2026-06-17/" - "balanced_1500/seed_42/pd_tabprep_challenger.cbm" - ), - help="Previously trained all-feature TabPrep CatBoost model for ranking.", - ) - parser.add_argument( - "--cases", - default=",".join(DEFAULT_CASES), - help="Comma-separated cases to run.", - ) - parser.add_argument( - "--ranking-method", - choices=["pvc", "shap_blend"], - default="pvc", - help="Feature ranking source: CatBoost PredictionValuesChange or blended PVC+SHAP ranks.", - ) - parser.add_argument( - "--shap-rows", - type=int, - default=30000, - help="Maximum validation rows used for SHAP ranking when --ranking-method=shap_blend.", - ) - parser.add_argument( - "--calibration-method", - default=None, - help="Override calibration.method from config.", - ) - parser.add_argument( - "--sample-rows", - type=int, - default=None, - help="Optional sample rows per split for smoke runs. Use 0 with --full-data.", - ) - parser.add_argument( - "--full-data", - action="store_true", - help="Disable sampling even when the config has sample_rows set.", - ) - return parser.parse_args() - - -def main() -> None: - """Run selected-feature CatBoost experiments.""" - args = parse_args() - config = _load_config(Path(args.config)) - if args.calibration_method is not None: - config.setdefault("calibration", {})["method"] = str(args.calibration_method) - run_tag = str(args.run_tag) - seed = int(args.seed) - tabprep_seed = int(args.tabprep_seed if args.tabprep_seed is not None else args.seed) - cases = [case.strip() for case in str(args.cases).split(",") if case.strip()] - sample_rows = 0 if args.full_data else _resolve_sample_rows(config, args.sample_rows) - started = time.perf_counter() - runtime_paths = _runtime_paths(config, run_tag=run_tag, seed=seed) - _write_runtime( - runtime_paths=runtime_paths, - phase="start", - state="running", - run_tag=run_tag, - extra={ - "seed": seed, - "tabprep_seed": tabprep_seed, - "variant": str(args.variant), - "cases": cases, - "sample_rows": sample_rows, - }, - ) - - train = pd.read_parquet(config["data"]["train_path"]) - calibration = pd.read_parquet(config["data"]["calibration_path"]) - test = pd.read_parquet(config["data"]["test_path"]) - if sample_rows > 0: - train = _sample_split(train, sample_rows=sample_rows, seed=seed) - calibration = _sample_split(calibration, sample_rows=sample_rows, seed=seed) - test = _sample_split(test, sample_rows=sample_rows, seed=seed) - feature_config = load_feature_config( - yaml_path=Path(config["data"]["feature_config_path"]), - prefer="yaml", - ) - - pool_features = resolve_tabprep_input_features( - train, - feature_config=feature_config, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - validate_no_forbidden_features( - pool_features, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - categorical_features = resolve_tabprep_categorical_features( - pool_features, - feature_config=feature_config, - ) - train_columns = [str(column) for column in train.columns] - core_features = _resolve_core_features(feature_config, train_columns) - catboost_features = _resolve_catboost_features(feature_config, train_columns) - woe_features = _resolve_woe_features(feature_config, train_columns) - - train_fit, train_val = temporal_train_val_split( - train, - val_fraction=float(config["validation"]["val_fraction"]), - date_col=str(config["validation"]["date_col"]), - ) - del train - gc.collect() - - transformer = TabPrepChallengerTransformer( - variant=str(args.variant), - input_features=pool_features, - categorical_features=categorical_features, - target=TARGET, - random_state=tabprep_seed, - extra_blacklist=config["tabprep"].get("extra_blacklist", []), - ) - logger.info("Fitting TabPrep {} once for selected-feature runs", args.variant) - generated_train_fit = transformer.fit_transform( - train_fit, - train_fit[TARGET].astype(int), - issue_dates=train_fit.get(str(config["validation"]["date_col"])), - ) - generated_train_val = transformer.transform(train_val) - generated_calibration = transformer.transform(calibration) - generated_test = transformer.transform(test) - - generated_features = list(generated_train_fit.columns) - manifest = transformer.feature_manifest() - pool_ranking, ranking, ranking_diagnostics = _load_selector_rankings( - selector_model_path=Path(args.selector_model), - pool_features=pool_features, - generated_features=generated_features, - ranking_method=str(args.ranking_method), - shap_rows=int(args.shap_rows), - train_val=train_val, - train_fit=train_fit, - generated_train_val=generated_train_val, - categorical_features=categorical_features, - ) - business_ranking = _business_ranking(ranking, manifest) - case_features = _build_case_features( - cases=cases, - core_features=core_features, - catboost_features=catboost_features, - pool_features=pool_features, - pool_ranking=pool_ranking, - woe_features=woe_features, - generated_ranking=ranking, - business_ranking=business_ranking, - ) - _write_selection_manifest( - config=config, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - cases=case_features, - ranking=ranking, - pool_ranking=pool_ranking, - business_ranking=business_ranking, - manifest=manifest, - ranking_diagnostics=ranking_diagnostics, - ) - _write_checkpoint( - runtime_paths=runtime_paths, - checkpoint_name="selection_manifest", - payload={ - "run_tag": run_tag, - "seed": seed, - "tabprep_seed": tabprep_seed, - "cases": list(case_features), - "pool_ranking_count": len(pool_ranking), - "generated_ranking_count": len(ranking), - "business_ranking_count": len(business_ranking), - }, - ) - - results: list[dict[str, Any]] = [] - for case_name, feature_set in case_features.items(): - generated_subset = [feature for feature in feature_set if feature in generated_features] - base_subset = [feature for feature in feature_set if feature not in generated_features] - result = _run_selected_case( - case_name=case_name, - run_tag=run_tag, - seed=seed, - tabprep_seed=tabprep_seed, - config=config, - train_fit=train_fit, - train_val=train_val, - calibration=calibration, - test=test, - base_features=base_subset, - categorical_features=categorical_features, - generated_train_fit=generated_train_fit[generated_subset], - generated_train_val=generated_train_val[generated_subset], - generated_calibration=generated_calibration[generated_subset], - generated_test=generated_test[generated_subset], - transformer_summary=transformer.state_summary(), - selection_sources={ - "core_features": core_features, - "pool_features": pool_features, - "woe_features": woe_features, - "generated_features": generated_subset, - }, - ) - results.append(result) - _write_runtime( - runtime_paths=runtime_paths, - phase="case_complete", - state="running", - run_tag=run_tag, - extra={ - "seed": seed, - "tabprep_seed": tabprep_seed, - "completed_cases": [row["case_name"] for row in results], - "latest_case": case_name, - "latest_auc": result.get("test_metrics", {}).get("auc_roc"), - "best_auc": max( - float(row.get("test_metrics", {}).get("auc_roc", float("-inf"))) - for row in results - ), - "elapsed_seconds": time.perf_counter() - started, - }, - ) - gc.collect() - - summary_dir = _case_dir( - Path(config["output"]["report_dir"]), - run_tag=run_tag, - case_name="summary", - seed=seed, - ) - atomic_write_json( - summary_dir / "selected_feature_experiment_summary.json", - { - "run_tag": run_tag, - "seed": seed, - "tabprep_seed": tabprep_seed, - "variant": str(args.variant), - "elapsed_seconds": time.perf_counter() - started, - "cases": list(case_features), - "results": results, - }, - ) - _write_checkpoint( - runtime_paths=runtime_paths, - checkpoint_name="summary", - payload={ - "summary_path": str(summary_dir / "selected_feature_experiment_summary.json"), - "results": results, - }, - ) - _write_runtime( - runtime_paths=runtime_paths, - phase="complete", - state="completed", - run_tag=run_tag, - extra={ - "seed": seed, - "tabprep_seed": tabprep_seed, - "elapsed_seconds": time.perf_counter() - started, - "summary_path": str(summary_dir / "selected_feature_experiment_summary.json"), - "completed_cases": [row["case_name"] for row in results], - }, - ) - - -def _load_config(path: Path) -> dict[str, Any]: - config = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(config, dict): - raise TypeError(f"Config must be a mapping: {path}") - config.setdefault("calibration", {}) - config["calibration"].setdefault("method", "auto") - config["calibration"].setdefault("candidates", ["platt", "isotonic", "venn_abers"]) - _validate_output_roots(config["output"].values()) - return config - - -def _resolve_core_features( - feature_config: Mapping[str, Any], - columns: Sequence[str], -) -> list[str]: - available = set(columns) - return [ - str(feature) - for feature in feature_config.get("CATBOOST_FEATURES", []) - if feature in available and feature not in STABLE_CORE_EXCLUDES - ] - - -def _resolve_catboost_features( - feature_config: Mapping[str, Any], - columns: Sequence[str], -) -> list[str]: - available = set(columns) - return [ - str(feature) - for feature in feature_config.get("CATBOOST_FEATURES", []) - if feature in available - ] - - -def _resolve_woe_features( - feature_config: Mapping[str, Any], - columns: Sequence[str], -) -> list[str]: - available = set(columns) - return [ - str(feature) for feature in feature_config.get("WOE_FEATURES", []) if feature in available - ] - - -def _load_selector_rankings( - *, - selector_model_path: Path, - pool_features: Sequence[str], - generated_features: Sequence[str], - ranking_method: str, - shap_rows: int, - train_val: pd.DataFrame, - train_fit: pd.DataFrame, - generated_train_val: pd.DataFrame, - categorical_features: Sequence[str], -) -> tuple[list[str], list[str], pd.DataFrame]: - model = CatBoostClassifier() - model.load_model(str(selector_model_path)) - feature_names = list(model.feature_names_) - pvc_importances = model.get_feature_importance(type="PredictionValuesChange") - scores = dict(zip(feature_names, [float(x) for x in pvc_importances], strict=False)) - if ranking_method == "shap_blend": - shap_scores = _selector_shap_scores( - model=model, - pool_features=pool_features, - generated_features=generated_features, - train_val=train_val, - generated_train_val=generated_train_val, - categorical_features=categorical_features, - max_rows=shap_rows, - ) - scores = _blend_rank_scores(scores, shap_scores) - - generated = set(generated_features) - generated_ranked = [ - feature - for feature, score in sorted( - scores.items(), - key=lambda item: float(item[1]), - reverse=True, - ) - if feature in generated and float(score) > 0.0 - ] - pool_ranked, ranking_diagnostics = _rank_pool_features( - pool_features=pool_features, - selector_scores=scores, - train_fit=train_fit, - train_val=train_val, - categorical_features=categorical_features, - ) - missing = [feature for feature in generated_features if feature not in feature_names] - if missing: - logger.warning("Selector model is missing {} generated features", len(missing)) - pool_missing = [feature for feature in pool_features if feature not in feature_names] - if pool_missing: - logger.warning("Selector model is missing {} pool features", len(pool_missing)) - logger.info( - "Loaded {} ranked pool features and {} ranked generated features from {} using {}", - len(pool_ranked), - len(generated_ranked), - selector_model_path, - ranking_method, - ) - return pool_ranked, generated_ranked, ranking_diagnostics - - -def _rank_pool_features( - *, - pool_features: Sequence[str], - selector_scores: Mapping[str, float], - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - categorical_features: Sequence[str], -) -> tuple[list[str], pd.DataFrame]: - """Rank base pool features even when the selector model omits them. - - Earlier TabPrep selector runs could produce a strong generated-feature - ranking while leaving ``pool_ranking`` empty. The champion reopen cases - need an explicit ranking for the 93-feature pool, so we blend selector PVC - or SHAP importance with leakage-safe univariate diagnostics fitted on the - train-fit slice and evaluated on the temporal validation slice. - """ - pool = [feature for feature in dict.fromkeys(pool_features) if feature in train_val.columns] - fallback = _pool_feature_diagnostics( - pool_features=pool, - train_fit=train_fit, - train_val=train_val, - categorical_features=categorical_features, - ) - selector_norm = _minmax_by_feature( - {feature: max(0.0, float(selector_scores.get(feature, 0.0))) for feature in pool} - ) - fallback_norm = _minmax_by_feature( - {row["feature"]: float(row["fallback_score"]) for row in fallback} - ) - - rows: list[dict[str, Any]] = [] - has_selector_signal = any(float(selector_scores.get(feature, 0.0)) > 0 for feature in pool) - for row in fallback: - feature = str(row["feature"]) - selector_component = float(selector_norm.get(feature, 0.0)) - fallback_component = float(fallback_norm.get(feature, 0.0)) - if has_selector_signal: - blended = 0.65 * selector_component + 0.35 * fallback_component - source = "selector_blend" - else: - blended = fallback_component - source = "fallback_univariate" - rows.append( - { - **row, - "selector_score": float(selector_scores.get(feature, 0.0)), - "selector_component": selector_component, - "fallback_component": fallback_component, - "ranking_score": blended, - "ranking_source": source, - } - ) - - diagnostics = pd.DataFrame(rows).sort_values( - ["ranking_score", "selector_score", "fallback_score", "feature"], - ascending=[False, False, False, True], - kind="mergesort", - ) - ranked = diagnostics.loc[diagnostics["ranking_score"] > 0, "feature"].astype(str).tolist() - if not ranked: - raise ValueError("Unable to rank any pool features for pooltop cases.") - return ranked, diagnostics.reset_index(drop=True) - - -def _pool_feature_diagnostics( - *, - pool_features: Sequence[str], - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - categorical_features: Sequence[str], -) -> list[dict[str, Any]]: - y_fit = train_fit[TARGET].astype(int) - y_val = train_val[TARGET].astype(int) - categorical = set(categorical_features) - rows: list[dict[str, Any]] = [] - for feature in pool_features: - if feature not in train_fit.columns or feature not in train_val.columns: - continue - if feature in categorical or train_fit[feature].dtype == "object": - val_score = _encode_categorical_from_fit( - train_fit[feature], - y_fit, - train_val[feature], - ) - drift = _categorical_psi(train_fit[feature], train_val[feature]) - feature_type = "categorical" - else: - val_score = _numeric_validation_score(train_fit[feature], train_val[feature]) - drift = _numeric_psi(train_fit[feature], train_val[feature]) - feature_type = "numeric" - score_values = pd.Series(val_score, index=train_val.index).astype(float) - auc_strength = _safe_auc_strength(y_val, score_values) - abs_corr = _safe_abs_corr(score_values, y_val) - iv = _information_value(train_val[feature], y_val, categorical=feature in categorical) - coverage = float(train_val[feature].notna().mean()) - fallback_score = ( - 0.45 * auc_strength - + 0.25 * np.sqrt(min(max(iv, 0.0), 1.0)) - + 0.20 * abs_corr - + 0.10 * coverage - - 0.15 * min(max(drift, 0.0), 1.0) - ) - rows.append( - { - "feature": str(feature), - "feature_type": feature_type, - "auc_strength": float(auc_strength), - "abs_corr": float(abs_corr), - "iv": float(iv), - "coverage": coverage, - "psi_fit_to_val": float(drift), - "fallback_score": float(max(fallback_score, 0.0)), - } - ) - return rows - - -def _numeric_validation_score(fit: pd.Series, val: pd.Series) -> pd.Series: - fit_num = pd.to_numeric(fit, errors="coerce") - val_num = pd.to_numeric(val, errors="coerce") - fill = float(fit_num.median()) if fit_num.notna().any() else 0.0 - if not np.isfinite(fill): - fill = 0.0 - return val_num.replace([np.inf, -np.inf], np.nan).fillna(fill) - - -def _encode_categorical_from_fit( - fit: pd.Series, - y_fit: pd.Series, - val: pd.Series, - *, - smoothing: float = 20.0, -) -> pd.Series: - global_mean = float(y_fit.mean()) - key_fit = fit.astype("string").fillna("__MISSING__") - grouped = ( - pd.DataFrame({"key": key_fit, "target": y_fit.astype(float)}) - .groupby("key")["target"] - .agg(["mean", "count"]) - ) - smoothed = (grouped["mean"] * grouped["count"] + global_mean * float(smoothing)) / ( - grouped["count"] + float(smoothing) - ) - key_val = val.astype("string").fillna("__MISSING__") - return key_val.map(smoothed.to_dict()).fillna(global_mean).astype(float) - - -def _safe_auc_strength(y_true: pd.Series, score: pd.Series) -> float: - y = y_true.astype(int).to_numpy() - values = pd.to_numeric(score, errors="coerce").replace([np.inf, -np.inf], np.nan) - if len(np.unique(y)) < 2 or int(values.nunique(dropna=True)) < 2: - return 0.0 - fill = float(values.median()) if values.notna().any() else 0.0 - if not np.isfinite(fill): - fill = 0.0 - auc = float(roc_auc_score(y, values.fillna(fill).to_numpy(dtype=float))) - return float(max(auc, 1.0 - auc) * 2.0 - 1.0) - - -def _safe_abs_corr(score: pd.Series, y_true: pd.Series) -> float: - values = pd.to_numeric(score, errors="coerce").replace([np.inf, -np.inf], np.nan) - fill = float(values.median()) if values.notna().any() else 0.0 - if not np.isfinite(fill): - fill = 0.0 - x = values.fillna(fill).to_numpy(dtype=float) - y = y_true.astype(float).to_numpy() - if np.std(x) <= 1e-12 or np.std(y) <= 1e-12: - return 0.0 - corr = float(np.corrcoef(x, y)[0, 1]) - return float(abs(corr)) if np.isfinite(corr) else 0.0 - - -def _information_value(series: pd.Series, y_true: pd.Series, *, categorical: bool) -> float: - y = y_true.astype(int) - if categorical: - bins = series.astype("string").fillna("__MISSING__") - top = set(bins.value_counts(dropna=False).head(30).index) - bins = bins.where(bins.isin(top), "__OTHER__") - else: - numeric = pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan) - if int(numeric.nunique(dropna=True)) < 2: - return 0.0 - try: - bins = pd.qcut( - numeric.rank(method="first"), q=min(10, numeric.nunique()), duplicates="drop" - ) - except ValueError: - return 0.0 - bins = bins.astype("string").fillna("__MISSING__") - frame = pd.DataFrame({"bin": bins, "target": y}) - grouped = frame.groupby("bin", observed=False)["target"].agg(["sum", "count"]) - bad = grouped["sum"].astype(float) + 0.5 - good = (grouped["count"] - grouped["sum"]).astype(float) + 0.5 - bad_dist = bad / bad.sum() - good_dist = good / good.sum() - iv = ((bad_dist - good_dist) * np.log(bad_dist / good_dist)).sum() - return float(iv) if np.isfinite(iv) else 0.0 - - -def _numeric_psi(fit: pd.Series, val: pd.Series) -> float: - fit_num = pd.to_numeric(fit, errors="coerce").replace([np.inf, -np.inf], np.nan).dropna() - val_num = pd.to_numeric(val, errors="coerce").replace([np.inf, -np.inf], np.nan).dropna() - if len(fit_num) < 10 or len(val_num) < 10 or int(fit_num.nunique()) < 2: - return 0.0 - quantiles = np.linspace(0.0, 1.0, 11) - edges = np.unique(np.nanquantile(fit_num.to_numpy(dtype=float), quantiles)) - if len(edges) < 3: - return 0.0 - edges[0] = -np.inf - edges[-1] = np.inf - expected, _ = np.histogram(fit_num, bins=edges) - actual, _ = np.histogram(val_num, bins=edges) - return _psi_from_counts(expected, actual) - - -def _categorical_psi(fit: pd.Series, val: pd.Series) -> float: - fit_key = fit.astype("string").fillna("__MISSING__") - val_key = val.astype("string").fillna("__MISSING__") - top = set(fit_key.value_counts(dropna=False).head(30).index) - fit_key = fit_key.where(fit_key.isin(top), "__OTHER__") - val_key = val_key.where(val_key.isin(top), "__OTHER__") - categories = sorted(set(fit_key.unique()) | set(val_key.unique())) - expected = np.asarray([(fit_key == cat).sum() for cat in categories], dtype=float) - actual = np.asarray([(val_key == cat).sum() for cat in categories], dtype=float) - return _psi_from_counts(expected, actual) - - -def _psi_from_counts(expected: np.ndarray, actual: np.ndarray) -> float: - eps = 1e-6 - expected_pct = np.clip(expected / max(float(expected.sum()), eps), eps, 1.0) - actual_pct = np.clip(actual / max(float(actual.sum()), eps), eps, 1.0) - psi = ((actual_pct - expected_pct) * np.log(actual_pct / expected_pct)).sum() - return float(psi) if np.isfinite(psi) else 0.0 - - -def _minmax_by_feature(scores: Mapping[str, float]) -> dict[str, float]: - clean = {feature: float(score) for feature, score in scores.items() if np.isfinite(score)} - if not clean: - return {} - values = np.asarray(list(clean.values()), dtype=float) - low = float(values.min()) - high = float(values.max()) - if high <= low: - return {feature: 1.0 if score > 0 else 0.0 for feature, score in clean.items()} - return {feature: (score - low) / (high - low) for feature, score in clean.items()} - - -def _selector_shap_scores( - *, - model: CatBoostClassifier, - pool_features: Sequence[str], - generated_features: Sequence[str], - train_val: pd.DataFrame, - generated_train_val: pd.DataFrame, - categorical_features: Sequence[str], - max_rows: int, -) -> dict[str, float]: - selector_features = list(model.feature_names_) - selected_generated = [feature for feature in generated_features if feature in selector_features] - base_features = [feature for feature in pool_features if feature in selector_features] - frame = _combine_features(train_val, base_features, generated_train_val[selected_generated]) - if max_rows > 0 and len(frame) > max_rows: - frame = frame.sample(n=max_rows, random_state=42).sort_index() - cat_features = [feature for feature in categorical_features if feature in selector_features] - x_cb = _prepare_catboost_frame(frame, selector_features, cat_features) - shap_pool = Pool(x_cb, cat_features=cat_features) - shap_raw = model.get_feature_importance(type="ShapValues", data=shap_pool) - shap_values = np.asarray(shap_raw[:, :-1], dtype=float) - mean_abs = np.abs(shap_values).mean(axis=0) - logger.info("Computed selector SHAP scores on {} rows", len(x_cb)) - return dict(zip(selector_features, [float(x) for x in mean_abs], strict=False)) - - -def _blend_rank_scores( - pvc_scores: Mapping[str, float], - shap_scores: Mapping[str, float], -) -> dict[str, float]: - features = sorted(set(pvc_scores) | set(shap_scores)) - pvc_rank = { - feature: rank - for rank, feature in enumerate( - sorted(features, key=lambda f: float(pvc_scores.get(f, 0.0)), reverse=True), - start=1, - ) - } - shap_rank = { - feature: rank - for rank, feature in enumerate( - sorted(features, key=lambda f: float(shap_scores.get(f, 0.0)), reverse=True), - start=1, - ) - } - return { - feature: 1.0 / np.sqrt(float(pvc_rank[feature]) * float(shap_rank[feature])) - for feature in features - } - - -def _business_ranking(ranking: Sequence[str], manifest: pd.DataFrame) -> list[str]: - if manifest.empty: - return list(ranking) - source_by_feature = dict(zip(manifest["feature"], manifest["source_features"], strict=False)) - generator_by_feature = dict(zip(manifest["feature"], manifest["generator"], strict=False)) - selected: list[str] = [] - for feature in ranking: - sources = str(source_by_feature.get(feature, "")).split("|") - if any(token in source for source in sources for token in BUSINESS_SOURCE_DENY_TOKENS): - continue - if str(generator_by_feature.get(feature, "")) == "rsfc" and len(sources) > 3: - continue - selected.append(feature) - return selected - - -def _build_case_features( - *, - cases: Sequence[str], - core_features: Sequence[str], - catboost_features: Sequence[str], - pool_features: Sequence[str], - pool_ranking: Sequence[str], - woe_features: Sequence[str], - generated_ranking: Sequence[str], - business_ranking: Sequence[str], -) -> dict[str, list[str]]: - out: dict[str, list[str]] = {} - for case in cases: - if case == "core42": - features = list(core_features) - elif case == "catboost44": - features = list(catboost_features) - elif case == "core42_woe": - features = [*core_features, *woe_features] - elif case == "pool93": - features = list(pool_features) - elif case == "pool93_woe": - features = [*pool_features, *woe_features] - elif case.startswith("pooltop") and "_tab" in case: - x, y = _pool_tab_case_sizes(case, generated_token="_tab") - features = [ - *_take_ranked(pool_ranking, x, case=case, label="pool"), - *_take_ranked(generated_ranking, y, case=case, label="generated"), - ] - elif case.startswith("pooltop") and "_business" in case: - x, y = _pool_tab_case_sizes(case, generated_token="_business") - features = [ - *_take_ranked(pool_ranking, x, case=case, label="pool"), - *_take_ranked(business_ranking, y, case=case, label="business generated"), - ] - elif case.startswith("pooltop") and case.endswith("_woe"): - x = _case_k(case.removesuffix("_woe"), prefix="pooltop") - features = [*_take_ranked(pool_ranking, x, case=case, label="pool"), *woe_features] - elif case.startswith("pooltop"): - x = _case_k(case, prefix="pooltop") - features = list(_take_ranked(pool_ranking, x, case=case, label="pool")) - elif case.startswith("pool93_top"): - k = _case_k(case, prefix="pool93_top") - features = [ - *pool_features, - *_take_ranked(generated_ranking, k, case=case, label="generated"), - ] - elif case.startswith("pool93_woe_top"): - k = _case_k(case, prefix="pool93_woe_top") - features = [ - *pool_features, - *woe_features, - *_take_ranked(generated_ranking, k, case=case, label="generated"), - ] - elif case.startswith("pool93_business"): - k = _case_k(case, prefix="pool93_business") - features = [ - *pool_features, - *_take_ranked(business_ranking, k, case=case, label="business generated"), - ] - elif case.startswith("core42_business"): - k = _case_k(case, prefix="core42_business") - features = [ - *core_features, - *_take_ranked(business_ranking, k, case=case, label="business generated"), - ] - else: - raise ValueError(f"Unknown feature-selection case: {case}") - out[case] = list(dict.fromkeys(features)) - return out - - -def _pool_tab_case_sizes(case: str, *, generated_token: str) -> tuple[int, int]: - pool_part, generated_part = case.split(generated_token, maxsplit=1) - x = _case_k(pool_part, prefix="pooltop") - y = int(generated_part) - return x, y - - -def _case_k(case: str, *, prefix: str) -> int: - raw = case.removeprefix(prefix) - if not raw: - raise ValueError(f"Case {case!r} requires a numeric suffix") - return int(raw) - - -def _take_ranked( - ranking: Sequence[str], - k: int, - *, - case: str, - label: str, -) -> list[str]: - if k <= 0: - return [] - if len(ranking) < k: - raise ValueError( - f"Case {case!r} requested top {k} {label} features, " - f"but only {len(ranking)} ranked features are available." - ) - return list(ranking[:k]) - - -def _write_selection_manifest( - *, - config: Mapping[str, Any], - run_tag: str, - seed: int, - tabprep_seed: int, - cases: Mapping[str, Sequence[str]], - ranking: Sequence[str], - pool_ranking: Sequence[str], - business_ranking: Sequence[str], - manifest: pd.DataFrame, - ranking_diagnostics: pd.DataFrame, -) -> None: - report_dir = _case_dir( - Path(config["output"]["report_dir"]), - run_tag=run_tag, - case_name="selection_manifest", - seed=seed, - ) - atomic_write_json( - report_dir / "selection_manifest.json", - { - "run_tag": run_tag, - "seed": seed, - "tabprep_seed": tabprep_seed, - "cases": {name: list(features) for name, features in cases.items()}, - "pool_ranking_top200": list(pool_ranking[:200]), - "generated_ranking_top200": list(ranking[:200]), - "business_ranking_top200": list(business_ranking[:200]), - }, - ) - atomic_write_parquet(manifest, report_dir / "generated_feature_manifest.parquet") - if not ranking_diagnostics.empty: - atomic_write_parquet(ranking_diagnostics, report_dir / "pool_ranking_diagnostics.parquet") - - -def _runtime_paths(config: Mapping[str, Any], *, run_tag: str, seed: int) -> dict[str, Path]: - model_dir = _case_dir( - Path(config["output"]["model_dir"]), - run_tag=run_tag, - case_name="runtime", - seed=seed, - ) - _validate_output_roots([model_dir]) - return { - "status": model_dir / "runtime_status.json", - "checkpoints": model_dir / "checkpoints", - } - - -def _write_runtime( - *, - runtime_paths: Mapping[str, Path], - phase: str, - state: str, - run_tag: str, - extra: Mapping[str, Any] | None = None, -) -> None: - write_runtime_status( - "champion_reopen_feature_selection", - phase=phase, - state=state, - run_tag=run_tag, - status_path=runtime_paths["status"], - extra=dict(extra or {}), - ) - - -def _write_checkpoint( - *, - runtime_paths: Mapping[str, Path], - checkpoint_name: str, - payload: Mapping[str, Any], -) -> None: - write_runtime_checkpoint( - "champion_reopen_feature_selection", - checkpoint_name, - dict(payload), - checkpoint_dir=runtime_paths["checkpoints"], - ) - - -def _run_selected_case( - *, - case_name: str, - run_tag: str, - seed: int, - tabprep_seed: int, - config: Mapping[str, Any], - train_fit: pd.DataFrame, - train_val: pd.DataFrame, - calibration: pd.DataFrame, - test: pd.DataFrame, - base_features: Sequence[str], - categorical_features: Sequence[str], - generated_train_fit: pd.DataFrame, - generated_train_val: pd.DataFrame, - generated_calibration: pd.DataFrame, - generated_test: pd.DataFrame, - transformer_summary: Mapping[str, Any], - selection_sources: Mapping[str, Any], -) -> dict[str, Any]: - model_dir = _case_dir( - Path(config["output"]["model_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - data_dir = _case_dir( - Path(config["output"]["data_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - report_dir = _case_dir( - Path(config["output"]["report_dir"]), run_tag=run_tag, case_name=case_name, seed=seed - ) - _validate_output_roots([model_dir, data_dir, report_dir]) - - generated_features = list(generated_train_fit.columns) - model_features = [*base_features, *generated_features] - cat_features = [feature for feature in categorical_features if feature in model_features] - - x_train = _combine_features(train_fit, base_features, generated_train_fit) - x_val = _combine_features(train_val, base_features, generated_train_val) - x_cal = _combine_features(calibration, base_features, generated_calibration) - x_test = _combine_features(test, base_features, generated_test) - - x_train_cb = _prepare_catboost_frame(x_train, model_features, cat_features) - x_val_cb = _prepare_catboost_frame(x_val, model_features, cat_features) - x_cal_cb = _prepare_catboost_frame(x_cal, model_features, cat_features) - x_test_cb = _prepare_catboost_frame(x_test, model_features, cat_features) - - params = _model_params(config, model_features=model_features, seed=seed) - model, train_metrics = train_catboost_default( - x_train_cb, - train_fit[TARGET].astype(int), - x_val_cb, - train_val[TARGET].astype(int), - X_test=x_test_cb, - y_test=test[TARGET].astype(int), - cat_features=cat_features, - params=params, - ) - raw_cal = model.predict_proba(x_cal_cb)[:, 1] - raw_test = model.predict_proba(x_test_cb)[:, 1] - calibrator, calibration_report = _select_calibrator( - method=str(config["calibration"]["method"]), - candidates=[str(item) for item in config["calibration"]["candidates"]], - y_cal=calibration[TARGET].astype(int).to_numpy(), - raw_cal=raw_cal, - ) - calibrated_test = np.asarray(_apply_calibrator(calibrator, raw_test), dtype=float) - calibrated_cal = np.asarray(_apply_calibrator(calibrator, raw_cal), dtype=float) - test_metrics = classification_metrics(test[TARGET].astype(int).to_numpy(), calibrated_test) - cal_metrics = classification_metrics( - calibration[TARGET].astype(int).to_numpy(), - calibrated_cal, - ) - fairness = _build_fairness_report(test, y_prob=calibrated_test, config=config) - - model_dir.mkdir(parents=True, exist_ok=True) - model_path = model_dir / "pd_selected_tabprep_challenger.cbm" - calibrator_path = model_dir / "pd_selected_tabprep_calibrator.pkl" - prediction_path = data_dir / "test_predictions.parquet" - status_path = model_dir / "selected_feature_training_status.json" - model.save_model(str(model_path)) - atomic_write_pickle(calibrator_path, calibrator) - predictions = _prediction_frame( - test, - raw_prob=raw_test, - calibrated_prob=calibrated_test, - case_name=case_name, - seed=seed, - ) - atomic_write_parquet(predictions, prediction_path) - if not fairness.empty: - atomic_write_parquet(fairness, report_dir / "fairness_report.parquet") - feature_importance = _feature_importance_frame(model) - atomic_write_parquet(feature_importance, report_dir / "feature_importance.parquet") - status = { - "run_tag": run_tag, - "case_name": case_name, - "seed": seed, - "tabprep_seed": tabprep_seed, - "model_path": str(model_path), - "calibrator_path": str(calibrator_path), - "prediction_path": str(prediction_path), - "n_base_features": len(base_features), - "n_generated_features": len(generated_features), - "n_model_features": len(model_features), - "categorical_features": cat_features, - "generated_features": generated_features, - "training_metrics": train_metrics, - "test_metrics": test_metrics, - "calibration_metrics": cal_metrics, - "calibration_selection": calibration_report, - "selection_sources": selection_sources, - "transformer_summary": transformer_summary, - } - atomic_write_json(status_path, status) - logger.info( - "{} seed {} done: test AUC={:.6f}, Brier={:.6f}, features={} generated={}", - case_name, - seed, - test_metrics["auc_roc"], - test_metrics["brier_score"], - len(model_features), - len(generated_features), - ) - return status - - -def _feature_importance_frame(model: CatBoostClassifier) -> pd.DataFrame: - return pd.DataFrame( - { - "feature": list(model.feature_names_), - "importance": model.get_feature_importance(type="PredictionValuesChange"), - } - ).sort_values("importance", ascending=False, kind="mergesort") - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/select_champion_reopen_finalists.py b/scripts/experiments/select_champion_reopen_finalists.py deleted file mode 100644 index c721ab0..0000000 --- a/scripts/experiments/select_champion_reopen_finalists.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Select CRPTO champion-reopen finalists after seed replay. - -This script is read-only over experiment outputs. It combines the seed-42 -feature-search result with seed-replay summaries, applies the promotion gates -from ``configs/experiments/champion_reopen.yaml``, and writes a ranked finalist -selection under the experiment report root. -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from statistics import mean, pstdev -from typing import Any - -import pandas as pd -import yaml - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", default="configs/experiments/champion_reopen.yaml") - parser.add_argument("--run-tag", default=None) - parser.add_argument("--top-n", type=int, default=5) - parser.add_argument( - "--require-complete", - action="store_true", - help="Fail if any configured seed-replay summary is missing.", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - config = _load_yaml(Path(args.config)) - run_tag = str(args.run_tag or config.get("run_tag", "champion-reopen-2026-06-19")) - report_root = Path(config["output"]["report_dir"]) - selection = select_finalists( - config=config, - run_tag=run_tag, - report_root=report_root, - top_n=int(args.top_n), - require_complete=bool(args.require_complete), - ) - output_dir = report_root / run_tag / "finalist_selection" - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "seed_replay_finalists.json").write_text( - json.dumps(selection, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - pd.DataFrame(selection["ranked_cases"]).to_csv( - output_dir / "seed_replay_ranked_cases.csv", index=False - ) - print(f"Wrote finalist selection: {output_dir / 'seed_replay_finalists.json'}") - print("selected:", ", ".join(row["case_name"] for row in selection["selected_finalists"])) - - -def select_finalists( - *, - config: dict[str, Any], - run_tag: str, - report_root: Path, - top_n: int, - require_complete: bool, -) -> dict[str, Any]: - champion = dict(config["champion_reopen"]["pd_promotion_gates"]) - replay_cases = list(config["champion_reopen"]["seed_replay_cases"]) - replay_seeds = list( - config["champion_reopen"].get( - "seed_replay_seeds", config.get("seeds", [42, 52, 62, 72, 82]) - ) - ) - expected_seeds = [42, *[int(seed) for seed in replay_seeds if int(seed) != 42]] - rows, missing = _load_seed_rows( - report_root=report_root, run_tag=run_tag, cases=replay_cases, seeds=expected_seeds - ) - if require_complete and missing: - raise SystemExit("Missing seed summaries: " + ", ".join(missing)) - ranked = _rank_cases(rows, champion=champion, expected_seed_count=len(expected_seeds)) - selected = _select_top_cases(ranked, top_n=top_n) - return { - "run_tag": run_tag, - "expected_seeds": expected_seeds, - "available_rows": len(rows), - "missing": missing, - "champion_metrics": { - "auc_roc": champion["champion_auc"], - "brier_score": champion["champion_brier"], - "ece": champion["champion_ece"], - }, - "gates": champion, - "selected_finalists": selected, - "ranked_cases": ranked, - } - - -def _load_seed_rows( - *, - report_root: Path, - run_tag: str, - cases: list[str], - seeds: list[int], -) -> tuple[list[dict[str, Any]], list[str]]: - rows: list[dict[str, Any]] = [] - missing: list[str] = [] - feature_summary = ( - report_root - / f"{run_tag}__feature_search" - / "summary" - / "seed_42" - / "selected_feature_experiment_summary.json" - ) - if feature_summary.exists(): - rows.extend( - _rows_from_summary( - feature_summary, allowed_cases=set(cases), source_stage="feature_search" - ) - ) - else: - missing.append(str(feature_summary)) - for seed in seeds: - if int(seed) == 42: - continue - summary = ( - report_root - / f"{run_tag}__seed_replay" - / "summary" - / f"seed_{int(seed)}" - / "selected_feature_experiment_summary.json" - ) - if summary.exists(): - rows.extend( - _rows_from_summary(summary, allowed_cases=set(cases), source_stage="seed_replay") - ) - else: - missing.append(str(summary)) - return rows, missing - - -def _rows_from_summary( - path: Path, *, allowed_cases: set[str], source_stage: str -) -> list[dict[str, Any]]: - payload = json.loads(path.read_text(encoding="utf-8")) - rows: list[dict[str, Any]] = [] - for item in payload.get("results", []): - case = str(item["case_name"]) - if case not in allowed_cases: - continue - metrics = dict(item["test_metrics"]) - rows.append( - { - "case_name": case, - "seed": int(item["seed"]), - "source_stage": source_stage, - "n_model_features": int(item["n_model_features"]), - "n_generated_features": int(item["n_generated_features"]), - "auc_roc": float(metrics["auc_roc"]), - "brier_score": float(metrics["brier_score"]), - "ece": float(metrics["ece"]), - } - ) - return rows - - -def _rank_cases( - rows: list[dict[str, Any]], - *, - champion: dict[str, Any], - expected_seed_count: int, -) -> list[dict[str, Any]]: - by_case: dict[str, list[dict[str, Any]]] = {} - for row in rows: - by_case.setdefault(row["case_name"], []).append(row) - ranked: list[dict[str, Any]] = [] - for case, case_rows in sorted(by_case.items()): - aucs = [row["auc_roc"] for row in case_rows] - briers = [row["brier_score"] for row in case_rows] - eces = [row["ece"] for row in case_rows] - seeds = sorted({int(row["seed"]) for row in case_rows}) - mean_auc = mean(aucs) - mean_brier = mean(briers) - mean_ece = mean(eces) - auc_std = pstdev(aucs) if len(aucs) > 1 else 0.0 - delta_auc = mean_auc - float(champion["champion_auc"]) - delta_brier = mean_brier - float(champion["champion_brier"]) - delta_ece = mean_ece - float(champion["champion_ece"]) - complete = len(seeds) >= expected_seed_count - paper_facing = case in {"pool93", "catboost44"} or "business" in case - gate_pass = bool( - complete - and delta_auc >= float(champion["min_auc_delta_mean"]) - and auc_std <= float(champion["max_auc_seed_std"]) - and delta_brier <= float(champion["max_brier_increase"]) - and delta_ece <= float(champion["max_ece_increase"]) - ) - score = ( - delta_auc - - max(0.0, auc_std - float(champion["max_auc_seed_std"])) * 2.0 - - max(0.0, delta_brier) * 10.0 - - max(0.0, delta_ece) * 0.5 - + (0.00025 if paper_facing else 0.0) - ) - ranked.append( - { - "case_name": case, - "seed_count": len(seeds), - "seeds": ",".join(str(seed) for seed in seeds), - "complete": complete, - "paper_facing": paper_facing, - "gate_pass": gate_pass, - "mean_auc_roc": mean_auc, - "delta_auc_roc": delta_auc, - "std_auc_roc": auc_std, - "mean_brier_score": mean_brier, - "delta_brier_score": delta_brier, - "mean_ece": mean_ece, - "delta_ece": delta_ece, - "min_auc_roc": min(aucs), - "max_auc_roc": max(aucs), - "n_model_features": int(case_rows[0]["n_model_features"]), - "n_generated_features": int(case_rows[0]["n_generated_features"]), - "selection_score": score, - } - ) - ranked.sort( - key=lambda row: ( - bool(row["gate_pass"]), - float(row["selection_score"]), - float(row["mean_auc_roc"]), - ), - reverse=True, - ) - for index, row in enumerate(ranked, start=1): - row["rank"] = index - return ranked - - -def _select_top_cases(ranked: list[dict[str, Any]], *, top_n: int) -> list[dict[str, Any]]: - if top_n <= 0: - return [] - gate_pass = [row for row in ranked if row["gate_pass"]] - selected = list(gate_pass[:top_n]) - if len(selected) < top_n: - seen = {row["case_name"] for row in selected} - for row in ranked: - if row["case_name"] in seen: - continue - selected.append(row) - seen.add(row["case_name"]) - if len(selected) >= top_n: - break - return selected - - -def _load_yaml(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise TypeError(f"Expected mapping config: {path}") - return payload - - -if __name__ == "__main__": - main() diff --git a/scripts/experiments/setup_cuopt_26_06_env.sh b/scripts/experiments/setup_cuopt_26_06_env.sh deleted file mode 100644 index 768066a..0000000 --- a/scripts/experiments/setup_cuopt_26_06_env.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -REPO_ROOT="${REPO_ROOT:-/mnt/c/Users/carlos/Documents/Paper_CRPTO}" -CUOPT_ENV_DIR="${CUOPT_ENV_DIR:-/home/eigenlinux/.venvs/crpto-cuopt-26-6}" -PYTHON_BIN="${PYTHON_BIN:-python3.11}" - -cd "${REPO_ROOT}" - -if command -v uv >/dev/null 2>&1; then - uv venv --seed --python "${PYTHON_BIN}" "${CUOPT_ENV_DIR}" -else - "${PYTHON_BIN}" -m venv "${CUOPT_ENV_DIR}" -fi - -if ! "${CUOPT_ENV_DIR}/bin/python" -m pip --version >/dev/null 2>&1; then - "${CUOPT_ENV_DIR}/bin/python" -m ensurepip --upgrade -fi -"${CUOPT_ENV_DIR}/bin/python" -m pip install --upgrade pip wheel setuptools -"${CUOPT_ENV_DIR}/bin/python" -m pip install \ - --extra-index-url=https://pypi.nvidia.com \ - "cuopt-cu13==26.6.*" -"${CUOPT_ENV_DIR}/bin/python" -m pip install \ - --no-deps -e . -"${CUOPT_ENV_DIR}/bin/python" -m pip install \ - "pandas>=2.3,<4" \ - "numpy>=2.0,<3" \ - "pyarrow>=18" \ - "scipy>=1.14,<2" \ - "pyyaml>=6.0" \ - "loguru>=0.7" \ - "pyomo>=6.10" \ - "highspy>=1.10" \ - "tqdm>=4.66" - -"${CUOPT_ENV_DIR}/bin/python" - <<'PY' -import json -import sys - -modules = {} -for name in ["cuopt", "cudf", "cupy", "pandas", "pyomo", "highspy"]: - module = __import__(name) - modules[name] = getattr(module, "__version__", "ok") - -print(json.dumps({"python": sys.executable, "modules": modules}, indent=2)) -PY - -echo "CUOPT_ENV_READY ${CUOPT_ENV_DIR}" diff --git a/scripts/experiments/watch_and_run_pool93_terminal_bound_search.sh b/scripts/experiments/watch_and_run_pool93_terminal_bound_search.sh deleted file mode 100644 index ead1406..0000000 --- a/scripts/experiments/watch_and_run_pool93_terminal_bound_search.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -TRIGGER_RUN_TAG="${1:-champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure}" -TERMINAL_RUN_TAG="${2:-champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal}" -POLL_SECONDS="${3:-180}" -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -TRIGGER_STATUS="${ROOT_DIR}/models/experiments/champion_reopen/${TRIGGER_RUN_TAG}/portfolio/runtime_status.json" -TERMINAL_LOG_DIR="${ROOT_DIR}/reports/run_logs/champion_reopen/${TERMINAL_RUN_TAG}" -TERMINAL_LOG="${TERMINAL_LOG_DIR}/local_exact_refine_bound_terminal12.log" -POSTPROCESS_LOG="${TERMINAL_LOG_DIR}/postprocess_and_consolidate.log" - -mkdir -p "${TERMINAL_LOG_DIR}" - -while true; do - if "${ROOT_DIR}/.venv-champion-search/bin/python" - "${TRIGGER_STATUS}" <<'PY' -import json -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -if not path.exists(): - raise SystemExit(1) -status = json.loads(path.read_text(encoding="utf-8")) -complete = status.get("phase") == "selection_complete" and status.get("state") == "completed" -raise SystemExit(0 if complete else 1) -PY - then - break - fi - sleep "${POLL_SECONDS}" -done - -{ - date -Is - echo "Starting terminal bound search after ${TRIGGER_RUN_TAG}" -} >> "${POSTPROCESS_LOG}" 2>&1 - -PYTHONUNBUFFERED=1 HIGHS_NATIVE_FALLBACK_SCIPY=1 \ - "${ROOT_DIR}/.venv-champion-search/bin/python" -u \ - "${ROOT_DIR}/scripts/search/run_pool93_ijds_local_refinement.py" \ - --run-tag "${TERMINAL_RUN_TAG}" \ - --profile claim_bound_terminal \ - --exact-threads 1 \ - --parallel-workers 12 \ - --checkpoint-every 25 \ - >> "${TERMINAL_LOG}" 2>&1 - -{ - date -Is - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_claim_governance.py" \ - --run-tag "${TERMINAL_RUN_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_frontier_claim_table.py" \ - --run-tag "${TERMINAL_RUN_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_consolidated_frontier.py" \ - --output-tag champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure \ - --run-tag "${TRIGGER_RUN_TAG}" \ - --run-tag "${TERMINAL_RUN_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_consolidated_governance.py" \ - --consolidated-tag champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_body_allocation_audit.py" \ - --consolidated-tag champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive \ - --threads 1 \ - --solver-backend highspy - date -Is -} >> "${POSTPROCESS_LOG}" 2>&1 diff --git a/scripts/experiments/watch_pool93_ijds_consolidated_postprocess.sh b/scripts/experiments/watch_pool93_ijds_consolidated_postprocess.sh deleted file mode 100644 index 1d4e0b1..0000000 --- a/scripts/experiments/watch_pool93_ijds_consolidated_postprocess.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -TRIGGER_RUN_TAG="${1:?trigger run tag required}" -POLL_SECONDS="${2:-300}" -OUTPUT_TAG="${3:-champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-with-bound-closure}" -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -STATUS_PATH="${ROOT_DIR}/models/experiments/champion_reopen/${TRIGGER_RUN_TAG}/portfolio/runtime_status.json" -LOG_DIR="${ROOT_DIR}/reports/run_logs/champion_reopen/${TRIGGER_RUN_TAG}" -LOG_PATH="${LOG_DIR}/consolidated_postprocess.log" - -mkdir -p "${LOG_DIR}" - -while true; do - if "${ROOT_DIR}/.venv-champion-search/bin/python" - "${STATUS_PATH}" <<'PY' -import json -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -if not path.exists(): - raise SystemExit(1) -status = json.loads(path.read_text(encoding="utf-8")) -complete = status.get("phase") == "selection_complete" and status.get("state") == "completed" -raise SystemExit(0 if complete else 1) -PY - then - break - fi - sleep "${POLL_SECONDS}" -done - -{ - date -Is - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_consolidated_frontier.py" \ - --output-tag "${OUTPUT_TAG}" \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext \ - --run-tag "${TRIGGER_RUN_TAG}" - date -Is -} >> "${LOG_PATH}" 2>&1 diff --git a/scripts/experiments/watch_pool93_ijds_definitive_consolidated_postprocess.sh b/scripts/experiments/watch_pool93_ijds_definitive_consolidated_postprocess.sh deleted file mode 100644 index 1e65caa..0000000 --- a/scripts/experiments/watch_pool93_ijds_definitive_consolidated_postprocess.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -TRIGGER_RUN_TAG="${1:?trigger run tag required}" -POLL_SECONDS="${2:-300}" -OUTPUT_TAG="${3:-champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive}" -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -STATUS_PATH="${ROOT_DIR}/models/experiments/champion_reopen/${TRIGGER_RUN_TAG}/portfolio/runtime_status.json" -LOG_DIR="${ROOT_DIR}/reports/run_logs/champion_reopen/${TRIGGER_RUN_TAG}" -LOG_PATH="${LOG_DIR}/consolidated_definitive_postprocess.log" - -mkdir -p "${LOG_DIR}" - -while true; do - if "${ROOT_DIR}/.venv-champion-search/bin/python" - "${STATUS_PATH}" <<'PY' -import json -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -if not path.exists(): - raise SystemExit(1) -status = json.loads(path.read_text(encoding="utf-8")) -complete = status.get("phase") == "selection_complete" and status.get("state") == "completed" -raise SystemExit(0 if complete else 1) -PY - then - break - fi - sleep "${POLL_SECONDS}" -done - -{ - date -Is - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_consolidated_frontier.py" \ - --output-tag "${OUTPUT_TAG}" \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure \ - --run-tag champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure \ - --run-tag "${TRIGGER_RUN_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_consolidated_governance.py" \ - --consolidated-tag "${OUTPUT_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_body_allocation_audit.py" \ - --consolidated-tag "${OUTPUT_TAG}" \ - --threads 1 \ - --solver-backend highspy - date -Is -} >> "${LOG_PATH}" 2>&1 diff --git a/scripts/experiments/watch_pool93_ijds_postprocess.sh b/scripts/experiments/watch_pool93_ijds_postprocess.sh deleted file mode 100644 index 100743a..0000000 --- a/scripts/experiments/watch_pool93_ijds_postprocess.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -RUN_TAG="${1:?run tag required}" -POLL_SECONDS="${2:-300}" -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -STATUS_PATH="${ROOT_DIR}/models/experiments/champion_reopen/${RUN_TAG}/portfolio/runtime_status.json" -LOG_DIR="${ROOT_DIR}/reports/run_logs/champion_reopen/${RUN_TAG}" -LOG_PATH="${LOG_DIR}/postprocess.log" - -mkdir -p "${LOG_DIR}" - -while true; do - if "${ROOT_DIR}/.venv-champion-search/bin/python" - "${STATUS_PATH}" <<'PY' -import json -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]) -if not path.exists(): - raise SystemExit(1) -status = json.loads(path.read_text(encoding="utf-8")) -complete = status.get("phase") == "selection_complete" and status.get("state") == "completed" -raise SystemExit(0 if complete else 1) -PY - then - break - fi - sleep "${POLL_SECONDS}" -done - -{ - date -Is - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_claim_governance.py" \ - --run-tag "${RUN_TAG}" - "${ROOT_DIR}/.venv-champion-search/bin/python" \ - "${ROOT_DIR}/scripts/search/build_pool93_ijds_frontier_claim_table.py" \ - --run-tag "${RUN_TAG}" - date -Is -} >> "${LOG_PATH}" 2>&1 diff --git a/scripts/export_dvc_metrics.py b/scripts/export_dvc_metrics.py deleted file mode 100644 index d2c8a2e..0000000 --- a/scripts/export_dvc_metrics.py +++ /dev/null @@ -1,262 +0,0 @@ -"""Export canonical KPI summaries for DVC metrics/plots.""" - -from __future__ import annotations - -import argparse -import json -import os -import pickle -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger -from sklearn.metrics import brier_score_loss, roc_auc_score, roc_curve - -from src.utils.script_helpers import load_json - -try: - from sklearn.metrics import d2_brier_score -except ImportError: # sklearn < 1.8 - d2_brier_score = None - -ROOT = Path(__file__).resolve().parents[1] -MODELS = ROOT / "models" -SCHEMA_VERSION = "2026-03-01.1" - - -def _load_pickle(path: Path) -> Any: - with open(path, "rb") as f: - return pickle.load(f) - - -def _load_json(path: Path) -> dict[str, Any]: - return load_json(path) - - -def _ece(y_true: np.ndarray, y_prob: np.ndarray, n_bins: int = 15) -> float: - bins = np.linspace(0.0, 1.0, n_bins + 1) - bin_ids = np.digitize(y_prob, bins[1:-1], right=True) - total = len(y_true) - if total == 0: - return 0.0 - - ece = 0.0 - for b in range(n_bins): - mask = bin_ids == b - n = int(mask.sum()) - if n == 0: - continue - frac = n / total - ece += frac * abs(float(y_true[mask].mean()) - float(y_prob[mask].mean())) - return float(ece) - - -def _pd_metrics() -> dict[str, float]: - preds = pd.read_parquet(ROOT / "data/processed/test_predictions.parquet") - y_true = preds["y_true"].astype(int).to_numpy() - score_col = "pd_calibrated" if "pd_calibrated" in preds.columns else "y_prob_final" - y_prob = preds[score_col].astype(float).to_numpy() - - auc = float(roc_auc_score(y_true, y_prob)) - fpr, tpr, _ = roc_curve(y_true, y_prob) - ks = float(np.max(tpr - fpr)) - metrics = { - "pd.auc": auc, - "pd.gini": 2.0 * auc - 1.0, - "pd.ks": ks, - "pd.brier": float(brier_score_loss(y_true, y_prob)), - "pd.ece": _ece(y_true, y_prob), - } - if d2_brier_score is not None: - metrics["pd.d2_brier"] = float(d2_brier_score(y_true, y_prob)) - return metrics - - -def _conformal_metrics() -> dict[str, float]: - with open(ROOT / "models/conformal_policy_status.json", encoding="utf-8") as f: - status = json.load(f) - return { - "conformal.coverage90": float(status.get("coverage_90", 0.0)), - "conformal.coverage95": float(status.get("coverage_95", 0.0)), - "conformal.avg_width90": float(status.get("avg_width_90", 0.0)), - "conformal.min_group_coverage90": float(status.get("min_group_coverage_90", 0.0)), - "conformal.overall_pass": float(int(bool(status.get("overall_pass", False)))), - } - - -def _ifrs9_metrics() -> dict[str, float]: - path = ROOT / "data/processed/ifrs9_scenario_summary.parquet" - if not path.exists(): - return {} - df = pd.read_parquet(path) - by_scenario = { - str(row["scenario"]): float(row["total_ecl"]) - for _, row in df[["scenario", "total_ecl"]].iterrows() - } - baseline = by_scenario.get("baseline", 0.0) - severe = by_scenario.get("severe", 0.0) - return { - "ifrs9.ecl_baseline": baseline, - "ifrs9.ecl_severe": severe, - "ifrs9.severe_uplift_pct": ((severe / baseline) - 1.0) * 100.0 if baseline else 0.0, - } - - -def _optimization_metrics() -> dict[str, float]: - path = ROOT / "models/pipeline_results.pkl" - if not path.exists(): - return {} - pipeline = _load_pickle(path) - if not isinstance(pipeline, dict): - raise TypeError("models/pipeline_results.pkl must contain a dict") - return { - "optimization.robust_return": float(pipeline.get("robust_return", 0.0)), - "optimization.nonrobust_return": float(pipeline.get("nonrobust_return", 0.0)), - "optimization.price_of_robustness": float(pipeline.get("price_of_robustness", 0.0)), - "optimization.robust_funded": float(pipeline.get("robust_funded", 0.0)), - "optimization.nonrobust_funded": float(pipeline.get("nonrobust_funded", 0.0)), - } - - -def _crpto_final_metrics() -> dict[str, float]: - """Expose the final CRPTO closure without overwriting operational KPIs.""" - path = ROOT / "models/final_project_promotion.json" - if not path.exists(): - return {} - - with open(path, encoding="utf-8") as f: - promotion = json.load(f) - champion = promotion.get("final_champion", {}) - conformal = promotion.get("conformal_upstream", {}).get("winner_metrics", {}) - region = promotion.get("robust_region_summary", {}) - - metrics = { - "crpto.final.robust_return": float(champion.get("realized_total_return", 0.0)), - "crpto.final.price_of_robustness": float(champion.get("price_of_robustness", 0.0)), - "crpto.final.price_of_robustness_pct": float(champion.get("price_of_robustness_pct", 0.0)), - "crpto.final.alpha01_exact_pass": float( - int(bool(champion.get("alpha01_exact_pass", False))) - ), - "crpto.final.alpha03_exact_pass": float( - int(bool(champion.get("alpha03_exact_pass", False))) - ), - "crpto.final.alpha10_exact_pass": float( - int(bool(champion.get("alpha10_exact_pass", False))) - ), - "crpto.final.alpha01_weighted_miscoverage_V": float( - champion.get("alpha01_weighted_miscoverage_V", 0.0) - ), - "crpto.final.alpha01_gamma_cp": float(champion.get("alpha01_gamma_cp", 0.0)), - "crpto.final.alpha01_violation": float(champion.get("alpha01_violation", 0.0)), - "crpto.final.robust_region_alpha01_pass_rate": float(region.get("alpha01_pass_rate", 0.0)), - "crpto.final.robust_region_n_policies": float(region.get("n_unique_policies", 0.0)), - "crpto.final.conformal_coverage90": float(conformal.get("coverage_90", 0.0)), - "crpto.final.conformal_coverage95": float(conformal.get("coverage_95", 0.0)), - "crpto.final.conformal_avg_width90": float(conformal.get("avg_width_90", 0.0)), - "crpto.final.conformal_min_group_coverage90": float( - conformal.get("min_group_coverage_90", 0.0) - ), - "crpto.final.conformal_winkler90": float(conformal.get("winkler_90", 0.0)), - } - return metrics - - -def _write_conformal_backtest_plot(out_path: Path) -> None: - df = pd.read_parquet(ROOT / "data/processed/conformal_backtest_monthly.parquet").copy() - keep = [ - "month", - "n", - "coverage_90", - "target_90", - "coverage_95", - "target_95", - "avg_width_90", - "coverage_90_roll3", - "coverage_95_roll3", - "avg_width_90_roll3", - ] - df = df[keep].sort_values("month") - df["month"] = pd.to_datetime(df["month"]).dt.strftime("%Y-%m-%d") - out_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(out_path, index=False) - - -def _write_robustness_frontier_plot(out_path: Path) -> None: - df = pd.read_parquet(ROOT / "data/processed/portfolio_robustness_frontier.parquet").copy() - keep = [ - "policy", - "risk_tolerance", - "uncertainty_aversion", - "price_of_robustness", - "price_of_robustness_pct", - "expected_return_net_point", - "worst_case_loss", - "worst_case_pd", - "point_pd", - "n_funded", - "solver_status", - ] - df = ( - df[keep] - .sort_values(["risk_tolerance", "policy", "uncertainty_aversion"]) - .reset_index(drop=True) - ) - out_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(out_path, index=False) - - -def main(run_tag: str | None = None) -> None: - out_dir = ROOT / "reports/dvc" - out_dir.mkdir(parents=True, exist_ok=True) - - resolved_run_tag = ( - str(run_tag or "").strip() or str(os.environ.get("PIPELINE_RUN_TAG", "")).strip() - ) - if not resolved_run_tag: - resolved_run_tag = f"manual-{datetime.now(UTC).strftime('%Y%m%d-%H%M%SZ')}" - - metrics = {} - metrics.update(_pd_metrics()) - metrics.update(_conformal_metrics()) - metrics.update(_ifrs9_metrics()) - metrics.update(_optimization_metrics()) - metrics.update(_crpto_final_metrics()) - - invalid = [k for k, v in metrics.items() if not np.isfinite(float(v))] - if invalid: - raise ValueError(f"Non-finite values found in DVC metrics export: {sorted(invalid)}") - - payload: dict[str, Any] = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(UTC).isoformat(), - "run_tag": resolved_run_tag, - "crpto_final_run_tag": _load_json(MODELS / "final_project_promotion.json").get( - "run_tag", "missing" - ) - if (MODELS / "final_project_promotion.json").exists() - else "missing", - "metrics": metrics, - } - # Keep top-level numeric keys for compatibility with DVC metrics and Streamlit helpers. - payload.update(metrics) - - metrics_path = out_dir / "metrics_summary.json" - with open(metrics_path, "w", encoding="utf-8") as f: - json.dump(payload, f, indent=2, sort_keys=True) - f.write("\n") - - _write_conformal_backtest_plot(out_dir / "conformal_coverage_backtest.csv") - _write_robustness_frontier_plot(out_dir / "robustness_frontier.csv") - - logger.info(f"Wrote DVC metrics summary: {metrics_path}") - logger.info(f"Metrics exported: {len(metrics)} keys (run_tag={resolved_run_tag})") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Export canonical DVC metrics.") - parser.add_argument("--run-tag", default=None) - args = parser.parse_args() - main(run_tag=args.run_tag) diff --git a/scripts/generate_governance_status.py b/scripts/generate_governance_status.py deleted file mode 100644 index db3f704..0000000 --- a/scripts/generate_governance_status.py +++ /dev/null @@ -1,991 +0,0 @@ -"""Generate governance drift status for MRM gating. - -Builds per-feature drift diagnostics (PSI, KS, CvM) and multivariate C2ST, -then emits: -- data/processed/drift_monitoring.parquet -- models/governance_status.json - -Usage: - uv run python scripts/generate_governance_status.py --config configs/mrm_policy.yaml -""" - -from __future__ import annotations - -import argparse -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from catboost import CatBoostClassifier, Pool -from loguru import logger -from sklearn.metrics import brier_score_loss, roc_auc_score - -from scripts.train_pd_model import _apply_training_regime -from src.evaluation.backtesting import ( - classifier_two_sample_test, - drift_monitoring_report, - population_stability_index, -) -from src.evaluation.explainability import dominant_reason_match_rate, rank_overlap_ratio -from src.evaluation.model_shift import interpret_model_shift -from src.models.pd_contract import load_contract, resolve_calibrator_path, resolve_model_path -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.baseline_registry import resolve_official_baseline_run_tag -from src.utils.io_utils import load_pickle_compat, read_split_with_fe_fallback -from src.utils.threshold_semantics import load_threshold_semantics, resolve_operational_threshold - -SCHEMA_VERSION = "2026-03-06.1" - - -@dataclass(frozen=True) -class GovernanceThresholds: - psi_threshold: float - ks_pvalue_min: float - cvm_pvalue_min: float - c2st_auc_max: float - max_feature_breach_ratio: float - c2st_max_rows: int - score_psi_max: float - auc_delta_max: float - brier_increase_max: float - calibration_gap_delta_max: float - performance_max_rows: int - min_rank_overlap_top10: float - max_explanation_shap_psi: float - min_reason_code_stability: float - explanation_min_rows_per_slice: int - psi_bins: int - random_state: int - - -@dataclass(frozen=True) -class GovernanceOutputPaths: - drift_path: Path - status_path: Path - explanation_drift_path: Path - fairness_status_path: Path - fairness_frontier_path: Path - challenger_report_path: Path - model_shift_status_path: Path - - -def _load_cfg(path: str) -> dict[str, Any]: - with open(path, encoding="utf-8") as f: - return yaml.safe_load(f) - - -def _load_feature_contract() -> list[str]: - contract_path = Path("models/pd_model_contract.json") - if not contract_path.exists(): - return [] - try: - contract = json.loads(contract_path.read_text(encoding="utf-8")) - except Exception: - return [] - features = contract.get("feature_names", []) - if not isinstance(features, list): - return [] - return [str(f) for f in features] - - -def _load_training_record() -> dict[str, Any]: - path = Path("models/pd_training_record.pkl") - if not path.exists(): - return {} - try: - payload = load_pickle_compat(path) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def _sample_frame(df: pd.DataFrame, max_rows: int, random_state: int) -> pd.DataFrame: - if max_rows <= 0 or len(df) <= max_rows: - return df.copy() - return df.sample(n=max_rows, random_state=random_state).copy() - - -def _load_calibrator() -> Any | None: - path = resolve_calibrator_path() - if path is None or not path.exists(): - return None - try: - return load_pickle_compat(path) - except Exception: - return None - - -def _apply_calibrator(calibrator: Any | None, y_prob_raw: np.ndarray) -> np.ndarray: - if calibrator is None: - return y_prob_raw.astype(float) - if hasattr(calibrator, "predict_proba"): - calibrated = calibrator.predict_proba(y_prob_raw.reshape(-1, 1)) - if calibrated.ndim == 2 and calibrated.shape[1] >= 2: - return calibrated[:, 1].astype(float) - if hasattr(calibrator, "predict"): - calibrated = calibrator.predict(y_prob_raw) - return np.asarray(calibrated, dtype=float) - return y_prob_raw.astype(float) - - -def _prepare_model_frame( - df: pd.DataFrame, - *, - features: list[str], - categorical_features: list[str], -) -> pd.DataFrame: - X = df.reindex(columns=features).copy() - inferred_cat = [ - col - for col in X.columns - if col in categorical_features or not pd.api.types.is_numeric_dtype(X[col]) - ] - for col in inferred_cat: - if col in X.columns: - X[col] = X[col].astype(str).fillna("missing") - return X - - -def _score_and_performance_report( - train_df: pd.DataFrame, - test_df: pd.DataFrame, - *, - random_state: int, - max_rows_per_split: int, - psi_bins: int, -) -> dict[str, float]: - contract = load_contract() - if not contract: - return {} - - feature_names = [str(f) for f in contract.get("feature_names", [])] - categorical_features = [str(f) for f in contract.get("categorical_features", [])] - if not feature_names: - return {} - - model = CatBoostClassifier() - model.load_model(resolve_model_path().as_posix()) - calibrator = _load_calibrator() - - train_eval = _sample_frame(train_df, max_rows=max_rows_per_split, random_state=random_state) - test_eval = _sample_frame(test_df, max_rows=max_rows_per_split, random_state=random_state) - - X_train = _prepare_model_frame( - train_eval, - features=feature_names, - categorical_features=categorical_features, - ) - X_test = _prepare_model_frame( - test_eval, - features=feature_names, - categorical_features=categorical_features, - ) - y_train = ( - pd.to_numeric(train_eval["default_flag"], errors="coerce").fillna(0).astype(int).to_numpy() - ) - y_test = ( - pd.to_numeric(test_eval["default_flag"], errors="coerce").fillna(0).astype(int).to_numpy() - ) - - cat_cols = [ - col - for col in X_train.columns - if col in categorical_features or not pd.api.types.is_numeric_dtype(X_train[col]) - ] - train_pool = Pool(X_train, cat_features=cat_cols) - test_pool = Pool(X_test, cat_features=cat_cols) - - train_raw = model.predict_proba(train_pool)[:, 1] - test_raw = model.predict_proba(test_pool)[:, 1] - train_score = _apply_calibrator(calibrator, train_raw) - test_score = _apply_calibrator(calibrator, test_raw) - - train_auc = float(roc_auc_score(y_train, train_score)) - test_auc = float(roc_auc_score(y_test, test_score)) - train_brier = float(brier_score_loss(y_train, train_score)) - test_brier = float(brier_score_loss(y_test, test_score)) - train_cal_gap = float(abs(float(np.mean(train_score)) - float(np.mean(y_train)))) - test_cal_gap = float(abs(float(np.mean(test_score)) - float(np.mean(y_test)))) - score_psi = float( - population_stability_index( - np.asarray(train_score, dtype=float), - np.asarray(test_score, dtype=float), - n_bins=psi_bins, - ) - ) - return { - "score_psi": score_psi, - "auc_train_reference": train_auc, - "auc_test_oot": test_auc, - "auc_delta_train_to_test": float(max(train_auc - test_auc, 0.0)), - "brier_train_reference": train_brier, - "brier_test_oot": test_brier, - "brier_increase_train_to_test": float(max(test_brier - train_brier, 0.0)), - "calibration_gap_train_reference": train_cal_gap, - "calibration_gap_test_oot": test_cal_gap, - "calibration_gap_delta": float(max(test_cal_gap - train_cal_gap, 0.0)), - "train_eval_rows": len(train_eval), - "test_eval_rows": len(test_eval), - } - - -def _resolve_numeric_features(train_df: pd.DataFrame, test_df: pd.DataFrame) -> list[str]: - contract_features = _load_feature_contract() - common = [f for f in contract_features if f in train_df.columns and f in test_df.columns] - - numeric = [] - for f in common: - if pd.api.types.is_numeric_dtype(train_df[f]) or pd.api.types.is_numeric_dtype(test_df[f]): - numeric.append(f) - - if numeric: - return numeric - - # Fallback: numeric intersection from both frames. - train_num = set(train_df.select_dtypes(include=["number"]).columns) - test_num = set(test_df.select_dtypes(include=["number"]).columns) - fallback = sorted(train_num.intersection(test_num)) - return [c for c in fallback if c != "default_flag"][:80] - - -def _safe_mean(series: pd.Series) -> float: - if series.empty: - return 0.0 - return float(series.mean()) - - -def _safe_float_value(value: Any, default: float = 0.0) -> float: - try: - return float(value) - except (TypeError, ValueError): - return default - - -def _safe_int_value(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _safe_list_value(value: Any) -> list[Any]: - return list(value) if isinstance(value, list | tuple) else [] - - -def _resolve_primary_threshold() -> float: - semantics = load_threshold_semantics() - if semantics: - return resolve_operational_threshold(semantics, default=0.5) - fairness_status_path = Path("models/fairness_audit_status.json") - if fairness_status_path.exists(): - try: - payload = json.loads(fairness_status_path.read_text(encoding="utf-8")) - return float(payload.get("primary_threshold", payload.get("prediction_threshold", 0.5))) - except Exception: - pass - decision_threshold_path = Path("models/decision_threshold.json") - if decision_threshold_path.exists(): - try: - payload = json.loads(decision_threshold_path.read_text(encoding="utf-8")) - return float(payload.get("selected_threshold", 0.5)) - except Exception: - pass - return 0.5 - - -def _explanation_feature_columns(shap_raw: pd.DataFrame) -> list[str]: - return [c.replace("shap_", "") for c in shap_raw.columns if c.startswith("shap_")] - - -def _recent_comparison_periods( - shap_raw: pd.DataFrame, - periods: list[str], - *, - min_rows_per_slice: int, -) -> tuple[list[str], pd.DataFrame] | None: - comparison_periods: list[str] = [] - for period in reversed(periods): - comparison_periods.insert(0, period) - comparison_df = shap_raw.loc[ - shap_raw["issue_quarter"].astype(str).isin(comparison_periods) - ].copy() - if len(comparison_df) >= min_rows_per_slice: - return comparison_periods, comparison_df - return None - - -def _explanation_segment_pairs( - shap_raw: pd.DataFrame, - reference_df: pd.DataFrame, - comparison_df: pd.DataFrame, - *, - min_rows_per_slice: int, -) -> list[tuple[str, str, pd.DataFrame, pd.DataFrame]]: - segment_pairs: list[tuple[str, str, pd.DataFrame, pd.DataFrame]] = [ - ("overall", "all", reference_df, comparison_df) - ] - if "grade" not in shap_raw.columns: - return segment_pairs - for grade in sorted(shap_raw["grade"].dropna().astype(str).unique().tolist()): - ref_seg = reference_df.loc[reference_df["grade"].astype(str) == grade].copy() - cmp_seg = comparison_df.loc[comparison_df["grade"].astype(str) == grade].copy() - if len(ref_seg) < min_rows_per_slice or len(cmp_seg) < min_rows_per_slice: - continue - segment_pairs.append(("grade", grade, ref_seg, cmp_seg)) - return segment_pairs - - -def _rank_shap_features(segment: pd.DataFrame, feature_cols: list[str]) -> list[str]: - return sorted( - feature_cols, - key=lambda feature: segment[f"shap_{feature}"].abs().mean(), - reverse=True, - ) - - -def _shap_psi_details( - ref_seg: pd.DataFrame, - cmp_seg: pd.DataFrame, - *, - focus_features: list[str], -) -> list[dict[str, float | str]]: - rows: list[dict[str, float | str]] = [] - for feature in focus_features: - col = f"shap_{feature}" - if col not in ref_seg.columns or col not in cmp_seg.columns: - continue - psi = population_stability_index( - pd.to_numeric(ref_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - pd.to_numeric(cmp_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - n_bins=8, - ) - rows.append({"feature": feature, "psi": float(psi)}) - return rows - - -def _explanation_drift_row( - *, - segment_type: str, - segment: str, - ref_seg: pd.DataFrame, - cmp_seg: pd.DataFrame, - feature_cols: list[str], - periods: list[str], - comparison_periods: list[str], - comparison_period_label: str, - primary_threshold: float, - min_rank_overlap_top10: float, - max_shap_psi: float, - min_reason_code_stability: float, - min_rows_per_slice: int, - pd_col: str, -) -> dict[str, Any]: - ref_ranking = _rank_shap_features(ref_seg, feature_cols) - cmp_ranking = _rank_shap_features(cmp_seg, feature_cols) - overlap = rank_overlap_ratio(ref_ranking, cmp_ranking, top_k=10) - focus_features = list(dict.fromkeys(ref_ranking[:5] + cmp_ranking[:5]))[:5] - shap_psis = _shap_psi_details(ref_seg, cmp_seg, focus_features=focus_features) - max_feature_psi = max((float(row["psi"]) for row in shap_psis), default=0.0) - avg_feature_psi = float(np.mean([float(row["psi"]) for row in shap_psis])) if shap_psis else 0.0 - reason_match_rate, reason_details = dominant_reason_match_rate( - ref_seg, - cmp_seg, - ref_ranking[:10], - pd_col=pd_col, - threshold=primary_threshold, - min_rows_per_band=max(15, int(min_rows_per_slice / 4)), - ) - pass_rank = bool(overlap >= min_rank_overlap_top10) - pass_dist = bool(max_feature_psi <= max_shap_psi) - pass_reason = bool(reason_match_rate >= min_reason_code_stability) - return { - "segment_type": segment_type, - "segment": segment, - "reference_period": "|".join([p for p in periods if p not in comparison_periods]), - "comparison_period": comparison_period_label, - "reference_n": len(ref_seg), - "comparison_n": len(cmp_seg), - "rank_overlap_top10": float(overlap), - "avg_shap_psi_top5": float(avg_feature_psi), - "max_shap_psi_top5": float(max_feature_psi), - "reason_code_match_rate": float(reason_match_rate), - "pass_rank_overlap": pass_rank, - "pass_distribution_shift": pass_dist, - "pass_reason_code_stability": pass_reason, - "passed_all": bool(pass_rank and pass_dist and pass_reason), - "feature_psi_details": json.dumps(shap_psis, default=str), - "reason_code_details": json.dumps(reason_details, default=str), - } - - -def _build_explanation_drift_report( - shap_raw: pd.DataFrame, - *, - primary_threshold: float, - min_rank_overlap_top10: float, - max_shap_psi: float, - min_reason_code_stability: float, - min_rows_per_slice: int, -) -> pd.DataFrame: - feature_cols = _explanation_feature_columns(shap_raw) - if shap_raw.empty or not feature_cols or "issue_quarter" not in shap_raw.columns: - return pd.DataFrame() - - periods = sorted( - [ - p - for p in shap_raw["issue_quarter"].astype(str).dropna().unique().tolist() - if p != "unknown" - ] - ) - if len(periods) < 2: - return pd.DataFrame() - - comparison_slice = _recent_comparison_periods( - shap_raw, - periods, - min_rows_per_slice=min_rows_per_slice, - ) - if comparison_slice is None: - return pd.DataFrame() - comparison_periods, comparison_df = comparison_slice - - reference_df = shap_raw.loc[ - ~shap_raw["issue_quarter"].astype(str).isin(comparison_periods) - ].copy() - if len(reference_df) < min_rows_per_slice or len(comparison_df) < min_rows_per_slice: - return pd.DataFrame() - comparison_period_label = "|".join(comparison_periods) - - segment_pairs = _explanation_segment_pairs( - shap_raw, - reference_df, - comparison_df, - min_rows_per_slice=min_rows_per_slice, - ) - pd_col = "pd_calibrated" if "pd_calibrated" in shap_raw.columns else "score_raw" - rows: list[dict[str, Any]] = [] - for segment_type, segment, ref_seg, cmp_seg in segment_pairs: - rows.append( - _explanation_drift_row( - segment_type=segment_type, - segment=segment, - ref_seg=ref_seg, - cmp_seg=cmp_seg, - feature_cols=feature_cols, - periods=periods, - comparison_periods=comparison_periods, - comparison_period_label=comparison_period_label, - primary_threshold=primary_threshold, - min_rank_overlap_top10=min_rank_overlap_top10, - max_shap_psi=max_shap_psi, - min_reason_code_stability=min_reason_code_stability, - min_rows_per_slice=min_rows_per_slice, - pd_col=pd_col, - ) - ) - return pd.DataFrame(rows) - - -def _resolve_thresholds( - triggers: dict[str, Any], - checks: dict[str, Any], -) -> GovernanceThresholds: - return GovernanceThresholds( - psi_threshold=float(triggers.get("psi_threshold", 0.25)), - ks_pvalue_min=float(checks.get("ks_pvalue_min", 0.01)), - cvm_pvalue_min=float(checks.get("cvm_pvalue_min", 0.01)), - c2st_auc_max=float(checks.get("c2st_auc_max", 0.60)), - max_feature_breach_ratio=float(checks.get("max_feature_breach_ratio", 0.15)), - c2st_max_rows=int(checks.get("c2st_max_rows_per_split", 50_000)), - score_psi_max=float(checks.get("score_psi_max", 0.15)), - auc_delta_max=float(checks.get("auc_delta_max", 0.05)), - brier_increase_max=float(checks.get("brier_increase_max", 0.02)), - calibration_gap_delta_max=float(checks.get("calibration_gap_delta_max", 0.02)), - performance_max_rows=int(checks.get("performance_max_rows_per_split", 100_000)), - min_rank_overlap_top10=float(checks.get("explanation_rank_overlap_top10_min", 0.60)), - max_explanation_shap_psi=float(checks.get("explanation_shap_psi_max", 0.25)), - min_reason_code_stability=float(checks.get("reason_code_stability_min", 0.55)), - explanation_min_rows_per_slice=int(checks.get("explanation_min_rows_per_slice", 80)), - psi_bins=int(checks.get("psi_bins", 10)), - random_state=int(checks.get("random_state", 42)), - ) - - -def _output_path(outputs: dict[str, Any], key: str, default: str) -> Path: - value = outputs.get(key, default) - return Path(str(default if value is None else value)) - - -def _resolve_output_paths(outputs: dict[str, Any]) -> GovernanceOutputPaths: - return GovernanceOutputPaths( - drift_path=_output_path( - outputs, - "drift_monitoring_path", - "data/processed/drift_monitoring.parquet", - ), - status_path=_output_path( - outputs, - "governance_status_path", - "models/governance_status.json", - ), - explanation_drift_path=_output_path( - outputs, - "explanation_drift_path", - "data/processed/explanation_drift.parquet", - ), - fairness_status_path=_output_path( - outputs, - "fairness_status_path", - "models/fairness_audit_status.json", - ), - fairness_frontier_path=_output_path( - outputs, - "fairness_frontier_path", - "data/processed/fairness_threshold_frontier.parquet", - ), - challenger_report_path=_output_path( - outputs, - "challenger_promotion_report_path", - "models/challenger_promotion_report.json", - ), - model_shift_status_path=_output_path( - outputs, - "model_shift_status_path", - "models/model_shift_status.json", - ), - ) - - -def _load_governance_frames() -> tuple[pd.DataFrame, pd.DataFrame]: - train_df = read_split_with_fe_fallback("data/processed/train_fe.parquet") - test_df = read_split_with_fe_fallback("data/processed/test_fe.parquet") - training_record = _load_training_record() - regime_cfg = training_record.get("training_regime", {}) if training_record else {} - if isinstance(regime_cfg, dict) and regime_cfg: - train_df, regime_meta = _apply_training_regime(train_df, regime_cfg, date_col="issue_d") - logger.info( - "Applied PD training regime to governance reference population: mode={} rows={}", - regime_meta.get("mode", regime_cfg.get("mode", "standard")), - len(train_df), - ) - return train_df, test_df - - -def _write_parquet(df: pd.DataFrame, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - df.to_parquet(path, index=False) - - -def _load_json_dict(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def _build_explanation_drift_if_available(thresholds: GovernanceThresholds) -> pd.DataFrame: - shap_raw_path = Path("data/processed/shap_raw_top20.parquet") - if not shap_raw_path.exists(): - return pd.DataFrame() - return _build_explanation_drift_report( - pd.read_parquet(shap_raw_path), - primary_threshold=_resolve_primary_threshold(), - min_rank_overlap_top10=thresholds.min_rank_overlap_top10, - max_shap_psi=thresholds.max_explanation_shap_psi, - min_reason_code_stability=thresholds.min_reason_code_stability, - min_rows_per_slice=thresholds.explanation_min_rows_per_slice, - ) - - -def _drift_breach_metrics( - drift_df: pd.DataFrame, - c2st: dict[str, Any], - performance_report: dict[str, float], - thresholds: GovernanceThresholds, -) -> dict[str, Any]: - n_features = len(drift_df) - psi_breaches = ( - int((~drift_df.get("pass_psi", pd.Series(dtype=bool))).sum()) if n_features else 0 - ) - ks_breaches = int((~drift_df.get("pass_ks", pd.Series(dtype=bool))).sum()) if n_features else 0 - cvm_breaches = ( - int((~drift_df.get("pass_cvm", pd.Series(dtype=bool))).sum()) if n_features else 0 - ) - feature_breach_ratio = float(psi_breaches / max(n_features, 1)) - distribution_warning_ratio = float((ks_breaches + cvm_breaches) / max(n_features * 2, 1)) - score_psi = float(performance_report.get("score_psi", 0.0)) - auc_delta = float(performance_report.get("auc_delta_train_to_test", 0.0)) - brier_increase = float(performance_report.get("brier_increase_train_to_test", 0.0)) - calibration_gap_delta = float(performance_report.get("calibration_gap_delta", 0.0)) - pass_psi = bool( - (float(drift_df["psi"].max()) if n_features else 0.0) <= thresholds.psi_threshold - ) - pass_breach_ratio = bool(feature_breach_ratio <= thresholds.max_feature_breach_ratio) - pass_score_psi = bool(score_psi <= thresholds.score_psi_max) - pass_auc_delta = bool(auc_delta <= thresholds.auc_delta_max) - pass_brier_increase = bool(brier_increase <= thresholds.brier_increase_max) - pass_calibration_gap_delta = bool(calibration_gap_delta <= thresholds.calibration_gap_delta_max) - return { - "n_features": n_features, - "psi_breaches": psi_breaches, - "ks_breaches": ks_breaches, - "cvm_breaches": cvm_breaches, - "feature_breach_ratio": feature_breach_ratio, - "distribution_warning_ratio": distribution_warning_ratio, - "max_psi": float(drift_df["psi"].max()) if n_features else 0.0, - "mean_psi": _safe_mean(drift_df["psi"]) if n_features else 0.0, - "min_ks_pvalue": float(drift_df["ks_pvalue"].min()) if n_features else 1.0, - "min_cvm_pvalue": float(drift_df["cvm_pvalue"].min()) if n_features else 1.0, - "c2st_auc": _safe_float_value(c2st["c2st_auc"]), - "c2st_materiality": str(c2st.get("materiality", "none")), - "c2st_effective_driver_count": _safe_int_value(c2st.get("effective_driver_count", 0)), - "c2st_top_drivers": _safe_list_value(c2st.get("top_drivers", [])), - "c2st_rows_used": _safe_int_value(c2st.get("n_rows", 0)), - "score_psi": score_psi, - "auc_delta": auc_delta, - "brier_increase": brier_increase, - "calibration_gap_delta": calibration_gap_delta, - "pass_psi": pass_psi, - "pass_breach_ratio": pass_breach_ratio, - "pass_score_psi": pass_score_psi, - "pass_auc_delta": pass_auc_delta, - "pass_brier_increase": pass_brier_increase, - "pass_calibration_gap_delta": pass_calibration_gap_delta, - "pass_predictive_drift": bool( - pass_psi - and pass_breach_ratio - and pass_score_psi - and pass_auc_delta - and pass_brier_increase - and pass_calibration_gap_delta - ), - "pass_c2st": bool(_safe_float_value(c2st["c2st_auc"]) <= thresholds.c2st_auc_max), - "performance_report": performance_report, - } - - -def _interpret_governance_shift( - metrics: dict[str, Any], - thresholds: GovernanceThresholds, -) -> dict[str, Any]: - return interpret_model_shift( - c2st_auc=float(metrics["c2st_auc"]), - c2st_materiality=str(metrics["c2st_materiality"]), - score_psi=float(metrics["score_psi"]), - auc_delta=float(metrics["auc_delta"]), - brier_increase=float(metrics["brier_increase"]), - calibration_gap_delta=float(metrics["calibration_gap_delta"]), - distribution_warning_ratio=float(metrics["distribution_warning_ratio"]), - score_psi_max=thresholds.score_psi_max, - auc_delta_max=thresholds.auc_delta_max, - brier_increase_max=thresholds.brier_increase_max, - calibration_gap_delta_max=thresholds.calibration_gap_delta_max, - ) - - -def _explanation_passes(explanation_drift: pd.DataFrame) -> dict[str, bool]: - explainability_pass = bool( - (not explanation_drift.empty) and explanation_drift["passed_all"].astype(bool).all() - ) - reason_code_stability_pass = bool( - (not explanation_drift.empty) - and explanation_drift["pass_reason_code_stability"].astype(bool).all() - ) - return { - "explainability_pass": explainability_pass, - "reason_code_stability_pass": reason_code_stability_pass, - } - - -def _series_min_or_zero(df: pd.DataFrame, column: str) -> float: - return float(df[column].min()) if not df.empty else 0.0 - - -def _series_max_or_zero(df: pd.DataFrame, column: str) -> float: - return float(df[column].max()) if not df.empty else 0.0 - - -def _top_explanation_breaches(explanation_drift: pd.DataFrame) -> list[dict[str, Any]]: - if explanation_drift.empty: - return [] - return ( - explanation_drift.sort_values( - ["passed_all", "max_shap_psi_top5", "rank_overlap_top10"], - ascending=[True, False, True], - ) - .head(10) - .to_dict(orient="records") - ) - - -def _fairness_primary_threshold(fairness_status: dict[str, Any]) -> float: - if not fairness_status: - return 0.5 - return _safe_float_value( - fairness_status.get("primary_threshold", fairness_status.get("prediction_threshold", 0.5)) - ) - - -def _warning_flags( - metrics: dict[str, Any], - explanation_checks: dict[str, bool], -) -> dict[str, bool]: - return { - "warn_c2st": bool(not metrics["pass_c2st"]), - "warn_distribution_tests": bool(metrics["ks_breaches"] > 0 or metrics["cvm_breaches"] > 0), - "warn_explainability": bool(not explanation_checks["explainability_pass"]), - "warn_reason_code_stability": bool(not explanation_checks["reason_code_stability_pass"]), - } - - -def _build_governance_status( - *, - config_path: str, - resolved_run_tag: str, - paths: GovernanceOutputPaths, - thresholds: GovernanceThresholds, - drift_df: pd.DataFrame, - explanation_drift: pd.DataFrame, - fairness_status: dict[str, Any], - challenger_report: dict[str, Any], - metrics: dict[str, Any], - model_shift: dict[str, Any], -) -> dict[str, Any]: - fairness_pass = bool(fairness_status.get("overall_pass", False)) - challenger_promotable = bool(challenger_report.get("challenger_promotable", False)) - explanation_checks = _explanation_passes(explanation_drift) - warning_flags = _warning_flags(metrics, explanation_checks) - overall_pass = bool(metrics["pass_predictive_drift"] and fairness_pass) - top_breaches = drift_df.head(10).to_dict(orient="records") if int(metrics["n_features"]) else [] - primary_threshold = _fairness_primary_threshold(fairness_status) - - return { - "overall_pass": overall_pass, - "checks": { - "pass_psi": bool(metrics["pass_psi"]), - "pass_breach_ratio": bool(metrics["pass_breach_ratio"]), - "pass_score_psi": bool(metrics["pass_score_psi"]), - "pass_auc_delta": bool(metrics["pass_auc_delta"]), - "pass_brier_increase": bool(metrics["pass_brier_increase"]), - "pass_calibration_gap_delta": bool(metrics["pass_calibration_gap_delta"]), - "pass_predictive_drift": bool(metrics["pass_predictive_drift"]), - "pass_fairness": fairness_pass, - "pass_c2st": bool(metrics["pass_c2st"]), - "pass_explainability": explanation_checks["explainability_pass"], - "pass_reason_code_stability": explanation_checks["reason_code_stability_pass"], - **warning_flags, - }, - "thresholds": { - "psi_threshold": thresholds.psi_threshold, - "ks_pvalue_min": thresholds.ks_pvalue_min, - "cvm_pvalue_min": thresholds.cvm_pvalue_min, - "c2st_auc_max": thresholds.c2st_auc_max, - "max_feature_breach_ratio": thresholds.max_feature_breach_ratio, - "score_psi_max": thresholds.score_psi_max, - "auc_delta_max": thresholds.auc_delta_max, - "brier_increase_max": thresholds.brier_increase_max, - "calibration_gap_delta_max": thresholds.calibration_gap_delta_max, - "explanation_rank_overlap_top10_min": thresholds.min_rank_overlap_top10, - "explanation_shap_psi_max": thresholds.max_explanation_shap_psi, - "reason_code_stability_min": thresholds.min_reason_code_stability, - }, - "summary": { - "n_features": int(metrics["n_features"]), - "max_psi": float(metrics["max_psi"]), - "mean_psi": float(metrics["mean_psi"]), - "min_ks_pvalue": float(metrics["min_ks_pvalue"]), - "min_cvm_pvalue": float(metrics["min_cvm_pvalue"]), - "c2st_auc": float(metrics["c2st_auc"]), - "psi_breaches": int(metrics["psi_breaches"]), - "ks_breaches": int(metrics["ks_breaches"]), - "cvm_breaches": int(metrics["cvm_breaches"]), - "feature_breach_ratio": float(metrics["feature_breach_ratio"]), - "distribution_warning_ratio": float(metrics["distribution_warning_ratio"]), - "c2st_rows_used": int(metrics["c2st_rows_used"]), - "c2st_materiality": str(metrics["c2st_materiality"]), - "c2st_effective_driver_count": int(metrics["c2st_effective_driver_count"]), - **metrics["performance_report"], - "n_explanation_segments": len(explanation_drift), - "min_rank_overlap_top10": _series_min_or_zero(explanation_drift, "rank_overlap_top10"), - "max_explanation_shap_psi": _series_max_or_zero(explanation_drift, "max_shap_psi_top5"), - "min_reason_code_stability": _series_min_or_zero( - explanation_drift, "reason_code_match_rate" - ), - "fairness_overall_pass": fairness_pass, - "fairness_primary_threshold": primary_threshold, - "challenger_promotable": challenger_promotable, - "model_shift_type": str(model_shift["shift_type"]), - "governance_posture": str(model_shift["governance_posture"]), - }, - "warnings": warning_flags, - "c2st": { - "auc": float(metrics["c2st_auc"]), - "materiality": str(metrics["c2st_materiality"]), - "effective_driver_count": int(metrics["c2st_effective_driver_count"]), - "top_drivers": metrics["c2st_top_drivers"], - }, - "model_shift": model_shift, - "artifacts": { - "drift_monitoring_path": str(paths.drift_path), - "explanation_drift_path": str(paths.explanation_drift_path), - "fairness_status_path": str(paths.fairness_status_path), - "fairness_frontier_path": str(paths.fairness_frontier_path), - "challenger_promotion_report_path": str(paths.challenger_report_path), - "model_shift_status_path": str(paths.model_shift_status_path), - }, - "top_drift_features": top_breaches, - "top_explanation_breaches": _top_explanation_breaches(explanation_drift), - "primary_threshold": primary_threshold, - "explainability_pass": explanation_checks["explainability_pass"], - "explanation_drift_pass": explanation_checks["explainability_pass"], - "reason_code_stability_pass": explanation_checks["reason_code_stability_pass"], - "challenger_promotable": challenger_promotable, - "policy_config": config_path, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - - -def _write_model_shift_status( - *, - path: Path, - status_path: Path, - model_shift: dict[str, Any], - resolved_run_tag: str, -) -> None: - _write_json( - path, - { - "diagnostic_only": True, - "overall_pass": bool(model_shift["governance_posture"] != "candidate_gate"), - "summary": model_shift, - "artifacts": {"governance_status_path": str(status_path)}, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - }, - ) - - -def _log_governance_outputs( - *, - paths: GovernanceOutputPaths, - status: dict[str, Any], - metrics: dict[str, Any], -) -> None: - logger.info("Saved drift monitoring: {}", paths.drift_path) - logger.info("Saved governance status: {}", paths.status_path) - logger.info("Saved model-shift interpretation: {}", paths.model_shift_status_path) - logger.info( - "Governance checks pass={} (max_psi={:.4f}, score_psi={:.4f}, auc_delta={:.4f}, brier_increase={:.4f}, c2st_auc={:.4f})", - status["overall_pass"], - metrics["max_psi"], - metrics["score_psi"], - metrics["auc_delta"], - metrics["brier_increase"], - metrics["c2st_auc"], - ) - - -def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: - cfg = _load_cfg(config_path) - semantics = load_threshold_semantics() - resolved_run_tag = resolve_run_tag( - run_tag, - fallback_candidates=[semantics.get("run_tag"), resolve_official_baseline_run_tag()], - require_explicit=True, - ) - - triggers = cfg.get("retraining_triggers", {}) - checks = cfg.get("governance_checks", {}) - outputs = cfg.get("governance_output", {}) - thresholds = _resolve_thresholds(triggers, checks) - paths = _resolve_output_paths(outputs) - - train_df, test_df = _load_governance_frames() - - features = _resolve_numeric_features(train_df, test_df) - if not features: - raise ValueError("No numeric features available for governance drift checks.") - - logger.info("Governance drift checks on {} numeric features", len(features)) - - drift_df = drift_monitoring_report( - train_df=train_df, - test_df=test_df, - features=features, - psi_threshold=thresholds.psi_threshold, - ks_pvalue_threshold=thresholds.ks_pvalue_min, - cvm_pvalue_threshold=thresholds.cvm_pvalue_min, - n_bins=thresholds.psi_bins, - ) - - c2st = classifier_two_sample_test( - train_df=train_df, - test_df=test_df, - features=features, - max_rows_per_split=thresholds.c2st_max_rows, - random_state=thresholds.random_state, - ) - performance_report = _score_and_performance_report( - train_df, - test_df, - random_state=thresholds.random_state, - max_rows_per_split=thresholds.performance_max_rows, - psi_bins=thresholds.psi_bins, - ) - metrics = _drift_breach_metrics(drift_df, c2st, performance_report, thresholds) - model_shift = _interpret_governance_shift(metrics, thresholds) - - _write_parquet(drift_df, paths.drift_path) - explanation_drift = _build_explanation_drift_if_available(thresholds) - _write_parquet(explanation_drift, paths.explanation_drift_path) - - fairness_status = _load_json_dict(paths.fairness_status_path) - challenger_report = _load_json_dict(paths.challenger_report_path) - status = _build_governance_status( - config_path=config_path, - resolved_run_tag=resolved_run_tag, - paths=paths, - thresholds=thresholds, - drift_df=drift_df, - explanation_drift=explanation_drift, - fairness_status=fairness_status, - challenger_report=challenger_report, - metrics=metrics, - model_shift=model_shift, - ) - _write_json(paths.status_path, status) - _write_model_shift_status( - path=paths.model_shift_status_path, - status_path=paths.status_path, - model_shift=model_shift, - resolved_run_tag=resolved_run_tag, - ) - _log_governance_outputs(paths=paths, status=status, metrics=metrics) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate governance drift status") - parser.add_argument("--config", default="configs/mrm_policy.yaml") - parser.add_argument("--run-tag", default=None) - args = parser.parse_args() - main(config_path=args.config, run_tag=args.run_tag) diff --git a/scripts/generate_mrm_report.py b/scripts/generate_mrm_report.py deleted file mode 100644 index 92627a1..0000000 --- a/scripts/generate_mrm_report.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Generate consolidated Model Risk Management (MRM) validation report. - -Aggregates status JSON files from pipeline, conformal, governance, -and fairness subsystems into a single MRM report following SR 11-7. - -Usage: - uv run python scripts/generate_mrm_report.py - uv run python scripts/generate_mrm_report.py --config configs/mrm_policy.yaml -""" - -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import yaml -from loguru import logger -from skops.card import Card -from skops.io import dump as skops_dump - -from src.utils.artifact_metadata import resolve_run_tag as resolve_artifact_run_tag -from src.utils.io_utils import load_pickle_compat - - -def _load_status(path: str | Path) -> dict: - """Load a JSON status file, returning empty dict if missing.""" - if not path: - return {} - p = Path(path) - if not p.exists() or p.is_dir(): - logger.warning(f"Status file not found: {p}") - return {} - with open(p) as f: - return json.load(f) - - -def _check_pass(status: dict) -> bool: - """Check if a status dict indicates overall pass.""" - if not status: - return False - # Try common patterns for overall pass - for key in ["overall_pass", "all_passed", "pass"]: - if key in status: - return bool(status[key]) - # For conformal: check if all checks passed - if "checks" in status: - checks = status["checks"] - if isinstance(checks, dict): - return all(bool(v) for v in checks.values()) - if isinstance(checks, list): - return all(c.get("passed", False) for c in checks) - # Pipeline summary artifact has no explicit overall pass flag. - return "pipeline" in status and "pd_model" in status - - -def _overall_compliance(statuses: dict[str, dict]) -> dict: - """Compute top-level compliance summary.""" - subsystem_pass = {} - for name, status in statuses.items(): - subsystem_pass[name] = _check_pass(status) - - return { - "overall_pass": all(subsystem_pass.values()) if subsystem_pass else False, - "subsystems": subsystem_pass, - "n_subsystems": len(statuses), - "n_passing": sum(subsystem_pass.values()), - } - - -def _safe_jsonable(value: Any) -> Any: - if isinstance(value, dict): - return {str(k): _safe_jsonable(v) for k, v in value.items()} - if isinstance(value, (list, tuple)): - return [_safe_jsonable(v) for v in value] - if isinstance(value, Path): - return str(value) - if hasattr(value, "tolist"): - try: - return value.tolist() - except Exception: - return str(value) - if isinstance(value, (str, int, float, bool)) or value is None: - return value - return str(value) - - -def _build_skops_sidecar( - cfg: dict[str, Any], statuses: dict[str, dict], output_path: Path -) -> dict[str, Any]: - model_cfg = cfg.get("model", {}) or {} - champion_artifact = str(model_cfg.get("champion_artifact", "")) - model_name = str(model_cfg.get("name", "CorePD")) - model_version = str(model_cfg.get("version", "unknown")) - - skops_dir = output_path.parent / "skops" - skops_dir.mkdir(parents=True, exist_ok=True) - - exports: list[dict[str, Any]] = [] - logreg_path = Path("models/pd_logreg_baseline.pkl") - logreg_payload = None - if logreg_path.exists(): - try: - logreg_payload = load_pickle_compat(logreg_path) - target_path = skops_dir / "pd_logreg_baseline.skops" - skops_dump(logreg_payload, target_path) - exports.append( - { - "name": "pd_logreg_baseline", - "source_path": str(logreg_path), - "export_path": str(target_path), - "status": "exported", - } - ) - except Exception as exc: - exports.append( - { - "name": "pd_logreg_baseline", - "source_path": str(logreg_path), - "status": "failed", - "error": str(exc), - } - ) - - card_summary = { - "model_name": model_name, - "model_version": model_version, - "champion_artifact": champion_artifact, - "subsystem_pass": { - name: bool(status.get("overall_pass", False)) for name, status in statuses.items() - }, - "limitations": [ - "Champion artifact remains CatBoost/CBM; skops export is limited to sklearn-compatible artifacts.", - "Fairlearn is integrated as a sidecar audit and does not replace the canonical fairness gate in this phase.", - "Observational causal outputs remain insights_only and are not promotion-eligible.", - ], - "exports": exports, - } - - if isinstance(logreg_payload, dict) and "model" in logreg_payload: - try: - card = Card(logreg_payload["model"]) - card.add_metrics( - section="Validation", - description="Pipeline-level subsystem gates consumed by MRM.", - pipeline_overall_pass=int( - bool(statuses.get("pipeline", {}).get("overall_pass", False)) - ), - conformal_overall_pass=int( - bool(statuses.get("conformal", {}).get("overall_pass", False)) - ), - governance_overall_pass=int( - bool(statuses.get("governance", {}).get("overall_pass", False)) - ), - fairness_overall_pass=int( - bool(statuses.get("fairness", {}).get("overall_pass", False)) - ), - ) - card.add_hyperparams() - model_card_md = card.render() - except Exception as exc: - model_card_md = ( - f"# {model_name} model card\n\n" - f"Skops card rendering failed: {exc}\n\n" - f"Exports:\n{json.dumps(_safe_jsonable(exports), indent=2)}\n" - ) - card_summary["card_render_status"] = "fallback" - card_summary["card_render_error"] = str(exc) - else: - card_summary["card_render_status"] = "rendered" - else: - model_card_md = ( - f"# {model_name} model card\n\n" - "No sklearn-compatible baseline artifact was available for skops rendering.\n" - ) - card_summary["card_render_status"] = "no_supported_model" - - model_card_html = ( - "
"
-        + model_card_md.replace("&", "&").replace("<", "<").replace(">", ">")
-        + "
" - ) - model_card_json_path = output_path.parent / "corepd_model_card.json" - model_card_html_path = output_path.parent / "corepd_model_card.html" - model_card_json_path.write_text( - json.dumps(_safe_jsonable(card_summary), indent=2, default=str), - encoding="utf-8", - ) - model_card_html_path.write_text(model_card_html, encoding="utf-8") - - return { - "exports": exports, - "model_card_json": str(model_card_json_path), - "model_card_html": str(model_card_html_path), - "card_render_status": card_summary.get("card_render_status", "unknown"), - } - - -def _resolve_run_tag(run_tag_arg: str | None) -> str: - """Resolve run_tag with official pipeline env fallback before pipeline_summary.""" - pipeline_path = Path("data/processed/pipeline_summary.json") - pipeline_tag = None - if pipeline_path.exists(): - try: - data = json.loads(pipeline_path.read_text(encoding="utf-8")) - tag = data.get("run_tag") - if tag: - pipeline_tag = str(tag) - except Exception: - pipeline_tag = None - return resolve_artifact_run_tag( - run_tag_arg, - fallback_candidates=[pipeline_tag], - allow_untracked=True, - ) - - -def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: - """Generate the MRM validation report.""" - with open(config_path) as f: - cfg = yaml.safe_load(f) - - artifacts = cfg["artifacts"] - - # Load all status files - statuses = { - "pipeline": _load_status(artifacts["pipeline_summary"]), - "conformal": _load_status(artifacts["conformal_status"]), - "governance": _load_status(artifacts["governance_status"]), - "fairness": _load_status(artifacts["fairness_status"]), - } - diagnostic_statuses = { - "pd_backtesting": _load_status(artifacts.get("pd_backtesting_status", "")), - "bootstrap_validation": _load_status(artifacts.get("bootstrap_validation_status", "")), - "monotonicity": _load_status(artifacts.get("monotonicity_status", "")), - "ifrs9_diagnostics": _load_status(artifacts.get("ifrs9_diagnostics_status", "")), - "encoding_stability": _load_status(artifacts.get("encoding_stability_status", "")), - "pd_validation_interpretation": _load_status( - artifacts.get("pd_validation_interpretation_status", "") - ), - "calibration_mapping": _load_status(artifacts.get("calibration_mapping_status", "")), - "model_shift": _load_status(artifacts.get("model_shift_status", "")), - } - - compliance = _overall_compliance(statuses) - resolved_run_tag = _resolve_run_tag(run_tag) - now_iso = datetime.now(tz=UTC).isoformat() - - report = { - "schema_version": "2026-03-14.1", - "generated_at_utc": now_iso, - "generated_at": now_iso, - "run_tag": resolved_run_tag, - "overall_pass": compliance["overall_pass"], - "model": cfg["model"], - "governance_policy": cfg["governance"], - "retraining_triggers": cfg["retraining_triggers"], - "challenger_criteria": cfg["challenger"], - "pipeline_summary": statuses["pipeline"], - "conformal_status": statuses["conformal"], - "governance_status": statuses["governance"], - "fairness_status": statuses["fairness"], - "diagnostic_statuses": diagnostic_statuses, - "compliance_summary": compliance, - } - - output_path = Path(cfg["output"]["mrm_report_json"]) - output_path.parent.mkdir(parents=True, exist_ok=True) - report["skops_governance"] = _build_skops_sidecar(cfg, statuses, output_path) - with open(output_path, "w", encoding="utf-8") as f: - json.dump(report, f, indent=2, default=str) - - # Write a compact status wrapper for governance dashboards and pipeline state. - status_path = Path(cfg["output"].get("mrm_status_json", "models/mrm_report_status.json")) - status_path.parent.mkdir(parents=True, exist_ok=True) - status_path.write_text( - json.dumps( - { - "schema_version": "2026-03-14.1", - "generated_at_utc": now_iso, - "run_tag": resolved_run_tag, - "overall_pass": compliance["overall_pass"], - "compliance_summary": compliance, - "report_path": str(output_path), - }, - indent=2, - ) - + "\n", - encoding="utf-8", - ) - - pass_label = "PASS" if compliance["overall_pass"] else "FAIL" - logger.info( - f"MRM report: {pass_label} " - f"({compliance['n_passing']}/{compliance['n_subsystems']} subsystems). " - f"Saved: {output_path}" - ) - logger.info(f"MRM status wrapper: {status_path}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate MRM validation report") - parser.add_argument("--config", default="configs/mrm_policy.yaml") - parser.add_argument("--run-tag", default=None, help="Run tag to stamp on the report") - args = parser.parse_args() - main(config_path=args.config, run_tag=args.run_tag) diff --git a/scripts/inspect_ijds_pdfs.py b/scripts/inspect_ijds_pdfs.py new file mode 100644 index 0000000..d1c5334 --- /dev/null +++ b/scripts/inspect_ijds_pdfs.py @@ -0,0 +1,179 @@ +"""Audit generated IJDS PDFs for page, layout, anonymity, and abstract contracts.""" + +from __future__ import annotations + +import argparse +import json +import re +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, cast + +import yaml +from pypdf import PdfReader + +from scripts.check_publication_integrity import ( + REVIEWER_FORBIDDEN_LITERALS, + REVIEWER_FORBIDDEN_PATTERNS, +) + +ROOT = Path(__file__).resolve().parents[1] +OFFICIAL_PDF = ROOT / "paper" / "submission" / "CRPTO_ijds_submission.pdf" +BODY_PDF = ROOT / "paper" / "CRPTO_ijds.pdf" +SUPPLEMENT_PDF = ROOT / "paper" / "supplement_ijds.pdf" +BODY_QMD = ROOT / "paper" / "CRPTO_ijds.qmd" +LETTER_POINTS = (612.0, 792.0) +BLANK_PAGE_MIN_ALNUM = 20 + + +@dataclass(frozen=True) +class PdfInspection: + """Compact machine-readable inspection of one reviewer-facing PDF.""" + + path: str + pages: int + page_sizes_points: tuple[str, ...] + non_letter_pages: tuple[int, ...] + blank_pages: tuple[int, ...] + identity_hits: tuple[str, ...] + fingerprint_hits: tuple[str, ...] + reference_start_page: int | None + + +def word_count(text: str) -> int: + """Count human-readable word tokens in an abstract.""" + return len(re.findall(r"\b[A-Za-z0-9]+(?:['-][A-Za-z0-9]+)*\b", text)) + + +def is_letter_size(width: float, height: float, *, tolerance: float = 1.0) -> bool: + """Return whether a page is US Letter in portrait or landscape orientation.""" + expected = (LETTER_POINTS, tuple(reversed(LETTER_POINTS))) + return any( + abs(width - target_width) <= tolerance and abs(height - target_height) <= tolerance + for target_width, target_height in expected + ) + + +def find_reference_start_page(page_texts: list[str]) -> int | None: + """Return the one-based page containing the standalone References heading.""" + heading = re.compile(r"(?:^|\n)\s*references\s*(?:\n|$)", re.IGNORECASE) + for page_number, text in enumerate(page_texts, start=1): + if heading.search(text): + return page_number + return None + + +def _load_abstract(path: Path = BODY_QMD) -> str: + raw = path.read_text(encoding="utf-8") + if not raw.startswith("---\n"): + raise ValueError(f"Missing YAML front matter in {path.relative_to(ROOT)}") + front_matter = raw.split("---", maxsplit=2)[1] + payload: object = yaml.safe_load(front_matter) + if not isinstance(payload, Mapping): + raise ValueError(f"Missing abstract in {path.relative_to(ROOT)}") + abstract = cast(Mapping[str, object], payload).get("abstract") + if not isinstance(abstract, str): + raise ValueError(f"Missing abstract in {path.relative_to(ROOT)}") + return abstract.strip() + + +def _page_size(page: Any) -> tuple[float, float]: + return float(page.mediabox.width), float(page.mediabox.height) + + +def inspect_pdf(path: Path, *, detect_references: bool = False) -> PdfInspection: + """Inspect one generated PDF without persisting extracted manuscript text.""" + if not path.is_file(): + raise FileNotFoundError(path) + reader = PdfReader(path) + page_texts = [(page.extract_text() or "") for page in reader.pages] + sizes = [_page_size(page) for page in reader.pages] + metadata = " ".join(str(value) for value in (reader.metadata or {}).values()) + searchable = "\n".join([metadata, *page_texts]) + normalized = searchable.lower() + + identity_hits = tuple( + sorted(literal for literal in REVIEWER_FORBIDDEN_LITERALS if literal in normalized) + ) + fingerprint_hits = tuple( + label for label, pattern in REVIEWER_FORBIDDEN_PATTERNS if pattern.search(searchable) + ) + blank_pages = tuple( + page_number + for page_number, text in enumerate(page_texts, start=1) + if len(re.sub(r"[^A-Za-z0-9]", "", text)) < BLANK_PAGE_MIN_ALNUM + ) + non_letter_pages = tuple( + page_number + for page_number, (width, height) in enumerate(sizes, start=1) + if not is_letter_size(width, height) + ) + unique_sizes = tuple(sorted({f"{width:.2f}x{height:.2f}" for width, height in sizes})) + return PdfInspection( + path=path.relative_to(ROOT).as_posix(), + pages=len(reader.pages), + page_sizes_points=unique_sizes, + non_letter_pages=non_letter_pages, + blank_pages=blank_pages, + identity_hits=identity_hits, + fingerprint_hits=fingerprint_hits, + reference_start_page=find_reference_start_page(page_texts) if detect_references else None, + ) + + +def build_report() -> dict[str, Any]: + """Build the full reviewer-facing PDF audit report.""" + inspections = ( + inspect_pdf(OFFICIAL_PDF, detect_references=True), + inspect_pdf(BODY_PDF), + inspect_pdf(SUPPLEMENT_PDF), + ) + official = inspections[0] + abstract = _load_abstract() + abstract_words = word_count(abstract) + abstract_single_paragraph = re.search(r"\n\s*\n", abstract) is None + content_pages = ( + official.reference_start_page - 1 if official.reference_start_page is not None else None + ) + + failures: list[str] = [] + for inspection in inspections: + if inspection.non_letter_pages: + failures.append(f"{inspection.path}: non-Letter pages {inspection.non_letter_pages}") + if inspection.blank_pages: + failures.append(f"{inspection.path}: blank pages {inspection.blank_pages}") + if inspection.identity_hits: + failures.append(f"{inspection.path}: identity tokens {inspection.identity_hits}") + if inspection.fingerprint_hits: + failures.append(f"{inspection.path}: fingerprints {inspection.fingerprint_hits}") + if official.reference_start_page is None: + failures.append(f"{official.path}: References heading not found") + elif content_pages is not None and content_pages > 25: + failures.append(f"{official.path}: {content_pages} pages before References exceeds 25") + if abstract_words > 300: + failures.append(f"abstract has {abstract_words} words; IJDS maximum is 300") + if not abstract_single_paragraph: + failures.append("abstract is not one paragraph") + + return { + "status": "pass" if not failures else "fail", + "abstract_words": abstract_words, + "abstract_single_paragraph": abstract_single_paragraph, + "official_pre_reference_pages": content_pages, + "documents": [asdict(inspection) for inspection in inspections], + "failures": failures, + } + + +def main(argv: list[str] | None = None) -> int: + """Run the PDF audit and emit a compact JSON record.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + report = build_report() + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/manage_ijds_dvc_capsule.py b/scripts/manage_ijds_dvc_capsule.py new file mode 100644 index 0000000..118f745 --- /dev/null +++ b/scripts/manage_ijds_dvc_capsule.py @@ -0,0 +1,160 @@ +"""Run DVC operations over the active IJDS pointer set.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import yaml + +from src.ijds_audit.publication_sources import ( + active_lineage_run_tags, + load_source_registry, +) + +ROOT = Path(__file__).resolve().parents[1] +TARGETS_PATH = ROOT / "configs/crpto_publication_targets.yaml" +DVC_REMOTE = "dagshub" + + +def active_dvc_pointers( + *, + root: Path = ROOT, + targets_path: Path = TARGETS_PATH, +) -> list[Path]: + """Load and validate the data/model pointers for every active run tag.""" + payload: Any = yaml.safe_load(targets_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Publication targets must be a YAML mapping.") + contract = payload.get("active_scientific_contract") + if not isinstance(contract, dict): + raise TypeError("Publication targets omit active_scientific_contract.") + registry_value = contract.get("source_registry") + if not isinstance(registry_value, str) or not registry_value: + raise TypeError("Publication targets omit the active source_registry path.") + registry_path = (root / registry_value).resolve() + registry_path.relative_to(root.resolve()) + registry = load_source_registry(registry_path, repo_root=root) + active_lineage_run_tags(registry) + values = registry["dvc_pointers"] + + resolved: list[Path] = [] + for value in values: + path = (root / value).resolve() + path.relative_to(root.resolve()) + if path.suffix != ".dvc" or not path.is_file(): + raise FileNotFoundError(f"Invalid active DVC pointer: {path}") + resolved.append(path) + return resolved + + +def _pointer_arguments(pointers: Sequence[Path], *, root: Path) -> list[str]: + root_resolved = root.resolve() + return [path.resolve().relative_to(root_resolved).as_posix() for path in pointers] + + +def cloud_status_payload( + *, + root: Path = ROOT, + targets_path: Path = TARGETS_PATH, + pointers: Sequence[Path] | None = None, +) -> Any: + """Return DVC's machine-readable local-cache versus remote status.""" + selected = ( + list(pointers) + if pointers is not None + else active_dvc_pointers(root=root, targets_path=targets_path) + ) + command = [ + "dvc", + "status", + "--cloud", + "--remote", + DVC_REMOTE, + "--json", + *_pointer_arguments(selected, root=root), + ] + result = subprocess.run( + command, + cwd=root, + check=True, + capture_output=True, + text=True, + ) + output = result.stdout.strip() + if not output: + raise RuntimeError("DVC returned no JSON while verifying the active IJDS capsule.") + try: + return json.loads(output) + except json.JSONDecodeError as exc: + raise RuntimeError( + "DVC returned invalid JSON while verifying the active IJDS capsule." + ) from exc + + +def verify_remote( + *, + root: Path = ROOT, + targets_path: Path = TARGETS_PATH, + pointers: Sequence[Path] | None = None, +) -> None: + """Fail unless every active IJDS object is present and equal in the remote.""" + payload = cloud_status_payload(root=root, targets_path=targets_path, pointers=pointers) + if payload: + if isinstance(payload, dict): + items = list(payload.items()) + excerpt: Any = dict(items[:25]) + omitted = max(0, len(items) - len(excerpt)) + elif isinstance(payload, list): + excerpt = payload[:25] + omitted = max(0, len(payload) - len(excerpt)) + else: + excerpt = payload + omitted = 0 + detail = json.dumps(excerpt, indent=2, sort_keys=True) + suffix = f"\n... {omitted} additional status entries omitted." if omitted else "" + raise RuntimeError( + "Active IJDS DVC objects differ from or are absent in the configured remote:\n" + f"{detail}{suffix}" + ) + + +def run_dvc(action: str, *, cloud: bool = False) -> None: + """Run pull or status against only the active IJDS pointers.""" + pointers = active_dvc_pointers() + relative = _pointer_arguments(pointers, root=ROOT) + if action == "pull": + command = ["dvc", "pull", "--remote", DVC_REMOTE, *relative] + elif action == "push": + command = ["dvc", "push", "--remote", DVC_REMOTE, *relative] + elif action == "status": + command = ["dvc", "status", *(["--cloud"] if cloud else ["--no-updates"]), *relative] + elif action == "verify-remote": + verify_remote(pointers=pointers) + print(f"Active IJDS DVC capsule is fully available in remote {DVC_REMOTE!r}.") + return + else: + raise ValueError(f"Unsupported DVC action: {action}") + subprocess.run(command, cwd=ROOT, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("pull", "push", "status", "verify-remote")) + parser.add_argument( + "--cloud", + action="store_true", + help="Compare the local cache with the configured remote during status.", + ) + args = parser.parse_args() + if args.cloud and args.action != "status": + parser.error("--cloud is valid only with the status action") + run_dvc(args.action, cloud=bool(args.cloud)) + + +if __name__ == "__main__": + main() diff --git a/scripts/rebuild_test_predictions_from_frozen.py b/scripts/rebuild_test_predictions_from_frozen.py deleted file mode 100644 index 815e0e9..0000000 --- a/scripts/rebuild_test_predictions_from_frozen.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Rebuild ``data/processed/test_predictions.parquet`` from frozen PD binaries. - -Regenerates the canonical test-prediction surface WITHOUT retraining: it -loads an existing frozen model bundle (champion candidate, default-params -CatBoost baseline, logistic baseline, probability calibrator) and replays -the exact feature preparation used by ``train_pd_model.py`` (training -regime, temporal fit/val split for imputation medians, feature resolution -with stable core). This is the unification tool for keeping the canonical -prediction surface bit-consistent with the conformal certificate lineage. - -The script refuses to write unless the recomputed calibrated scores match -the frozen conformal intervals' ``y_pred`` within 1e-9, which ties the new -parquet to the exact funded-set certificate. - -Usage: - uv run python scripts/rebuild_test_predictions_from_frozen.py \ - --frozen-dir models/search_pd/pd-hpo-local-2026-04-03-1325 -""" - -from __future__ import annotations - -import argparse -import pickle -from pathlib import Path - -import numpy as np -import pandas as pd -import yaml -from catboost import CatBoostClassifier -from loguru import logger - -from scripts.train_pd_model import _prepare_catboost_frame, _prepare_logreg_frame -from src.models.calibration import load_probability_calibrator -from src.models.conformal import apply_probability_calibrator -from src.utils.io_utils import read_with_fallback - -ROOT = Path(__file__).resolve().parents[1] -TARGET = "default_flag" -FROZEN_INTERVALS_PATH = ROOT / "data" / "processed" / "conformal_intervals_mondrian.parquet" -CERTIFICATE_CONSISTENCY_TOL = 1e-9 - - -def main( - frozen_dir: str = "models/search_pd/pd-hpo-local-2026-04-03-1325", - config_path: str = "configs/crpto_pd_model.yaml", - output_path: str = "data/processed/test_predictions.parquet", - skip_certificate_check: bool = False, -) -> int: - frozen = ROOT / frozen_dir - config = yaml.safe_load((ROOT / config_path).read_text(encoding="utf-8")) - - tuned_path = frozen / "pd_candidate_model.cbm" - default_path = frozen / "pd_local_hpo_default.cbm" - logreg_path = frozen / "pd_logreg_baseline.pkl" - calibrator_path = frozen / "pd_candidate_calibrator.pkl" - for path in (tuned_path, default_path, logreg_path, calibrator_path): - if not path.is_file(): - raise FileNotFoundError(f"Missing frozen artifact: {path}") - - logger.info("Loading frozen PD bundle from {}", frozen) - tuned = CatBoostClassifier() - tuned.load_model(str(tuned_path)) - default = CatBoostClassifier() - default.load_model(str(default_path)) - with logreg_path.open("rb") as fh: - lr_bundle = pickle.load(fh) - lr_model = lr_bundle["model"] - logreg_features = [str(c) for c in lr_bundle["feature_names"]] - lr_fill = lr_bundle["fill_values"] - calibrator = load_probability_calibrator(str(calibrator_path)) - - data_cfg = config.get("data", {}) - test = read_with_fallback( - data_cfg.get("test_path", "data/processed/test_fe.parquet"), - "data/processed/test.parquet", - ) - - # CatBoost features come from the model's own metadata (the frozen - # binary records its training feature order and categorical indices), - # which is exactly what the conformal layer consumed. - catboost_features = list(tuned.feature_names_) - cat_idx = set(tuned.get_cat_feature_indices()) - categorical_features = [f for i, f in enumerate(catboost_features) if i in cat_idx] - logger.info( - "Frozen-bundle features: catboost={} logreg={} categorical={}", - len(catboost_features), - len(logreg_features), - len(categorical_features), - ) - - X_test_cb = _prepare_catboost_frame(test, catboost_features, categorical_features) - # The logistic baseline pickle carries its own training-median fill - # values, so the imputation is the literal April one, not a recompute. - X_test_lr, _ = _prepare_logreg_frame(test, logreg_features, fill_values=lr_fill) - - y_test = test[TARGET].astype(int) - y_prob_tuned = tuned.predict_proba(X_test_cb)[:, 1] - y_prob_default = default.predict_proba(X_test_cb)[:, 1] - y_prob_lr = lr_model.predict_proba(X_test_lr)[:, 1] - y_prob_final = apply_probability_calibrator(calibrator, y_prob_tuned) - - if not skip_certificate_check: - if not FROZEN_INTERVALS_PATH.is_file(): - raise FileNotFoundError( - f"Certificate consistency check needs {FROZEN_INTERVALS_PATH}; " - "pass --skip-certificate-check only for non-champion bundles." - ) - frozen_pred = pd.read_parquet(FROZEN_INTERVALS_PATH, columns=["y_pred"])["y_pred"].to_numpy( - dtype=float - ) - max_diff = float(np.max(np.abs(frozen_pred - np.asarray(y_prob_final, dtype=float)))) - logger.info( - "Certificate consistency: max |pd_calibrated - frozen y_pred| = {:.3e}", max_diff - ) - if max_diff > CERTIFICATE_CONSISTENCY_TOL: - raise AssertionError( - f"Rebuilt calibrated scores drift {max_diff:.3e} from the frozen " - f"conformal y_pred (tol {CERTIFICATE_CONSISTENCY_TOL:g}). The frozen " - "bundle does not match the certificate lineage; aborting write." - ) - - preds_df = pd.DataFrame( - { - "loan_id": test["id"].astype(str) if "id" in test.columns else test.index.astype(str), - "y_true": y_test.values.astype(float), - "y_prob_lr": y_prob_lr.astype(float), - "y_prob_cb_default": y_prob_default.astype(float), - "y_prob_cb_tuned": y_prob_tuned.astype(float), - "y_prob_final": np.asarray(y_prob_final, dtype=float), - "pd_calibrated": np.asarray(y_prob_final, dtype=float), - "pd_logreg": y_prob_lr.astype(float), - } - ) - out = ROOT / output_path - out.parent.mkdir(parents=True, exist_ok=True) - preds_df.to_parquet(out, index=False) - logger.info("Wrote {} ({} rows)", out, len(preds_df)) - - from sklearn.metrics import brier_score_loss, roc_auc_score - - auc = roc_auc_score(y_test, preds_df["pd_calibrated"]) - brier = brier_score_loss(y_test, preds_df["pd_calibrated"]) - logger.info("pd_calibrated metrics: AUC={:.4f} Brier={:.4f}", auc, brier) - return 0 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--frozen-dir", default="models/search_pd/pd-hpo-local-2026-04-03-1325") - parser.add_argument("--config", default="configs/crpto_pd_model.yaml") - parser.add_argument("--output", default="data/processed/test_predictions.parquet") - parser.add_argument("--skip-certificate-check", action="store_true") - args = parser.parse_args() - raise SystemExit( - main( - frozen_dir=args.frozen_dir, - config_path=args.config, - output_path=args.output, - skip_certificate_check=args.skip_certificate_check, - ) - ) diff --git a/scripts/render_quarto.sh b/scripts/render_quarto.sh deleted file mode 100644 index 3917a51..0000000 --- a/scripts/render_quarto.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -quarto "$@" diff --git a/scripts/run_alpha_sweep.py b/scripts/run_alpha_sweep.py deleted file mode 100644 index 1d7d5e9..0000000 --- a/scripts/run_alpha_sweep.py +++ /dev/null @@ -1,468 +0,0 @@ -"""Conformal prediction alpha sweep for CRPTO Pareto frontier. - -For each alpha level, generates split-conformal PD intervals (global and/or -Mondrian by grade), computes coverage/width diagnostics, and estimates a -simplified portfolio return proxy to trace the coverage-vs-return Pareto frontier. - -Usage: - uv run python scripts/run_alpha_sweep.py - uv run python scripts/run_alpha_sweep.py --mondrian - uv run python scripts/run_alpha_sweep.py --run-tag crpto-sweep-v1 --mondrian -""" - -from __future__ import annotations - -import argparse -import json -import pickle -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from catboost import CatBoostClassifier -from loguru import logger - -from src.models.conformal import ( - create_pd_intervals, - create_pd_intervals_mondrian, - validate_coverage, -) -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag - -SCHEMA_VERSION = "2026-03-16.1" - -ALPHAS = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] - -# Portfolio proxy: loans with pd_high below this threshold are "low-risk eligible". -# 0.20 corresponds to the upper CP interval staying below grade-B risk territory. -# (The operational pipeline uses per-grade alpha tuning which can reach <0.10 for grade A; -# a fixed-alpha sweep with alpha=0.10 gives Grade A min pd_high ~0.15, so 0.10 yields 0 eligible.) -LOW_RISK_THRESHOLD = 0.20 - -# Assumed average annual interest margin for portfolio return proxy (basis points) -AVG_INTEREST_MARGIN = 0.05 # 5% - - -def _load_model() -> CatBoostClassifier: - """Load canonical CatBoost PD model.""" - model_path = Path("models/pd_canonical.cbm") - if not model_path.exists(): - raise FileNotFoundError(f"Canonical model not found: {model_path}") - model = CatBoostClassifier() - model.load_model(str(model_path)) - logger.info(f"Loaded CatBoost model from {model_path}") - return model - - -def _load_calibrator(): - """Load canonical probability calibrator.""" - cal_path = Path("models/pd_canonical_calibrator.pkl") - if not cal_path.exists(): - logger.warning(f"Calibrator not found at {cal_path}, proceeding without calibration") - return None - with open(cal_path, "rb") as f: - calibrator = pickle.load(f) - logger.info(f"Loaded calibrator from {cal_path}") - return calibrator - - -def _load_contract() -> dict: - """Load model contract for feature names.""" - contract_path = Path("models/pd_model_contract.json") - if not contract_path.exists(): - raise FileNotFoundError(f"Model contract not found: {contract_path}") - with open(contract_path) as f: - return json.load(f) - - -def _load_data( - feature_cols: list[str], - model: Any, -) -> tuple[pd.DataFrame, pd.Series, pd.DataFrame, pd.Series, pd.Series, pd.Series, pd.Series]: - """Load calibration and test data with feature subsetting. - - Returns: - X_cal, y_cal, X_test, y_test, int_rate_test, grade_cal, grade_test - """ - cal_path = Path("data/processed/calibration_fe.parquet") - test_path = Path("data/processed/test_fe.parquet") - for p in [cal_path, test_path]: - if not p.exists(): - raise FileNotFoundError(f"Data file not found: {p}") - - cal = pd.read_parquet(cal_path) - test = pd.read_parquet(test_path) - logger.info(f"Loaded calibration ({len(cal):,} rows) and test ({len(test):,} rows)") - - # Use model's own cat feature indices for type resolution - cat_idx = set(model.get_cat_feature_indices()) - cat_names = {feature_cols[i] for i in cat_idx if i < len(feature_cols)} - - X_cal = cal[feature_cols].copy() - X_test = test[feature_cols].copy() - for col in feature_cols: - if col in cat_names: - X_cal[col] = X_cal[col].astype("string").fillna("UNKNOWN").astype(str) - X_test[col] = X_test[col].astype("string").fillna("UNKNOWN").astype(str) - else: - X_cal[col] = pd.to_numeric(X_cal[col], errors="coerce") - X_test[col] = pd.to_numeric(X_test[col], errors="coerce") - y_cal = cal["default_flag"] - y_test = test["default_flag"] - - # Interest rate for portfolio return proxy - int_rate_test = ( - pd.to_numeric(test["int_rate"], errors="coerce").fillna(0.12) / 100.0 - if "int_rate" in test.columns - else pd.Series(np.full(len(test), AVG_INTEREST_MARGIN), index=test.index) - ) - - # Grade for Mondrian conformal - grade_cal = ( - cal["grade"].fillna("UNKNOWN").astype(str) - if "grade" in cal.columns - else pd.Series(["UNKNOWN"] * len(cal)) - ) - grade_test = ( - test["grade"].fillna("UNKNOWN").astype(str) - if "grade" in test.columns - else pd.Series(["UNKNOWN"] * len(test)) - ) - - return X_cal, y_cal, X_test, y_test, int_rate_test, grade_cal, grade_test - - -def _compute_portfolio_proxy( - y_pred: np.ndarray, - y_intervals: np.ndarray, - int_rate: np.ndarray, -) -> dict[str, float]: - """Compute simplified portfolio return proxy. - - Approach: - - Identify loans where pd_high < LOW_RISK_THRESHOLD (eligible pool) - - Portfolio return = sum of (interest_margin - expected_loss) for eligible loans - - Expected loss (point) = pd_point * LGD_assumed (0.40) - - Expected loss (worst) = pd_high * LGD_assumed (0.40) - """ - lgd = 0.40 - pd_high = y_intervals[:, 1] - - eligible_mask = pd_high < LOW_RISK_THRESHOLD - n_eligible = int(eligible_mask.sum()) - n_total = len(y_pred) - eligible_pct = n_eligible / n_total if n_total > 0 else 0.0 - - if n_eligible == 0: - return { - "n_eligible": 0, - "n_total": n_total, - "eligible_pct": 0.0, - "portfolio_return_point": 0.0, - "portfolio_return_worst": 0.0, - "avg_pd_eligible": float("nan"), - "avg_pd_high_eligible": float("nan"), - "avg_int_rate_eligible": float("nan"), - } - - int_rate_arr = np.asarray(int_rate, dtype=float) - margin_eligible = int_rate_arr[eligible_mask] - pd_point_eligible = y_pred[eligible_mask] - pd_high_eligible = pd_high[eligible_mask] - - # Per-loan net return: interest margin minus expected loss - net_return_point = (margin_eligible - pd_point_eligible * lgd).sum() - net_return_worst = (margin_eligible - pd_high_eligible * lgd).sum() - - return { - "n_eligible": n_eligible, - "n_total": n_total, - "eligible_pct": float(eligible_pct), - "portfolio_return_point": float(net_return_point), - "portfolio_return_worst": float(net_return_worst), - "avg_pd_eligible": float(pd_point_eligible.mean()), - "avg_pd_high_eligible": float(pd_high_eligible.mean()), - "avg_int_rate_eligible": float(margin_eligible.mean()), - } - - -def run_sweep( - model: CatBoostClassifier, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - y_test: pd.Series, - int_rate_test: pd.Series, - alphas: list[float], - mondrian: bool = False, - grade_cal: pd.Series | None = None, - grade_test: pd.Series | None = None, - calibrator: Any | None = None, -) -> list[dict]: - """Run conformal interval sweep across alpha levels. - - Args: - model: Fitted CatBoost classifier. - X_cal: Calibration features. - y_cal: Calibration labels. - X_test: Test features. - y_test: Test labels. - int_rate_test: Interest rate series for test loans. - alphas: List of alpha (significance) levels. - mondrian: If True, use Mondrian (grade-conditional) intervals. - grade_cal: Grade labels for calibration set (required if mondrian=True). - grade_test: Grade labels for test set (required if mondrian=True). - - Returns: - List of dicts, one per alpha, with coverage and portfolio metrics. - """ - results: list[dict] = [] - y_test_arr = np.asarray(y_test, dtype=float) - int_rate_arr = np.asarray(int_rate_test, dtype=float) - method_label = "mondrian" if mondrian else "global" - - for alpha in alphas: - logger.info(f"--- [{method_label}] Alpha = {alpha:.2f} (target = {1 - alpha:.0%}) ---") - - if mondrian: - if grade_cal is None or grade_test is None: - raise ValueError("grade_cal and grade_test required for Mondrian sweep") - y_pred, y_intervals, diag = create_pd_intervals_mondrian( - model, - X_cal, - y_cal, - X_test, - grade_cal, - grade_test, - alpha=alpha, - calibrator=calibrator, - ) - # Compute per-group empirical coverage from y_test - g_test_arr = np.asarray(grade_test, dtype=str) - covered = (y_test_arr >= y_intervals[:, 0]) & (y_test_arr <= y_intervals[:, 1]) - group_coverages = {} - for g in sorted(set(g_test_arr)): - mask = g_test_arr == g - if mask.sum() > 0: - group_coverages[f"cov_{g}"] = float(covered[mask].mean()) - min_group_cov = ( - float(min(group_coverages.values())) if group_coverages else float("nan") - ) - else: - y_pred, y_intervals = create_pd_intervals( - model, - X_cal, - y_cal, - X_test, - alpha=alpha, - calibrator=calibrator, - ) - group_coverages = {} - min_group_cov = float("nan") - - # Coverage diagnostics - cov_metrics = validate_coverage(y_test_arr, y_intervals, alpha=alpha) - - # Width diagnostics - widths = y_intervals[:, 1] - y_intervals[:, 0] - width_stats = { - "avg_width": float(widths.mean()), - "median_width": float(np.median(widths)), - "min_width": float(widths.min()), - "max_width": float(widths.max()), - "std_width": float(widths.std()), - } - - # Portfolio proxy - portfolio = _compute_portfolio_proxy(y_pred, y_intervals, int_rate_arr) - - row = { - "alpha": alpha, - "confidence_level": 1.0 - alpha, - "method": method_label, - "min_group_coverage": min_group_cov, - **cov_metrics, - **width_stats, - **portfolio, - **group_coverages, - } - results.append(row) - - logger.info( - f" coverage={cov_metrics['empirical_coverage']:.4f}, " - f"avg_width={width_stats['avg_width']:.4f}, " - f"min_group_cov={min_group_cov:.4f}, " - f"eligible={portfolio['n_eligible']:,}/{portfolio['n_total']:,} " - f"({portfolio['eligible_pct']:.1%})" - ) - - return results - - -def main() -> int: - """Entry point for conformal alpha sweep.""" - parser = argparse.ArgumentParser( - description="Conformal prediction alpha sweep for Pareto frontier" - ) - parser.add_argument( - "--run-tag", - type=str, - default=None, - help="Run tag for artifact provenance tracking", - ) - parser.add_argument( - "--alphas", - type=str, - default=None, - help="Comma-separated alpha values (overrides default sweep)", - ) - parser.add_argument( - "--mondrian", - action="store_true", - default=False, - help="Run Mondrian (grade-conditional) sweep instead of global", - ) - parser.add_argument( - "--both", - action="store_true", - default=False, - help="Run both global and Mondrian sweeps (combined output)", - ) - args = parser.parse_args() - - run_tag = resolve_run_tag(args.run_tag, allow_untracked=True) - logger.info(f"Alpha sweep starting | run_tag={run_tag}") - - # Parse custom alphas if provided - alphas = ALPHAS - if args.alphas: - alphas = [float(a.strip()) for a in args.alphas.split(",")] - logger.info(f"Custom alphas: {alphas}") - - # Load artifacts - model = _load_model() - calibrator = _load_calibrator() - contract = _load_contract() - feature_cols = contract["feature_names"] - - X_cal, y_cal, X_test, y_test, int_rate_test, grade_cal, grade_test = _load_data( - feature_cols, model - ) - - # Determine which sweeps to run - run_global = not args.mondrian or args.both - run_mondrian = args.mondrian or args.both - - all_results: list[dict] = [] - - if run_global: - logger.info("=== Running GLOBAL conformal sweep ===") - global_results = run_sweep( - model, - X_cal, - y_cal, - X_test, - y_test, - int_rate_test, - alphas, - mondrian=False, - calibrator=calibrator, - ) - all_results.extend(global_results) - - if run_mondrian: - logger.info("=== Running MONDRIAN (grade-conditional) conformal sweep ===") - mondrian_results = run_sweep( - model, - X_cal, - y_cal, - X_test, - y_test, - int_rate_test, - alphas, - mondrian=True, - grade_cal=grade_cal, - grade_test=grade_test, - calibrator=calibrator, - ) - all_results.extend(mondrian_results) - - # Save Pareto parquet - pareto_df = pd.DataFrame(all_results) - method_suffix = ( - "_mondrian" if (args.mondrian and not args.both) else ("_both" if args.both else "") - ) - pareto_path = Path(f"data/processed/alpha_sweep_pareto{method_suffix}.parquet") - pareto_path.parent.mkdir(parents=True, exist_ok=True) - pareto_df.to_parquet(pareto_path, index=False) - logger.info(f"Saved Pareto frontier: {pareto_path} ({len(pareto_df)} rows)") - - # Save status JSON - metadata = build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=run_tag, - allow_untracked=True, - extra={ - "alphas": alphas, - "n_alphas": len(alphas), - "n_cal": len(X_cal), - "n_test": len(X_test), - "low_risk_threshold": LOW_RISK_THRESHOLD, - "avg_interest_margin": AVG_INTEREST_MARGIN, - "mondrian": run_mondrian, - "global": run_global, - }, - ) - - # Compute summaries per method - summaries: dict[str, object] = {} - for method in pareto_df["method"].unique(): - mdf = pareto_df[pareto_df["method"] == method] - best_idx = mdf["empirical_coverage"].idxmax() - tight_idx = mdf["avg_width"].idxmin() - elig_idx = mdf["n_eligible"].idxmax() - summaries[method] = { - "best_coverage_alpha": float(mdf.loc[best_idx, "alpha"]), - "best_coverage": float(mdf.loc[best_idx, "empirical_coverage"]), - "tightest_avg_width_alpha": float(mdf.loc[tight_idx, "alpha"]), - "tightest_avg_width": float(mdf.loc[tight_idx, "avg_width"]), - "most_eligible_alpha": float(mdf.loc[elig_idx, "alpha"]), - "most_eligible_n": int(mdf.loc[elig_idx, "n_eligible"]), - } - - status = { - **metadata, - "pareto_path": str(pareto_path), - "summaries": summaries, - } - status_path = Path("models/alpha_sweep_status.json") - status_path.parent.mkdir(parents=True, exist_ok=True) - with open(status_path, "w") as f: - json.dump(status, f, indent=2, default=str) - logger.info(f"Saved status: {status_path}") - - # Log summary table - logger.info("=== Alpha Sweep Summary ===") - summary_cols = [ - c - for c in [ - "method", - "alpha", - "empirical_coverage", - "min_group_coverage", - "avg_width", - "n_eligible", - "eligible_pct", - "portfolio_return_point", - "portfolio_return_worst", - ] - if c in pareto_df.columns - ] - logger.info("\n" + pareto_df[summary_cols].to_string(index=False)) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_comparison.py b/scripts/run_comparison.py deleted file mode 100644 index c9eea75..0000000 --- a/scripts/run_comparison.py +++ /dev/null @@ -1,1219 +0,0 @@ -"""Snapshot and compare baseline/current artifacts with promotion gates. - -Usage: - uv run python scripts/run_comparison.py snapshot --run-tag 2026-02-26-night - uv run python scripts/run_comparison.py compare --run-tag 2026-02-26-night -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import pickle -import subprocess -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml - -ROOT = Path(__file__).resolve().parents[1] -DATA = ROOT / "data" / "processed" -MODELS = ROOT / "models" -REPORTS = ROOT / "reports" -OUT_ROOT = REPORTS / "run_comparisons" -SCHEMA_VERSION = "2026-03-16.1" -COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS = 72 * 3600 -FAIRNESS_POLICY_PATH = ROOT / "configs" / "fairness_policy.yaml" - -# Artifacts documented as insights_only / not regenerated in every run. -# These are exempt from run_tag coherence checks when that is the ONLY mismatch. -_CAUSAL_INSIGHTS_ARTIFACTS = frozenset( - { - "models/causal_effect_status.json", - "models/causal_policy_rule.json", - "models/causal_policy_oot_status.json", - "models/cate_portfolio_status.json", - } -) - -# Gates whose pass/fail determines *operational* promotion readiness. -# artifact_coherence and semantic_coherence are bookkeeping gates (advisory). -_OPERATIONAL_GATE_NAMES = frozenset( - { - "pd_quality", - "conformal_policy", - "ab_no_regression", - "fairness_relative", - "fairness_absolute_business", - "survival_quality", - "export_contracts", - } -) - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def _read_pickle(path: Path) -> Any: - with open(path, "rb") as f: - return pickle.load(f) - - -def _sha256(path: Path) -> str | None: - if not path.exists(): - return None - h = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - h.update(chunk) - return h.hexdigest() - - -def _safe_float(value: Any, default: float = float("nan")) -> float: - try: - return float(value) - except Exception: - return default - - -def _to_builtin(value: Any) -> Any: - """Recursively convert numpy scalars/containers to JSON-serializable Python types.""" - if isinstance(value, dict): - return {str(k): _to_builtin(v) for k, v in value.items()} - if isinstance(value, list): - return [_to_builtin(v) for v in value] - if isinstance(value, tuple): - return tuple(_to_builtin(v) for v in value) - if isinstance(value, np.generic): - return value.item() - return value - - -def _git(cmd: list[str]) -> str: - try: - p = subprocess.run(cmd, cwd=str(ROOT), check=False, text=True, capture_output=True) - return p.stdout.strip() - except Exception: - return "" - - -def _path_for_report(path: Path) -> str: - try: - return str(path.relative_to(ROOT)) - except ValueError: - return str(path) - - -def _versions_snapshot() -> dict[str, Any]: - out: dict[str, Any] = {} - try: - proc = subprocess.run( - ["uv", "pip", "list", "--python", ".venv/bin/python", "--format=json"], - cwd=str(ROOT), - check=False, - text=True, - capture_output=True, - ) - if proc.returncode == 0 and proc.stdout.strip(): - pkgs = json.loads(proc.stdout) - out["main_env"] = { - p["name"]: p["version"] for p in pkgs if "name" in p and "version" in p - } - except Exception: - pass - try: - proc = subprocess.run( - ["conda", "list", "-n", "rapids", "--json"], - cwd=str(ROOT), - check=False, - text=True, - capture_output=True, - ) - if proc.returncode == 0 and proc.stdout.strip(): - pkgs = json.loads(proc.stdout) - out["rapids_env"] = { - p["name"]: p["version"] for p in pkgs if "name" in p and "version" in p - } - except Exception: - pass - return out - - -def _collect_metrics() -> dict[str, Any]: - metrics_summary_raw = _read_json(REPORTS / "dvc" / "metrics_summary.json") - if isinstance(metrics_summary_raw.get("metrics"), dict): - metrics_summary = dict(metrics_summary_raw.get("metrics", {})) - metrics_summary_meta = { - "schema_version": metrics_summary_raw.get("schema_version"), - "generated_at_utc": metrics_summary_raw.get("generated_at_utc"), - "run_tag": metrics_summary_raw.get("run_tag"), - } - else: - metrics_summary = metrics_summary_raw - metrics_summary_meta = {} - model_comparison = _read_json(DATA / "model_comparison.json") - pipeline_summary = _read_json(DATA / "pipeline_summary.json") - conformal = _read_json(MODELS / "conformal_policy_status.json") - fairness = _read_json(MODELS / "fairness_audit_status.json") - fairness_decision_policy = _read_json(MODELS / "fairness_decision_policy.json") - governance = _read_json(MODELS / "governance_status.json") - ab_status = _read_json(MODELS / "ab_simulation_status.json") - causal_effect_status = _read_json(MODELS / "causal_effect_status.json") - causal_rule_status = _read_json(MODELS / "causal_policy_rule.json") - causal_oot_status = _read_json(MODELS / "causal_policy_oot_status.json") - cate_status = _read_json(MODELS / "cate_portfolio_status.json") - lgd_ead_conformal_status = _read_json(MODELS / "conformal_lgd_ead_status.json") - threshold_semantics = _read_json(MODELS / "threshold_semantics.json") - time_series_status = _read_json(MODELS / "time_series_status.json") - storytelling_snapshot = _read_json(REPORTS / "storytelling_snapshot.json") - - survival_summary = {} - survival_path = MODELS / "survival_summary.pkl" - if survival_path.exists(): - try: - survival_summary = _read_pickle(survival_path) - except Exception: - survival_summary = {} - - ifrs9 = {} - ifrs9_path = DATA / "ifrs9_scenario_summary.parquet" - if ifrs9_path.exists(): - try: - df = pd.read_parquet(ifrs9_path) - for _, row in df.iterrows(): - key = str(row.get("scenario", "unknown")) - ifrs9[key] = { - "total_ecl": _safe_float(row.get("total_ecl", np.nan)), - } - except Exception: - ifrs9 = {} - - robustness_summary = [] - rob_path = DATA / "portfolio_robustness_summary.parquet" - if rob_path.exists(): - try: - robustness_summary = pd.read_parquet(rob_path).to_dict(orient="records") - except Exception: - robustness_summary = [] - - return { - "dvc_metrics": metrics_summary, - "dvc_metrics_meta": metrics_summary_meta, - "model_comparison": model_comparison, - "pipeline_summary": pipeline_summary, - "conformal_status": conformal, - "fairness_status": fairness, - "fairness_decision_policy": fairness_decision_policy, - "governance_status": governance, - "survival_summary": survival_summary, - "ifrs9_summary": ifrs9, - "portfolio_robustness_summary": robustness_summary, - "ab_simulation_status": ab_status, - "causal_effect_status": causal_effect_status, - "causal_policy_rule_status": causal_rule_status, - "causal_policy_oot_status": causal_oot_status, - "cate_portfolio_status": cate_status, - "conformal_lgd_ead_status": lgd_ead_conformal_status, - "threshold_semantics": threshold_semantics, - "time_series_status": time_series_status, - "storytelling_snapshot": storytelling_snapshot, - } - - -def _artifact_index() -> dict[str, dict[str, Any]]: - targets = { - "reports/dvc/metrics_summary.json": REPORTS / "dvc" / "metrics_summary.json", - "data/processed/model_comparison.json": DATA / "model_comparison.json", - "data/processed/pipeline_summary.json": DATA / "pipeline_summary.json", - "models/conformal_policy_status.json": MODELS / "conformal_policy_status.json", - "models/fairness_audit_status.json": MODELS / "fairness_audit_status.json", - "models/governance_status.json": MODELS / "governance_status.json", - "models/conformal_lgd_ead_status.json": MODELS / "conformal_lgd_ead_status.json", - "models/survival_summary.pkl": MODELS / "survival_summary.pkl", - "data/processed/portfolio_robustness_summary.parquet": DATA - / "portfolio_robustness_summary.parquet", - "data/processed/portfolio_robustness_frontier.parquet": DATA - / "portfolio_robustness_frontier.parquet", - "data/processed/ifrs9_scenario_summary.parquet": DATA / "ifrs9_scenario_summary.parquet", - "reports/gpu_benchmark/gpu_bench_meta.json": REPORTS - / "gpu_benchmark" - / "gpu_bench_meta.json", - "reports/gpu_benchmark/cuml_benchmark.csv": REPORTS - / "gpu_benchmark" - / "cuml_benchmark.csv", - "reports/gpu_benchmark/cugraph_benchmark.csv": REPORTS - / "gpu_benchmark" - / "cugraph_benchmark.csv", - "reports/gpu_benchmark/cuopt_benchmark.csv": REPORTS - / "gpu_benchmark" - / "cuopt_benchmark.csv", - "reports/gpu_benchmark/cudf_polars_benchmark.csv": REPORTS - / "gpu_benchmark" - / "cudf_polars_benchmark.csv", - "reports/gpu_benchmark/cupy_benchmark.csv": REPORTS - / "gpu_benchmark" - / "cupy_benchmark.csv", - } - out: dict[str, dict[str, Any]] = {} - for key, path in targets.items(): - out[key] = { - "exists": path.exists(), - "sha256": _sha256(path), - "size_bytes": int(path.stat().st_size) if path.exists() else 0, - } - return out - - -def _snapshot_payload(run_tag: str) -> dict[str, Any]: - pipeline_family = str(os.environ.get("PIPELINE_FAMILY", "")).strip() or None - pipeline_profile = str(os.environ.get("PIPELINE_PROFILE", "")).strip() or None - artifact_scope = str(os.environ.get("PIPELINE_ARTIFACT_SCOPE", "")).strip() or None - promotion_state = str(os.environ.get("PIPELINE_PROMOTION_STATE", "")).strip() or None - upstream = str(os.environ.get("UPSTREAM_CANONICAL_RUN_TAG", "")).strip() or None - writes = str(os.environ.get("WRITES_CANONICAL_ARTIFACTS", "")).strip() - return { - "schema_version": SCHEMA_VERSION, - "run_tag": run_tag, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "pipeline_family": pipeline_family, - "pipeline_profile": pipeline_profile, - "artifact_scope": artifact_scope, - "promotion_state": promotion_state, - "upstream_canonical_run_tag": upstream, - "writes_canonical_artifacts": None - if writes == "" - else writes.lower() in {"1", "true", "yes", "on"}, - "git": { - "head": _git(["git", "rev-parse", "HEAD"]), - "branch": _git(["git", "branch", "--show-current"]), - "status_short": _git(["git", "status", "--short"]), - }, - "versions": _versions_snapshot(), - "artifacts": _artifact_index(), - "metrics": _collect_metrics(), - } - - -@dataclass -class GateResult: - name: str - passed: bool - details: dict[str, Any] - - -@dataclass -class StatusMetadataObservation: - row: dict[str, Any] - run_tag: str | None - generated_at: datetime | None - missing_metadata: bool - mismatched_run_tag: bool - - -def _load_fairness_policy_contract(config_path: Path = FAIRNESS_POLICY_PATH) -> dict[str, Any]: - """Return fairness business threshold contract from policy config.""" - if not config_path.exists(): - return {} - try: - payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - except Exception: - return {} - policy = payload.get("policy", {}) if isinstance(payload, dict) else {} - threshold_policy = payload.get("threshold_policy", {}) if isinstance(payload, dict) else {} - out: dict[str, Any] = {} - if isinstance(policy, dict) and "prediction_threshold" in policy: - out["prediction_threshold"] = _safe_float(policy.get("prediction_threshold")) - if isinstance(policy, dict) and "outcome_mode" in policy: - out["outcome_mode"] = str(policy.get("outcome_mode") or "").strip().lower() - if isinstance(threshold_policy, dict) and "use_artifact" in threshold_policy: - out["use_artifact"] = bool(threshold_policy.get("use_artifact")) - try: - out["policy_path"] = str(config_path.relative_to(ROOT)) - except ValueError: - out["policy_path"] = str(config_path) - return out - - -def _fairness_n_attributes(payload: dict[str, Any]) -> int: - for key in ("n_attributes", "n_total"): - value = payload.get(key) - try: - if value is not None: - return max(0, int(value)) - except Exception: - continue - attrs = payload.get("attributes") - if isinstance(attrs, list): - return len(attrs) - return 0 - - -def _parse_iso_datetime(value: Any) -> datetime | None: - if not isinstance(value, str) or not value.strip(): - return None - candidate = value.strip().replace("Z", "+00:00") - try: - return datetime.fromisoformat(candidate) - except ValueError: - return None - - -def _status_metadata_sources(cur_metrics: dict[str, Any]) -> dict[str, Any]: - sources = { - "reports/dvc/metrics_summary.json": cur_metrics.get("dvc_metrics_meta", {}), - "data/processed/pipeline_summary.json": cur_metrics.get("pipeline_summary", {}), - "models/conformal_policy_status.json": cur_metrics.get("conformal_status", {}), - "models/fairness_audit_status.json": cur_metrics.get("fairness_status", {}), - "models/governance_status.json": cur_metrics.get("governance_status", {}), - "models/ab_simulation_status.json": cur_metrics.get("ab_simulation_status", {}), - } - optional_sources = { - "models/causal_effect_status.json": cur_metrics.get("causal_effect_status", {}), - "models/causal_policy_rule.json": cur_metrics.get("causal_policy_rule_status", {}), - "models/causal_policy_oot_status.json": cur_metrics.get("causal_policy_oot_status", {}), - "models/cate_portfolio_status.json": cur_metrics.get("cate_portfolio_status", {}), - "models/time_series_status.json": cur_metrics.get("time_series_status", {}), - } - for artifact_name, payload in optional_sources.items(): - if isinstance(payload, dict) and payload: - sources[artifact_name] = payload - return sources - - -def _missing_status_fields( - *, - schema_version: Any, - generated_at_utc: Any, - run_tag: Any, -) -> list[str]: - return [ - key - for key, value in { - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - }.items() - if value in (None, "", []) - ] - - -def _status_metadata_observation( - artifact_name: str, - payload: Any, - *, - expected_run_tag: str, -) -> StatusMetadataObservation: - payload = payload if isinstance(payload, dict) else {} - schema_version = payload.get("schema_version") - generated_at_utc = payload.get("generated_at_utc") - run_tag = payload.get("run_tag") - missing_fields = _missing_status_fields( - schema_version=schema_version, - generated_at_utc=generated_at_utc, - run_tag=run_tag, - ) - run_tag_text = None if run_tag in (None, "", []) else str(run_tag) - return StatusMetadataObservation( - row={ - "artifact": artifact_name, - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - "missing_metadata_fields": missing_fields, - }, - run_tag=run_tag_text, - generated_at=_parse_iso_datetime(generated_at_utc), - missing_metadata=bool(missing_fields), - mismatched_run_tag=run_tag_text is not None and run_tag_text != expected_run_tag, - ) - - -def _timestamp_skew_seconds(generated_times: list[datetime]) -> float | None: - if len(generated_times) < 2: - return None - return float((max(generated_times) - min(generated_times)).total_seconds()) - - -def _run_tag_coherence( - *, - run_tags: list[str], - mismatched_artifacts: list[str], - expected_run_tag: str, -) -> dict[str, Any]: - unique_run_tags = sorted(set(run_tags)) - run_tag_consistent = len(unique_run_tags) == 1 - run_tag_matches_expected = run_tag_consistent and unique_run_tags == [expected_run_tag] - non_causal_mismatches = [ - artifact for artifact in mismatched_artifacts if artifact not in _CAUSAL_INSIGHTS_ARTIFACTS - ] - causal_only_mismatch = bool(mismatched_artifacts) and len(non_causal_mismatches) == 0 - return { - "run_tags_observed": unique_run_tags, - "run_tag_consistent": run_tag_consistent, - "run_tag_matches_expected": run_tag_matches_expected, - "run_tag_matches_expected_operational": run_tag_matches_expected or causal_only_mismatch, - "causal_only_mismatch": causal_only_mismatch, - "non_causal_mismatched_run_tag_artifacts": non_causal_mismatches, - } - - -def _collect_status_metadata( - cur_metrics: dict[str, Any], *, expected_run_tag: str -) -> dict[str, Any]: - observations = [ - _status_metadata_observation( - artifact_name, - payload, - expected_run_tag=expected_run_tag, - ) - for artifact_name, payload in _status_metadata_sources(cur_metrics).items() - ] - rows = [observation.row for observation in observations] - run_tags = [ - observation.run_tag for observation in observations if observation.run_tag is not None - ] - generated_times = [ - observation.generated_at - for observation in observations - if observation.generated_at is not None - ] - missing_metadata_artifacts = [ - str(observation.row["artifact"]) - for observation in observations - if observation.missing_metadata - ] - mismatched_run_tag_artifacts = [ - str(observation.row["artifact"]) - for observation in observations - if observation.mismatched_run_tag - ] - - tag_coherence = _run_tag_coherence( - run_tags=run_tags, - mismatched_artifacts=mismatched_run_tag_artifacts, - expected_run_tag=expected_run_tag, - ) - timestamp_skew_seconds = _timestamp_skew_seconds(generated_times) - timestamp_coherent = timestamp_skew_seconds is None or timestamp_skew_seconds <= float( - COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS - ) - all_have_metadata = len(missing_metadata_artifacts) == 0 - passed = bool( - all_have_metadata - and tag_coherence["run_tag_matches_expected_operational"] - and timestamp_coherent - ) - - return { - "expected_run_tag": expected_run_tag, - "critical_artifacts": rows, - "all_have_metadata": all_have_metadata, - "missing_metadata_artifacts": missing_metadata_artifacts, - **tag_coherence, - "mismatched_run_tag_artifacts": mismatched_run_tag_artifacts, - "timestamp_skew_seconds": timestamp_skew_seconds, - "timestamp_coherent": bool(timestamp_coherent), - "timestamp_max_skew_seconds": int(COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS), - "passed": passed, - } - - -def _gate_artifact_coherence(cur_metrics: dict[str, Any], run_tag: str) -> GateResult: - details = _collect_status_metadata(cur_metrics, expected_run_tag=run_tag) - return GateResult("artifact_coherence", bool(details.get("passed", False)), details) - - -def _finite_float(value: Any) -> float | None: - out = _safe_float(value) - if np.isnan(out): - return None - return float(out) - - -def _coherent_float_group(values: list[float | None], *, tol: float = 1e-9) -> bool: - observed = [float(v) for v in values if v is not None] - if len(observed) <= 1: - return True - return max(observed) - min(observed) <= tol - - -def _gate_semantic_coherence(cur_metrics: dict[str, Any]) -> GateResult: - threshold_semantics = cur_metrics.get("threshold_semantics", {}) or {} - fairness_status = cur_metrics.get("fairness_status", {}) or {} - fairness_policy = cur_metrics.get("fairness_decision_policy", {}) or {} - time_series = cur_metrics.get("time_series_status", {}) or {} - storytelling = cur_metrics.get("storytelling_snapshot", {}) or {} - conformal = cur_metrics.get("conformal_status", {}) or {} - - operational_thresholds = { - "threshold_semantics.fairness_primary_threshold": _finite_float( - threshold_semantics.get("fairness_primary_threshold") - ), - "threshold_semantics.decision_policy_global_threshold": _finite_float( - threshold_semantics.get("decision_policy_global_threshold") - ), - "fairness_status.primary_threshold": _finite_float( - fairness_status.get("primary_threshold") - ), - "fairness_status.prediction_threshold": _finite_float( - fairness_status.get("prediction_threshold") - ), - "fairness_decision_policy.global_threshold": _finite_float( - fairness_policy.get("global_threshold") - ), - "storytelling.headline_metrics.fairness_primary_threshold": _finite_float( - (storytelling.get("headline_metrics", {}) or {}).get("fairness_primary_threshold") - ), - } - operational_thresholds_ok = _coherent_float_group(list(operational_thresholds.values())) - - pd_internal_threshold = _finite_float(threshold_semantics.get("pd_internal_selected_threshold")) - operational_threshold = _finite_float(threshold_semantics.get("fairness_primary_threshold")) - threshold_role_separation_ok = ( - pd_internal_threshold is None - or operational_threshold is None - or abs(pd_internal_threshold - operational_threshold) > 1e-9 - ) - - time_series_interval_promotable = bool( - (time_series.get("interval_champion", {}) or {}).get("promotable", False) - ) - time_series_final_decision = str( - (time_series.get("final_interval_decision", {}) or {}).get("status", "") or "" - ).strip() - storytelling_ts_promotable = storytelling.get("time_series_interval_promotable") - storytelling_ts_decision = storytelling.get("time_series_final_interval_decision") - - time_series_storytelling_ok = ( - storytelling_ts_promotable is None - or bool(storytelling_ts_promotable) == time_series_interval_promotable - ) and (storytelling_ts_decision in (None, "", time_series_final_decision)) - - conformal_gate_pass = bool( - conformal.get("gate_overall_pass", conformal.get("overall_pass", False)) - ) - storytelling_conformal_gate = storytelling.get( - "conformal_overall_pass", storytelling.get("conformal_gate_overall_pass") - ) - storytelling_conformal_ok = storytelling_conformal_gate in ( - None, - conformal_gate_pass, - ) and storytelling.get("conformal_methodological_justification_pass") in ( - None, - bool(conformal.get("methodological_justification_pass", False)), - ) - - checks = { - "operational_thresholds_ok": bool(operational_thresholds_ok), - "threshold_role_separation_ok": bool(threshold_role_separation_ok), - "time_series_storytelling_ok": bool(time_series_storytelling_ok), - "storytelling_conformal_ok": bool(storytelling_conformal_ok), - } - return GateResult( - "semantic_coherence", - bool(all(checks.values())), - { - "checks": checks, - "operational_thresholds": operational_thresholds, - "time_series": { - "status_interval_promotable": time_series_interval_promotable, - "status_final_decision": time_series_final_decision, - "storytelling_interval_promotable": storytelling_ts_promotable, - "storytelling_final_decision": storytelling_ts_decision, - }, - "conformal": { - "status_gate_overall_pass": conformal_gate_pass, - "storytelling_gate_overall_pass": storytelling_conformal_gate, - "status_methodological_justification_pass": bool( - conformal.get("methodological_justification_pass", False) - ), - "storytelling_methodological_justification_pass": storytelling.get( - "conformal_methodological_justification_pass" - ), - }, - }, - ) - - -def _gate_pd(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("dvc_metrics", {}) - c = cur.get("dvc_metrics", {}) - b_auc = _safe_float(b.get("pd.auc")) - c_auc = _safe_float(c.get("pd.auc")) - b_ece = _safe_float(b.get("pd.ece")) - c_ece = _safe_float(c.get("pd.ece")) - b_d2 = _safe_float(b.get("pd.d2_brier")) - c_d2 = _safe_float(c.get("pd.d2_brier")) - d2_brier_tolerance = 0.002 - auc_ok = np.isnan(b_auc) or np.isnan(c_auc) or (c_auc >= b_auc - 0.005) - ece_ok = np.isnan(b_ece) or np.isnan(c_ece) or (c_ece <= b_ece * 1.2 + 1e-12) - d2_ok = np.isnan(b_d2) or np.isnan(c_d2) or (c_d2 >= b_d2 - d2_brier_tolerance) - return GateResult( - "pd_quality", - bool(auc_ok and ece_ok and d2_ok), - { - "baseline": {"auc": b_auc, "ece": b_ece, "d2_brier": b_d2}, - "current": {"auc": c_auc, "ece": c_ece, "d2_brier": c_d2}, - "checks": {"auc_ok": auc_ok, "ece_ok": ece_ok, "d2_brier_ok": d2_ok}, - "thresholds": { - "auc_min_delta": -0.005, - "ece_max_multiplier": 1.2, - "d2_brier_tolerance": d2_brier_tolerance, - }, - "deltas": { - "auc_delta": None if np.isnan(b_auc) or np.isnan(c_auc) else c_auc - b_auc, - "ece_delta": None if np.isnan(b_ece) or np.isnan(c_ece) else c_ece - b_ece, - "d2_brier_delta": None if np.isnan(b_d2) or np.isnan(c_d2) else c_d2 - b_d2, - }, - }, - ) - - -def _gate_conformal(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("conformal_status", {}) - c = cur.get("conformal_status", {}) - b_cov90 = _safe_float(b.get("coverage_90")) - c_cov90 = _safe_float(c.get("coverage_90")) - b_cov95 = _safe_float(b.get("coverage_95")) - c_cov95 = _safe_float(c.get("coverage_95")) - b_min_grp = _safe_float(b.get("min_group_coverage_90")) - c_min_grp = _safe_float(c.get("min_group_coverage_90")) - b_winkler90 = _safe_float(b.get("winkler_90")) - c_winkler90 = _safe_float(c.get("winkler_90")) - b_critical = _safe_float(b.get("critical_alerts")) - c_critical = _safe_float(c.get("critical_alerts")) - cov90_ok = np.isnan(b_cov90) or np.isnan(c_cov90) or (c_cov90 >= b_cov90 - 0.03) - cov95_ok = np.isnan(b_cov95) or np.isnan(c_cov95) or (c_cov95 >= b_cov95 - 0.03) - min_group_ok = np.isnan(b_min_grp) or np.isnan(c_min_grp) or (c_min_grp >= b_min_grp - 0.03) - # Business/ops checks: keep Winkler and critical alerts explicit in promotion gate. - winkler90_ok = ( - np.isnan(b_winkler90) or np.isnan(c_winkler90) or (c_winkler90 <= b_winkler90 + 0.10) - ) - critical_alerts_ok = np.isnan(b_critical) or np.isnan(c_critical) or (c_critical <= b_critical) - - conformal_promotion_pass = bool( - cov90_ok and cov95_ok and min_group_ok and winkler90_ok and critical_alerts_ok - ) - - return GateResult( - "conformal_policy", - conformal_promotion_pass, - { - "baseline": b, - "current": c, - "checks": { - "coverage90_ok": bool(cov90_ok), - "coverage95_ok": bool(cov95_ok), - "min_group_coverage90_ok": bool(min_group_ok), - "winkler90_ok": bool(winkler90_ok), - "critical_alerts_ok": bool(critical_alerts_ok), - "conformal_promotion_pass": bool(conformal_promotion_pass), - }, - "diagnostics": { - "retired_backtest_checks": (c.get("methodological_justification", {}) or {}).get( - "retired_backtest_checks", [] - ), - "policy_overall_pass": bool(c.get("overall_pass", False)), - }, - }, - ) - - -def _ab_total_returns(status: dict[str, Any]) -> tuple[float, float]: - return ( - _safe_float((status.get("metrics_a") or {}).get("total_return")), - _safe_float((status.get("metrics_b") or {}).get("total_return")), - ) - - -def _ab_current_no_regression( - status: dict[str, Any], - control_return: float, - robust_return: float, -) -> tuple[bool, float, float, dict[str, Any]]: - no_reg = ( - status.get("no_regression", {}) if isinstance(status.get("no_regression"), dict) else {} - ) - cross_gate = ( - status.get("cross_scenario_gate", {}) - if isinstance(status.get("cross_scenario_gate"), dict) - else {} - ) - diff = _safe_float( - no_reg.get("diff_total_return"), - default=( - robust_return - control_return - if np.isfinite(control_return) and np.isfinite(robust_return) - else float("nan") - ), - ) - tolerance = _safe_float( - no_reg.get("tolerance_total_return"), - default=(abs(control_return) * 0.05 if np.isfinite(control_return) else float("nan")), - ) - passed = ( - bool(no_reg.get("passed")) - if "passed" in no_reg - else (np.isnan(diff) or np.isnan(tolerance) or (diff >= -tolerance)) - ) - if str(status.get("decision_scenario", "")).strip() == "selective_ambiguity_defer" and bool( - cross_gate.get("passed", False) - ): - passed = True - return bool(passed), float(diff), float(tolerance), cross_gate - - -def _ab_baseline_checks( - *, - baseline_control_return: float, - baseline_robust_return: float, - current_control_return: float, - current_robust_return: float, - current_diff: float, -) -> tuple[dict[str, bool], float]: - baseline_diff = ( - baseline_robust_return - baseline_control_return - if np.isfinite(baseline_control_return) and np.isfinite(baseline_robust_return) - else float("nan") - ) - baseline_tol = ( - abs(baseline_control_return) * 0.05 - if np.isfinite(baseline_control_return) - else float("nan") - ) - return ( - { - "control_vs_baseline_ok": bool( - np.isnan(baseline_control_return) - or np.isnan(current_control_return) - or (current_control_return >= baseline_control_return - baseline_tol) - ), - "robust_vs_baseline_ok": bool( - np.isnan(baseline_robust_return) - or np.isnan(current_robust_return) - or (current_robust_return >= baseline_robust_return - baseline_tol) - ), - "gap_vs_baseline_ok": bool( - np.isnan(baseline_diff) - or np.isnan(current_diff) - or (current_diff >= baseline_diff - baseline_tol) - ), - }, - float(baseline_diff), - ) - - -def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("ab_simulation_status", {}) - c = cur.get("ab_simulation_status", {}) - - b_a, b_b = _ab_total_returns(b) - c_a, c_b = _ab_total_returns(c) - self_no_reg_ok, c_diff, c_tol, cross_gate = _ab_current_no_regression(c, c_a, c_b) - baseline_checks, b_diff = _ab_baseline_checks( - baseline_control_return=b_a, - baseline_robust_return=b_b, - current_control_return=c_a, - current_robust_return=c_b, - current_diff=c_diff, - ) - - passed = bool(self_no_reg_ok) - - comparison = c.get("comparison", {}) if isinstance(c.get("comparison"), dict) else {} - return GateResult( - "ab_no_regression", - passed, - { - "checks": { - "self_no_regression_ok": bool(self_no_reg_ok), - "cross_scenario_gate_ok": bool(cross_gate.get("passed", False)), - **baseline_checks, - }, - "warnings": { - "control_vs_baseline_warning": bool(not baseline_checks["control_vs_baseline_ok"]), - "robust_vs_baseline_warning": bool(not baseline_checks["robust_vs_baseline_ok"]), - "gap_vs_baseline_warning": bool(not baseline_checks["gap_vs_baseline_ok"]), - }, - "current": { - "control_total_return": c_a, - "robust_total_return": c_b, - "diff_total_return": c_diff, - "tolerance_total_return": c_tol, - "n_candidates_used": c.get("n_candidates_used"), - }, - "baseline": { - "control_total_return": b_a, - "robust_total_return": b_b, - "diff_total_return": b_diff, - }, - "diagnostics": { - "p_value": _safe_float(comparison.get("p_value")), - "significant": bool(comparison.get("significant", False)), - "significance_role": "diagnostic", - "gate_mode": "no_regression", - }, - }, - ) - - -def _gate_fairness(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("fairness_status", {}) - c = cur.get("fairness_status", {}) - b_passed = int(b.get("n_passed", 0) or 0) - c_passed = int(c.get("n_passed", 0) or 0) - b_total = _fairness_n_attributes(b) - c_total = _fairness_n_attributes(c) - return GateResult( - "fairness_relative", - c_passed >= b_passed, - { - "baseline_n_passed": b_passed, - "current_n_passed": c_passed, - "baseline_n_total": b_total, - "current_n_total": c_total, - "baseline_overall_pass": bool(b.get("overall_pass", False)), - "current_overall_pass": bool(c.get("overall_pass", False)), - }, - ) - - -def _gate_fairness_absolute_business(_base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - c = cur.get("fairness_status", {}) - policy_contract = _load_fairness_policy_contract() - - expected_threshold = _safe_float(policy_contract.get("prediction_threshold")) - expected_outcome_mode = str(policy_contract.get("outcome_mode", "") or "").strip().lower() - expected_use_artifact = policy_contract.get("use_artifact", None) - - current_threshold = _safe_float(c.get("prediction_threshold")) - current_source = str(c.get("prediction_threshold_source", "") or "").strip() - current_outcome_mode = str(c.get("outcome_mode", "") or "").strip().lower() - source_uses_artifact = current_source.startswith("artifact") - if not source_uses_artifact: - source_uses_artifact = "artifact" in current_source - - threshold_ok = bool(expected_use_artifact) or ( - np.isnan(expected_threshold) - or np.isnan(current_threshold) - or bool(abs(current_threshold - expected_threshold) <= 1e-9) - ) - if expected_use_artifact is None or ( - current_source == "" and bool(expected_use_artifact) is False - ): - source_ok = True - else: - source_ok = source_uses_artifact == bool(expected_use_artifact) - outcome_mode_ok = (not expected_outcome_mode) or (current_outcome_mode == expected_outcome_mode) - - n_passed = int(c.get("n_passed", 0) or 0) - n_total = _fairness_n_attributes(c) - all_attributes_ok = n_total <= 0 or n_passed >= n_total - overall_pass_ok = bool(c.get("overall_pass", False)) - - passed = bool( - overall_pass_ok and all_attributes_ok and threshold_ok and source_ok and outcome_mode_ok - ) - return GateResult( - "fairness_absolute_business", - passed, - { - "policy_contract": { - "prediction_threshold": expected_threshold, - "outcome_mode": expected_outcome_mode, - "use_artifact": expected_use_artifact, - "policy_path": policy_contract.get("policy_path"), - }, - "current": { - "prediction_threshold": current_threshold, - "prediction_threshold_source": current_source, - "outcome_mode": current_outcome_mode, - "n_passed": n_passed, - "n_total": n_total, - "overall_pass": bool(c.get("overall_pass", False)), - }, - "checks": { - "overall_pass_ok": overall_pass_ok, - "all_attributes_ok": bool(all_attributes_ok), - "threshold_match_ok": bool(threshold_ok), - "threshold_source_ok": bool(source_ok), - "outcome_mode_ok": bool(outcome_mode_ok), - }, - }, - ) - - -def _gate_survival(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("survival_summary", {}) - c = cur.get("survival_summary", {}) - b_cox = _safe_float(b.get("cox_concordance_index")) - c_cox = _safe_float(c.get("cox_concordance_index")) - b_rsf = _safe_float(b.get("rsf_c_index_test")) - c_rsf = _safe_float(c.get("rsf_c_index_test")) - cox_ok = np.isnan(b_cox) or np.isnan(c_cox) or (c_cox >= b_cox - 0.01) - rsf_ok = np.isnan(b_rsf) or np.isnan(c_rsf) or (c_rsf >= b_rsf - 0.01) - return GateResult( - "survival_quality", - bool(cox_ok and rsf_ok), - { - "baseline": {"cox_cindex": b_cox, "rsf_cindex": b_rsf}, - "current": {"cox_cindex": c_cox, "rsf_cindex": c_rsf}, - "checks": {"cox_ok": cox_ok, "rsf_ok": rsf_ok}, - }, - ) - - -def _gate_exports(cur: dict[str, Any]) -> GateResult: - metrics = cur.get("metrics", {}) - model_comparison = metrics.get("model_comparison", {}) - pipeline_summary = metrics.get("pipeline_summary", {}) - missing = [] - for key in ["schema_version", "generated_at_utc", "models", "final_test_metrics"]: - if key not in model_comparison: - missing.append(f"model_comparison.{key}") - for key in ["schema_version", "generated_at_utc", "flattened_summary"]: - if key not in pipeline_summary: - missing.append(f"pipeline_summary.{key}") - return GateResult( - "export_contracts", - len(missing) == 0, - {"missing_keys": missing}, - ) - - -def _compare_artifacts(base: dict[str, Any], cur: dict[str, Any]) -> dict[str, Any]: - out: dict[str, Any] = {} - b_idx = base.get("artifacts", {}) - c_idx = cur.get("artifacts", {}) - for key in sorted(set(b_idx) | set(c_idx)): - b = b_idx.get(key, {}) - c = c_idx.get(key, {}) - out[key] = { - "baseline_exists": bool(b.get("exists", False)), - "current_exists": bool(c.get("exists", False)), - "hash_changed": b.get("sha256") != c.get("sha256"), - "size_bytes_baseline": int(b.get("size_bytes", 0) or 0), - "size_bytes_current": int(c.get("size_bytes", 0) or 0), - } - return out - - -def _markdown_report(report: dict[str, Any]) -> str: - gates = report["gates"] - lines = [ - f"# Run Comparison: {report['run_tag']}", - "", - f"- Generated: {report['generated_at_utc']}", - f"- Overall gates pass: `{report['overall_pass']}`", - f"- Conformal promotion pass: `{report.get('conformal_promotion_pass', False)}`", - f"- Conformal retired backtest checks: `{len(report.get('conformal_retired_backtest_checks', []))}`", - f"- Artifact coherence pass: `{report.get('artifact_coherence_pass', False)}`", - f"- Semantic coherence pass: `{report.get('semantic_coherence_pass', False)}`", - f"- Fairness absolute (business) pass: `{report.get('fairness_absolute_business_pass', False)}`", - f"- A/B gate mode: `{report.get('ab_gate_mode', 'no_regression')}`", - f"- A/B no-regression pass: `{report.get('ab_no_regression_pass', False)}`", - f"- A/B significance (diagnostic): `{report.get('ab_significant', False)}`", - "", - "## Gates", - ] - for gate in gates: - status = "PASS" if gate["passed"] else "FAIL" - lines.append(f"- `{gate['name']}`: **{status}**") - lines.extend(["", "## Artifact Changes"]) - changed = [ - (k, v) - for k, v in report["artifact_changes"].items() - if v.get("hash_changed") or (not v.get("baseline_exists")) != (not v.get("current_exists")) - ] - if not changed: - lines.append("- No tracked artifact hash changes.") - else: - for key, meta in changed: - lines.append( - f"- `{key}`: hash_changed={meta['hash_changed']}, " - f"baseline_exists={meta['baseline_exists']}, current_exists={meta['current_exists']}" - ) - return "\n".join(lines) + "\n" - - -def _write_snapshot(run_tag: str) -> Path: - out_dir = OUT_ROOT / run_tag - out_dir.mkdir(parents=True, exist_ok=True) - path = out_dir / "baseline_snapshot.json" - path.write_text( - json.dumps(_snapshot_payload(run_tag), indent=2, ensure_ascii=False), encoding="utf-8" - ) - print(f"[compare] Baseline snapshot saved: {path.relative_to(ROOT)}") - return path - - -def _comparison_gate_results( - *, - baseline_metrics: dict[str, Any], - current_metrics: dict[str, Any], - current_snapshot: dict[str, Any], - run_tag: str, -) -> list[GateResult]: - return [ - _gate_artifact_coherence(current_metrics, run_tag), - _gate_semantic_coherence(current_metrics), - _gate_pd(baseline_metrics, current_metrics), - _gate_conformal(baseline_metrics, current_metrics), - _gate_ab_no_regression(baseline_metrics, current_metrics), - _gate_fairness(baseline_metrics, current_metrics), - _gate_fairness_absolute_business(baseline_metrics, current_metrics), - _gate_survival(baseline_metrics, current_metrics), - _gate_exports(current_snapshot), - ] - - -def _gate_lookup(gate_results: list[GateResult]) -> dict[str, GateResult]: - return {gate.name: gate for gate in gate_results} - - -def _gate_pass(gates: dict[str, GateResult], name: str) -> bool: - gate = gates.get(name) - return bool(gate.passed) if gate is not None else False - - -def _gate_details(gates: dict[str, GateResult], name: str) -> dict[str, Any]: - gate = gates.get(name) - return gate.details if gate is not None else {} - - -def _comparison_gate_fields(gate_results: list[GateResult]) -> dict[str, Any]: - gates = _gate_lookup(gate_results) - conformal_details = _gate_details(gates, "conformal_policy") - conformal_diagnostics = conformal_details.get("diagnostics", {}) - ab_diagnostics = _gate_details(gates, "ab_no_regression").get("diagnostics", {}) - return { - "overall_pass": bool(all(gate.passed for gate in gate_results)), - "operational_overall_pass": bool( - all(gate.passed for gate in gate_results if gate.name in _OPERATIONAL_GATE_NAMES) - ), - "artifact_coherence_pass": _gate_pass(gates, "artifact_coherence"), - "artifact_coherence": _gate_details(gates, "artifact_coherence"), - "semantic_coherence_pass": _gate_pass(gates, "semantic_coherence"), - "semantic_coherence": _gate_details(gates, "semantic_coherence"), - "conformal_promotion_pass": bool( - conformal_details.get("checks", {}).get("conformal_promotion_pass", False) - ), - "conformal_retired_backtest_checks": conformal_diagnostics.get( - "retired_backtest_checks", [] - ), - "ab_no_regression_pass": _gate_pass(gates, "ab_no_regression"), - "fairness_absolute_business_pass": _gate_pass(gates, "fairness_absolute_business"), - "ab_gate_mode": str(ab_diagnostics.get("gate_mode", "no_regression")), - "ab_significant": bool(ab_diagnostics.get("significant", False)), - "ab_significance_role": str(ab_diagnostics.get("significance_role", "diagnostic")), - } - - -def _comparison_quality_contract() -> dict[str, Any]: - return { - "conformal_checks_required": 13, - "ab_gate_mode": "no_regression", - "ab_significance_role": "diagnostic", - "fairness_gates": ["fairness_relative", "fairness_absolute_business"], - "fairness_policy_path": _path_for_report(FAIRNESS_POLICY_PATH), - "artifact_coherence_required": True, - "semantic_coherence_required": True, - "required_status_metadata": ["schema_version", "generated_at_utc", "run_tag"], - } - - -def _comparison_report( - *, - run_tag: str, - baseline_path: Path, - baseline: dict[str, Any], - current: dict[str, Any], - gate_results: list[GateResult], -) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "run_tag": run_tag, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "baseline_path": _path_for_report(baseline_path), - **_comparison_gate_fields(gate_results), - "gates": [{"name": g.name, "passed": g.passed, "details": g.details} for g in gate_results], - "artifact_changes": _compare_artifacts(baseline, current), - "baseline_head": baseline.get("git", {}).get("head", ""), - "current_head": current.get("git", {}).get("head", ""), - "quality_contract": _comparison_quality_contract(), - } - - -def _write_comparison_files(run_tag: str, report: dict[str, Any]) -> tuple[Path, Path]: - out_dir = OUT_ROOT / run_tag - out_dir.mkdir(parents=True, exist_ok=True) - json_path = out_dir / "comparison.json" - md_path = out_dir / "comparison.md" - json_path.write_text( - json.dumps(_to_builtin(report), indent=2, ensure_ascii=False), - encoding="utf-8", - ) - md_path.write_text(_markdown_report(report), encoding="utf-8") - print(f"[compare] Comparison JSON: {json_path.relative_to(ROOT)}") - print(f"[compare] Comparison MD: {md_path.relative_to(ROOT)}") - return json_path, md_path - - -def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: - baseline_path = baseline_path.expanduser().resolve() - baseline = json.loads(baseline_path.read_text(encoding="utf-8")) - current = _snapshot_payload(run_tag) - gate_results = _comparison_gate_results( - baseline_metrics=baseline["metrics"], - current_metrics=current["metrics"], - current_snapshot=current, - run_tag=run_tag, - ) - report = _comparison_report( - run_tag=run_tag, - baseline_path=baseline_path, - baseline=baseline, - current=current, - gate_results=gate_results, - ) - return _write_comparison_files(run_tag, report) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Snapshot/compare run artifacts with gates.") - sub = parser.add_subparsers(dest="cmd", required=True) - - p_snapshot = sub.add_parser("snapshot") - p_snapshot.add_argument("--run-tag", required=True) - - p_compare = sub.add_parser("compare") - p_compare.add_argument("--run-tag", required=True) - p_compare.add_argument("--baseline", default=None, help="Path to baseline_snapshot.json") - - args = parser.parse_args() - - if args.cmd == "snapshot": - _write_snapshot(args.run_tag) - return - - baseline_path = ( - Path(args.baseline).expanduser().resolve() - if args.baseline - else (OUT_ROOT / args.run_tag / "baseline_snapshot.json") - ) - if not baseline_path.exists(): - raise FileNotFoundError(f"Baseline snapshot not found: {baseline_path}") - _write_compare(args.run_tag, baseline_path) - - -if __name__ == "__main__": - main() diff --git a/scripts/run_complexity_report.py b/scripts/run_complexity_report.py new file mode 100644 index 0000000..22b9c03 --- /dev/null +++ b/scripts/run_complexity_report.py @@ -0,0 +1,53 @@ +"""Report cyclomatic complexity for the declared active code surface.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from collections.abc import Mapping, Sequence + +from scripts.run_ty_advisory import iter_python_files + +RADON_REQUIREMENT = "radon==6.0.1" +_PYTHON_ENV_KEYS = ("PYTHONHOME", "VIRTUAL_ENV") + + +def build_radon_command(*, uvx: str, files: Sequence[str]) -> list[str]: + """Build the pinned Radon command for active Python files.""" + return [ + uvx, + "--from", + RADON_REQUIREMENT, + "radon", + "cc", + *files, + "--show-complexity", + "--min", + "C", + ] + + +def isolated_uvx_environment(environ: Mapping[str, str]) -> dict[str, str]: + """Remove interpreter bindings that can corrupt a nested uvx runtime.""" + cleaned = dict(environ) + for key in _PYTHON_ENV_KEYS: + cleaned.pop(key, None) + return cleaned + + +def main() -> int: + uvx = shutil.which("uvx") + if uvx is None: + raise RuntimeError("uvx is required to run the complexity report.") + files = iter_python_files(scope="active") + result = subprocess.run( + build_radon_command(uvx=uvx, files=files), + check=False, + env=isolated_uvx_environment(os.environ), + ) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_cqr_comparison.py b/scripts/run_cqr_comparison.py deleted file mode 100644 index 409d1e1..0000000 --- a/scripts/run_cqr_comparison.py +++ /dev/null @@ -1,373 +0,0 @@ -"""CQR (Conformalized Quantile Regression) comparison for CRPTO. - -Compares three conformal uncertainty methods for PD intervals: -1. Symmetric split-conformal (global, existing) -2. Mondrian split-conformal (grade-conditional, existing) -3. CQR — asymmetric intervals from quantile regression residuals - -CQR uses MAPIE ConformalizedQuantileRegressor with CatBoost quantile models, -producing asymmetric [pd_low, pd_high] intervals that adapt to the local -uncertainty distribution, unlike symmetric split-conformal. - -Usage: - uv run python scripts/run_cqr_comparison.py -""" - -from __future__ import annotations - -import json -import pickle -from pathlib import Path -from typing import Any, cast - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from loguru import logger - -from src.models.conformal import ( - create_pd_intervals, - create_pd_intervals_mondrian, -) -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag - -SCHEMA_VERSION = "2026-03-16.1" - -ALPHA = 0.10 # Target: 90% coverage -LOW_RISK_THRESHOLD = 0.10 -N_CAL_SPLIT = 0.5 # CQR needs its own cal split (from existing cal set) - - -def _load_model_and_data() -> tuple: - """Load CatBoost model, contract, and processed datasets.""" - model_path = Path("models/pd_canonical.cbm") - contract_path = Path("models/pd_model_contract.json") - cal_path = Path("data/processed/calibration_fe.parquet") - test_path = Path("data/processed/test_fe.parquet") - - for p in [model_path, contract_path, cal_path, test_path]: - if not p.exists(): - raise FileNotFoundError(f"Required file not found: {p}") - - from catboost import CatBoostClassifier - - model = CatBoostClassifier() - model.load_model(str(model_path)) - logger.info("Loaded CatBoost classifier from {}", model_path) - - with open(contract_path, encoding="utf-8") as contract_handle: - contract = cast(dict[str, Any], json.load(contract_handle)) - feature_cols: list[str] = contract["feature_names"] - - # Load calibrator (optional) - cal_pkl = Path("models/pd_canonical_calibrator.pkl") - calibrator = None - if cal_pkl.exists(): - with open(cal_pkl, "rb") as calibrator_handle: - calibrator = pickle.load(calibrator_handle) - logger.info("Loaded calibrator from {}", cal_pkl) - - # Load data - cal = pd.read_parquet(cal_path) - test = pd.read_parquet(test_path) - logger.info("Loaded cal ({:,}) and test ({:,}) rows", len(cal), len(test)) - - # Type conversion using model's cat feature indices - cat_idx = set(model.get_cat_feature_indices()) - cat_names = {feature_cols[i] for i in cat_idx if i < len(feature_cols)} - - def prep_X(df: pd.DataFrame) -> pd.DataFrame: - X = df[feature_cols].copy() - for col in feature_cols: - if col in cat_names: - X[col] = X[col].astype("string").fillna("UNKNOWN").astype(str) - else: - X[col] = pd.to_numeric(X[col], errors="coerce") - return X - - X_cal = prep_X(cal) - X_test = prep_X(test) - y_cal = cal["default_flag"].astype(float) - y_test = test["default_flag"].astype(float) - grade_cal = ( - cal["grade"].fillna("UNKNOWN").astype(str) - if "grade" in cal.columns - else pd.Series(["UNKNOWN"] * len(cal)) - ) - grade_test = ( - test["grade"].fillna("UNKNOWN").astype(str) - if "grade" in test.columns - else pd.Series(["UNKNOWN"] * len(test)) - ) - - return ( - model, - calibrator, - feature_cols, - cat_names, - X_cal, - y_cal, - X_test, - y_test, - grade_cal, - grade_test, - ) - - -def _run_cqr( - model_cls, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - y_test: pd.Series, - cat_names: set[str], - alpha: float = ALPHA, -) -> tuple[np.ndarray, dict]: - """Run CQR using CatBoost quantile regressors + MAPIE. - - Splits the calibration set in two halves: - - First half: fit quantile models (lower/upper quantile at alpha/2) - - Second half: conformalize residuals - - Returns: - y_intervals shape (n_test, 2), diagnostics dict - """ - from mapie.regression import ConformalizedQuantileRegressor - - n_cal = len(X_cal) - split = int(n_cal * N_CAL_SPLIT) - idx = np.arange(n_cal) - np.random.seed(42) - np.random.shuffle(idx) - train_idx, cal_idx = idx[:split], idx[split:] - - X_cqr_train = X_cal.iloc[train_idx] - y_cqr_train = y_cal.iloc[train_idx] - X_cqr_cal = X_cal.iloc[cal_idx] - y_cqr_cal = y_cal.iloc[cal_idx] - - logger.info( - "CQR split: train_quantile={:,}, conformalize={:,}", len(X_cqr_train), len(X_cqr_cal) - ) - - # MAPIE converts X to numpy before calling predict — CatBoost cat_features must be absent. - # Use numeric-only columns for CQR quantile regressors. - num_cols = [c for c in X_cal.columns if c not in cat_names] - logger.info( - "CQR using {:,} numeric features (dropping {:,} cat)", len(num_cols), len(cat_names) - ) - X_cqr_train_num = X_cqr_train[num_cols].fillna(0.0) - X_cqr_cal_num = X_cqr_cal[num_cols].fillna(0.0) - X_test_num = X_test[num_cols].fillna(0.0) - - # MAPIE CQR requires 3 quantile regressors: [alpha/2, 1-alpha/2, 0.5] - q_lo = alpha / 2.0 - q_hi = 1.0 - alpha / 2.0 - q_mid = 0.5 - - def make_qreg(quantile: float) -> CatBoostRegressor: - return CatBoostRegressor( - loss_function=f"Quantile:alpha={quantile}", - iterations=200, - learning_rate=0.05, - depth=6, - random_seed=42, - verbose=0, - ) - - qreg_lo = make_qreg(q_lo) - qreg_hi = make_qreg(q_hi) - qreg_mid = make_qreg(q_mid) - - logger.info("Fitting lower quantile regressor (q={:.3f})...", q_lo) - qreg_lo.fit(X_cqr_train_num, y_cqr_train) - logger.info("Fitting upper quantile regressor (q={:.3f})...", q_hi) - qreg_hi.fit(X_cqr_train_num, y_cqr_train) - logger.info("Fitting median quantile regressor (q={:.3f})...", q_mid) - qreg_mid.fit(X_cqr_train_num, y_cqr_train) - - # MAPIE CQR: pass list [alpha/2, 1-alpha/2, 0.5] in that order - cqr = ConformalizedQuantileRegressor( - estimator=cast(Any, [qreg_lo, qreg_hi, qreg_mid]), - confidence_level=1.0 - alpha, - prefit=True, - ) - cqr.conformalize(X_cqr_cal_num, y_cqr_cal) - - # Predict intervals on test set: returns (y_pred, y_intervals) where - # y_intervals has shape (n_test, 2, n_alphas) — take [:, :, 0] for single alpha - _, intervals_raw = cqr.predict_interval(X_test_num) - lo = np.clip(intervals_raw[:, 0, 0], 0.0, 1.0) - hi = np.clip(intervals_raw[:, 1, 0], 0.0, 1.0) - y_intervals = np.column_stack([lo, hi]) - - # Diagnostics - covered = (y_test.to_numpy() >= y_intervals[:, 0]) & (y_test.to_numpy() <= y_intervals[:, 1]) - widths = y_intervals[:, 1] - y_intervals[:, 0] - - diag = { - "empirical_coverage": float(covered.mean()), - "avg_width": float(widths.mean()), - "median_width": float(np.median(widths)), - "min_width": float(widths.min()), - "max_width": float(widths.max()), - "std_width": float(widths.std()), - "n_eligible": int((y_intervals[:, 1] < LOW_RISK_THRESHOLD).sum()), - "n_test": len(y_test), - } - logger.info( - "CQR: coverage={:.4f}, avg_width={:.4f}, eligible={:,}", - diag["empirical_coverage"], - diag["avg_width"], - diag["n_eligible"], - ) - return y_intervals, diag - - -def _coverage_diagnostics( - y_test: np.ndarray, - y_intervals: np.ndarray, - grade_test: np.ndarray, - method: str, -) -> dict: - """Compute coverage and per-group metrics.""" - covered = (y_test >= y_intervals[:, 0]) & (y_test <= y_intervals[:, 1]) - widths = y_intervals[:, 1] - y_intervals[:, 0] - - group_coverages: dict[str, float] = {} - for g in sorted(set(grade_test)): - mask = grade_test == g - if mask.sum() > 0: - group_coverages[g] = float(covered[mask].mean()) - - return { - "method": method, - "empirical_coverage": float(covered.mean()), - "min_group_coverage": float(min(group_coverages.values())) - if group_coverages - else float("nan"), - "avg_width": float(widths.mean()), - "median_width": float(np.median(widths)), - "std_width": float(widths.std()), - "n_eligible": int((y_intervals[:, 1] < LOW_RISK_THRESHOLD).sum()), - "n_test": len(y_test), - "eligible_pct": float((y_intervals[:, 1] < LOW_RISK_THRESHOLD).mean()), - "group_coverages": group_coverages, - } - - -def main() -> int: - """Run CQR vs symmetric conformal comparison.""" - run_tag = resolve_run_tag(None, allow_untracked=True) - logger.info("CQR comparison starting | run_tag={}", run_tag) - - ( - model, - calibrator, - feature_cols, - cat_names, - X_cal, - y_cal, - X_test, - y_test, - grade_cal, - grade_test, - ) = _load_model_and_data() - - y_test_arr = np.asarray(y_test, dtype=float) - grade_test_arr = np.asarray(grade_test, dtype=str) - - results: list[dict] = [] - - # ── Method 1: Global split-conformal ────────────────────────────────────── - logger.info("=== Method 1: Global split-conformal ===") - y_pred_global, y_int_global = create_pd_intervals(model, X_cal, y_cal, X_test, alpha=ALPHA) - results.append( - _coverage_diagnostics(y_test_arr, y_int_global, grade_test_arr, "global_splitconf") - ) - - # ── Method 2: Mondrian split-conformal ──────────────────────────────────── - logger.info("=== Method 2: Mondrian split-conformal ===") - y_pred_mond, y_int_mond, diag_mond = create_pd_intervals_mondrian( - model, X_cal, y_cal, X_test, grade_cal, grade_test, alpha=ALPHA - ) - results.append( - _coverage_diagnostics(y_test_arr, y_int_mond, grade_test_arr, "mondrian_splitconf") - ) - - # ── Method 3: CQR ───────────────────────────────────────────────────────── - logger.info("=== Method 3: CQR (asymmetric quantile) ===") - try: - y_int_cqr, cqr_diag = _run_cqr(model, X_cal, y_cal, X_test, y_test, cat_names, ALPHA) - results.append( - _coverage_diagnostics(y_test_arr, y_int_cqr, grade_test_arr, "cqr_asymmetric") - ) - cqr_status = "success" - except Exception as exc: - logger.error("CQR failed: {}", exc) - y_int_cqr = None - cqr_status = f"failed: {exc}" - - # ── Build comparison table ───────────────────────────────────────────────── - comparison_df = pd.DataFrame( - [{k: v for k, v in r.items() if k != "group_coverages"} for r in results] - ) - comparison_path = Path("data/processed/cqr_comparison.parquet") - comparison_df.to_parquet(comparison_path, index=False) - logger.info("Saved CQR comparison: {}", comparison_path) - - # Save intervals for CQR - if y_int_cqr is not None: - intervals_df = pd.DataFrame( - { - "pd_low_cqr_90": y_int_cqr[:, 0], - "pd_high_cqr_90": y_int_cqr[:, 1], - "pd_width_cqr_90": y_int_cqr[:, 1] - y_int_cqr[:, 0], - } - ) - cqr_intervals_path = Path("data/processed/conformal_intervals_cqr.parquet") - intervals_df.to_parquet(cqr_intervals_path, index=False) - logger.info("Saved CQR intervals: {}", cqr_intervals_path) - - # Build status JSON - metadata = build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=run_tag, - allow_untracked=True, - extra={"alpha": ALPHA, "low_risk_threshold": LOW_RISK_THRESHOLD}, - ) - status = { - **metadata, - "alpha": ALPHA, - "n_test": len(y_test), - "cqr_status": cqr_status, - "methods": { - r["method"]: {k: v for k, v in r.items() if k not in ("group_coverages",)} - for r in results - }, - "per_group_coverage": {r["method"]: r.get("group_coverages", {}) for r in results}, - "summary": { - "best_coverage_method": max(results, key=lambda x: x["empirical_coverage"])["method"], - "tightest_method": min(results, key=lambda x: x["avg_width"])["method"], - "most_eligible_method": max(results, key=lambda x: x["n_eligible"])["method"], - }, - } - status_path = Path("models/cqr_comparison_status.json") - with open(status_path, "w") as f: - json.dump(status, f, indent=2, default=str) - logger.info("Saved status: {}", status_path) - - # Summary table - logger.info("=== CQR Comparison Summary ===") - logger.info( - "\n" - + comparison_df[ - ["method", "empirical_coverage", "min_group_coverage", "avg_width", "n_eligible"] - ].to_string(index=False) - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_cqr_mondrian.py b/scripts/run_cqr_mondrian.py deleted file mode 100644 index 16f5fac..0000000 --- a/scripts/run_cqr_mondrian.py +++ /dev/null @@ -1,436 +0,0 @@ -"""CQR Mondrian Hybrid — grade-conditional asymmetric conformal intervals. - -Combines the best of two worlds: -- CQR (Conformalized Quantile Regression): asymmetric intervals adapting - to local uncertainty (not symmetric around prediction) -- Mondrian (group-conditional): separate calibration per grade group, - guaranteeing conditional coverage P(Y ∈ C(X) | grade=g) ≥ 1-alpha - -Standard CQR: fit global quantile regressors, conformalize globally. -CQR Mondrian: fit global quantile regressors, conformalize PER GRADE. - -This produces grade-specific conformalization offsets for the asymmetric -quantile bounds — tighter intervals for low-risk grades (A, B) and wider -for high-risk grades (E, F, G). - -Usage: - uv run python scripts/run_cqr_mondrian.py -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np -import pandas as pd -from catboost import CatBoostRegressor -from loguru import logger - -from src.models.conformal import ( - create_pd_intervals, - create_pd_intervals_mondrian, -) -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag - -SCHEMA_VERSION = "2026-03-16.1" -ALPHA = 0.10 -LOW_RISK_THRESHOLD = 0.10 -N_CAL_TRAIN_SPLIT = 0.5 # fraction of cal set used to train quantile regressors - - -def _load_data() -> tuple: - """Load calibration and test FE data with feature type handling.""" - model_path = Path("models/pd_canonical.cbm") - contract_path = Path("models/pd_model_contract.json") - cal_path = Path("data/processed/calibration_fe.parquet") - test_path = Path("data/processed/test_fe.parquet") - - for p in [model_path, contract_path, cal_path, test_path]: - if not p.exists(): - raise FileNotFoundError(f"Missing: {p}") - - from catboost import CatBoostClassifier - - clf = CatBoostClassifier() - clf.load_model(str(model_path)) - - with open(contract_path) as f: - contract = json.load(f) - feature_cols: list[str] = contract["feature_names"] - - cat_idx = set(clf.get_cat_feature_indices()) - cat_names = {feature_cols[i] for i in cat_idx if i < len(feature_cols)} - num_cols = [c for c in feature_cols if c not in cat_names] - - cal = pd.read_parquet(cal_path) - test = pd.read_parquet(test_path) - logger.info("Loaded cal ({:,}) test ({:,})", len(cal), len(test)) - - def prep_X(df: pd.DataFrame) -> pd.DataFrame: - X = df[feature_cols].copy() - for col in feature_cols: - if col in cat_names: - X[col] = X[col].astype("string").fillna("UNKNOWN").astype(str) - else: - X[col] = pd.to_numeric(X[col], errors="coerce") - return X - - X_cal = prep_X(cal) - X_test = prep_X(test) - y_cal = cal["default_flag"].astype(float) - y_test = test["default_flag"].astype(float) - grade_cal = cal["grade"].fillna("UNKNOWN").astype(str) - grade_test = test["grade"].fillna("UNKNOWN").astype(str) - - # Numeric-only features for quantile regressors (MAPIE converts to numpy) - X_cal_num = cal[num_cols].copy() - X_test_num = test[num_cols].copy() - for col in num_cols: - X_cal_num[col] = pd.to_numeric(X_cal_num[col], errors="coerce").fillna(0.0) - X_test_num[col] = pd.to_numeric(X_test_num[col], errors="coerce").fillna(0.0) - - return clf, X_cal, X_test, y_cal, y_test, grade_cal, grade_test, X_cal_num, X_test_num, num_cols - - -def _fit_quantile_regressors( - X_train_num: pd.DataFrame, - y_train: pd.Series, - alpha: float, - n_trees: int = 200, -) -> tuple: - """Fit lower and upper quantile regressors on numeric features.""" - q_lo, q_hi = alpha / 2.0, 1.0 - alpha / 2.0 - - def make_qreg(q: float) -> CatBoostRegressor: - return CatBoostRegressor( - loss_function=f"Quantile:alpha={q}", - iterations=n_trees, - learning_rate=0.05, - depth=6, - random_seed=42, - verbose=0, - ) - - logger.info("Fitting quantile regressor q={:.3f} on {:,} samples...", q_lo, len(X_train_num)) - qreg_lo = make_qreg(q_lo) - qreg_lo.fit(X_train_num, y_train) - - logger.info("Fitting quantile regressor q={:.3f}...", q_hi) - qreg_hi = make_qreg(q_hi) - qreg_hi.fit(X_train_num, y_train) - - return qreg_lo, qreg_hi - - -def _cqr_global( - qreg_lo: CatBoostRegressor, - qreg_hi: CatBoostRegressor, - X_cal_num: pd.DataFrame, - y_cal: pd.Series, - X_test_num: pd.DataFrame, - alpha: float, -) -> np.ndarray: - """Standard CQR: global calibration of residuals.""" - q_lo_pred = qreg_lo.predict(X_cal_num) - q_hi_pred = qreg_hi.predict(X_cal_num) - y_cal_arr = np.asarray(y_cal, dtype=float) - - # CQR nonconformity scores - scores = np.maximum(q_lo_pred - y_cal_arr, y_cal_arr - q_hi_pred) - n = len(scores) - q_level = np.ceil((n + 1) * (1 - alpha)) / n - q_level = min(q_level, 1.0) - offset = float(np.quantile(scores, q_level, method="higher")) - - lo_pred = qreg_lo.predict(X_test_num) - offset - hi_pred = qreg_hi.predict(X_test_num) + offset - return np.column_stack([np.clip(lo_pred, 0, 1), np.clip(hi_pred, 0, 1)]) - - -def _cqr_mondrian( - qreg_lo: CatBoostRegressor, - qreg_hi: CatBoostRegressor, - X_cal_num: pd.DataFrame, - y_cal: pd.Series, - X_test_num: pd.DataFrame, - grade_cal: pd.Series, - grade_test: pd.Series, - alpha: float, - min_group_size: int = 500, -) -> tuple[np.ndarray, dict]: - """CQR Mondrian: grade-conditional calibration of quantile residuals. - - Fits global quantile regressors but computes conformalization offset - separately per grade group (Mondrian partition). - - Nonconformity score: max(q_lo(x) - y, y - q_hi(x)) - Per-grade offset: (n_g+1)(1-alpha)/n_g quantile of group scores. - """ - q_lo_pred_cal = qreg_lo.predict(X_cal_num) - q_hi_pred_cal = qreg_hi.predict(X_cal_num) - y_cal_arr = np.asarray(y_cal, dtype=float) - g_cal = np.asarray(grade_cal, dtype=str) - g_test = np.asarray(grade_test, dtype=str) - - scores = np.maximum(q_lo_pred_cal - y_cal_arr, y_cal_arr - q_hi_pred_cal) - - # Global fallback quantile - n_global = len(scores) - q_lev_global = np.ceil((n_global + 1) * (1 - alpha)) / n_global - global_offset = float(np.quantile(scores, min(q_lev_global, 1.0), method="higher")) - - # Per-grade offsets - all_grades = sorted(set(g_cal).union(set(g_test))) - group_offsets: dict[str, float] = {} - group_counts: dict[str, int] = {} - fallback_groups: list[str] = [] - - for g in all_grades: - mask = g_cal == g - n_g = int(mask.sum()) - group_counts[g] = n_g - if n_g >= min_group_size: - q_lev = np.ceil((n_g + 1) * (1 - alpha)) / n_g - group_offsets[g] = float(np.quantile(scores[mask], min(q_lev, 1.0), method="higher")) - else: - group_offsets[g] = global_offset - fallback_groups.append(g) - - logger.info( - "CQR Mondrian offsets: {} groups, {} fallbacks | offsets: {}", - len(all_grades), - len(fallback_groups), - {g: f"{v:.4f}" for g, v in sorted(group_offsets.items())}, - ) - - # Apply per-group offsets to test predictions - q_lo_test = qreg_lo.predict(X_test_num) - q_hi_test = qreg_hi.predict(X_test_num) - offsets_test = np.array([group_offsets.get(str(g), global_offset) for g in g_test]) - - lo_final = np.clip(q_lo_test - offsets_test, 0.0, 1.0) - hi_final = np.clip(q_hi_test + offsets_test, 0.0, 1.0) - intervals = np.column_stack([lo_final, hi_final]) - - diag = { - "global_offset": global_offset, - "group_offsets": group_offsets, - "group_counts": group_counts, - "fallback_groups": fallback_groups, - "avg_width": float((hi_final - lo_final).mean()), - } - return intervals, diag - - -def _coverage_report( - y_test: np.ndarray, - intervals: np.ndarray, - grade_test: np.ndarray, - method: str, - alpha: float, -) -> dict: - covered = (y_test >= intervals[:, 0]) & (y_test <= intervals[:, 1]) - widths = intervals[:, 1] - intervals[:, 0] - group_cov: dict[str, float] = {} - group_width: dict[str, float] = {} - for g in sorted(set(grade_test)): - mask = grade_test == g - if mask.sum() > 0: - group_cov[g] = float(covered[mask].mean()) - group_width[g] = float(widths[mask].mean()) - return { - "method": method, - "alpha": alpha, - "target_coverage": 1 - alpha, - "empirical_coverage": float(covered.mean()), - "coverage_gap": float((1 - alpha) - covered.mean()), - "min_group_coverage": float(min(group_cov.values())) if group_cov else float("nan"), - "avg_width": float(widths.mean()), - "median_width": float(np.median(widths)), - "std_width": float(widths.std()), - "n_eligible": int((intervals[:, 1] < LOW_RISK_THRESHOLD).sum()), - "eligible_pct": float((intervals[:, 1] < LOW_RISK_THRESHOLD).mean()), - "group_coverage": group_cov, - "group_avg_width": group_width, - } - - -def main() -> int: - run_tag = resolve_run_tag(None, allow_untracked=True) - logger.info("CQR Mondrian hybrid starting | run_tag={}", run_tag) - - (clf, X_cal, X_test, y_cal, y_test, grade_cal, grade_test, X_cal_num, X_test_num, num_cols) = ( - _load_data() - ) - - y_test_arr = np.asarray(y_test, dtype=float) - g_test_arr = np.asarray(grade_test, dtype=str) - - # Split calibration set: first half trains quantile regressors, second half for conformalization - n_cal = len(X_cal) - split = int(n_cal * N_CAL_TRAIN_SPLIT) - idx = np.arange(n_cal) - np.random.seed(42) - np.random.shuffle(idx) - train_idx, conf_idx = idx[:split], idx[split:] - - X_qr_train = X_cal_num.iloc[train_idx] - y_qr_train = y_cal.iloc[train_idx] - X_qr_conf = X_cal_num.iloc[conf_idx] - y_qr_conf = y_cal.iloc[conf_idx] - grade_conf = grade_cal.iloc[conf_idx] - - logger.info( - "QR train: {:,} | Conformalize: {:,} | Test: {:,}", - len(X_qr_train), - len(X_qr_conf), - len(X_test), - ) - - # Fit quantile regressors once (shared by CQR global and CQR Mondrian) - qreg_lo, qreg_hi = _fit_quantile_regressors(X_qr_train, y_qr_train, ALPHA) - - results: list[dict] = [] - intervals_dict: dict[str, np.ndarray] = {} - - # ── Method 1: Symmetric global (reference) ─────────────────────────────── - logger.info("=== Method 1: Symmetric global split-conformal ===") - y_pred_g, y_int_g = create_pd_intervals(clf, X_cal, y_cal, X_test, alpha=ALPHA) - r = _coverage_report(y_test_arr, y_int_g, g_test_arr, "symmetric_global", ALPHA) - results.append({k: v for k, v in r.items() if k not in ("group_coverage", "group_avg_width")}) - intervals_dict["symmetric_global"] = y_int_g - logger.info( - " cov={:.4f}, width={:.4f}, min_grp={:.4f}", - r["empirical_coverage"], - r["avg_width"], - r["min_group_coverage"], - ) - - # ── Method 2: Symmetric Mondrian (reference) ───────────────────────────── - logger.info("=== Method 2: Symmetric Mondrian split-conformal ===") - y_pred_m, y_int_m, _ = create_pd_intervals_mondrian( - clf, X_cal, y_cal, X_test, grade_cal, grade_test, alpha=ALPHA - ) - r = _coverage_report(y_test_arr, y_int_m, g_test_arr, "symmetric_mondrian", ALPHA) - results.append({k: v for k, v in r.items() if k not in ("group_coverage", "group_avg_width")}) - intervals_dict["symmetric_mondrian"] = y_int_m - logger.info( - " cov={:.4f}, width={:.4f}, min_grp={:.4f}", - r["empirical_coverage"], - r["avg_width"], - r["min_group_coverage"], - ) - - # ── Method 3: CQR Global ────────────────────────────────────────────────── - logger.info("=== Method 3: CQR Global (asymmetric, global calibration) ===") - y_int_cqr_g = _cqr_global(qreg_lo, qreg_hi, X_qr_conf, y_qr_conf, X_test_num, ALPHA) - r = _coverage_report(y_test_arr, y_int_cqr_g, g_test_arr, "cqr_global", ALPHA) - results.append({k: v for k, v in r.items() if k not in ("group_coverage", "group_avg_width")}) - intervals_dict["cqr_global"] = y_int_cqr_g - logger.info( - " cov={:.4f}, width={:.4f}, min_grp={:.4f}", - r["empirical_coverage"], - r["avg_width"], - r["min_group_coverage"], - ) - - # ── Method 4: CQR Mondrian (asymmetric, grade-conditional) ─────────────── - logger.info("=== Method 4: CQR Mondrian (asymmetric + grade-conditional) ===") - y_int_cqr_m, diag_m = _cqr_mondrian( - qreg_lo, - qreg_hi, - X_qr_conf, - y_qr_conf, - X_test_num, - grade_conf, - grade_test, - ALPHA, - ) - r = _coverage_report(y_test_arr, y_int_cqr_m, g_test_arr, "cqr_mondrian", ALPHA) - results.append({k: v for k, v in r.items() if k not in ("group_coverage", "group_avg_width")}) - intervals_dict["cqr_mondrian"] = y_int_cqr_m - logger.info( - " cov={:.4f}, width={:.4f}, min_grp={:.4f}", - r["empirical_coverage"], - r["avg_width"], - r["min_group_coverage"], - ) - - # ── Save outputs ────────────────────────────────────────────────────────── - comparison_df = pd.DataFrame(results) - comp_path = Path("data/processed/cqr_mondrian_comparison.parquet") - comparison_df.to_parquet(comp_path, index=False) - logger.info("Saved comparison: {}", comp_path) - - # Save CQR Mondrian intervals - cqr_m_df = pd.DataFrame( - { - "pd_low_cqr_mondrian_90": y_int_cqr_m[:, 0], - "pd_high_cqr_mondrian_90": y_int_cqr_m[:, 1], - "pd_width_cqr_mondrian_90": y_int_cqr_m[:, 1] - y_int_cqr_m[:, 0], - "grade": grade_test.values, - } - ) - int_path = Path("data/processed/conformal_intervals_cqr_mondrian.parquet") - cqr_m_df.to_parquet(int_path, index=False) - logger.info("Saved CQR Mondrian intervals: {}", int_path) - - # Per-group coverage for all methods - group_cov_records: list[dict] = [] - for method_name, intervals in intervals_dict.items(): - covered = (y_test_arr >= intervals[:, 0]) & (y_test_arr <= intervals[:, 1]) - widths = intervals[:, 1] - intervals[:, 0] - for g in sorted(set(g_test_arr)): - mask = g_test_arr == g - if mask.sum() > 0: - group_cov_records.append( - { - "method": method_name, - "grade": g, - "n": int(mask.sum()), - "coverage": float(covered[mask].mean()), - "avg_width": float(widths[mask].mean()), - } - ) - group_df = pd.DataFrame(group_cov_records) - grp_path = Path("data/processed/cqr_mondrian_group_coverage.parquet") - group_df.to_parquet(grp_path, index=False) - logger.info("Saved per-group coverage: {}", grp_path) - - # Status JSON - metadata = build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=run_tag, - allow_untracked=True, - extra={"alpha": ALPHA, "n_numeric_features": len(num_cols)}, - ) - status: dict[str, object] = { - **metadata, - "alpha": ALPHA, - "n_test": len(y_test), - "cqr_mondrian_group_offsets": diag_m["group_offsets"], - "methods": {r["method"]: r for r in results}, - "summary": { - "best_coverage": max(results, key=lambda x: x["empirical_coverage"])["method"], - "tightest": min(results, key=lambda x: x["avg_width"])["method"], - "best_min_group_cov": max(results, key=lambda x: x["min_group_coverage"])["method"], - }, - } - status_path = Path("models/cqr_mondrian_status.json") - with open(status_path, "w") as f: - json.dump(status, f, indent=2, default=str) - logger.info("Saved status: {}", status_path) - - # Summary table - logger.info("=== CQR Mondrian Comparison ===") - display = comparison_df[ - ["method", "empirical_coverage", "min_group_coverage", "avg_width", "n_eligible"] - ] - logger.info("\n" + display.to_string(index=False)) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_crpto_pipeline.py b/scripts/run_crpto_pipeline.py deleted file mode 100644 index 5040697..0000000 --- a/scripts/run_crpto_pipeline.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -"""Run the standalone CRPTO pipeline in fixed, auditable steps.""" - -from __future__ import annotations - -import argparse -import json -import subprocess -from datetime import UTC, datetime -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -LOG_ROOT = ROOT / "reports" / "run_logs" - -STEPS = { - "preflight": [ - "python -m pytest -q tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py", - ], - "core_data_pd": [ - "python src/data/make_dataset.py --input data/raw/Loan_status_2007-2020Q3.csv --output data/processed/loan_master.parquet", - "python src/data/prepare_dataset.py", - "python scripts/materialize_feature_artifacts.py --config configs/crpto_pd_model.yaml", - "python scripts/train_pd_model.py --config configs/crpto_pd_model.yaml", - ], - "core_conformal": [ - "python scripts/generate_conformal_intervals.py --config configs/crpto_conformal_policy.yaml", - "python scripts/benchmark_conformal_variants.py", - "python scripts/backtest_conformal_coverage.py", - "python scripts/validate_conformal_policy.py --config configs/crpto_conformal_policy.yaml", - ], - "core_portfolio": [ - "python scripts/optimize_portfolio.py --config configs/crpto_optimization.yaml", - "python scripts/optimize_portfolio_tradeoff.py --config configs/crpto_optimization.yaml", - "python scripts/simulate_ab_test.py", - ], - "diagnostics_governance": [ - "python scripts/run_fairness_audit.py --config configs/crpto_fairness_policy.yaml", - "python scripts/generate_governance_status.py", - "python scripts/generate_mrm_report.py --config configs/crpto_mrm_policy.yaml", - ], - "publication_exports": [ - "python scripts/export_crpto_tables.py", - "python scripts/analyze_crpto_evidence.py", - "python scripts/build_crpto_journal_package.py", - "python scripts/generate_crpto_figures.py --paper crpto", - "python scripts/run_crpto_vs_spo_stability.py", - "QUARTO_PYTHON=.venv/bin/python quarto render book --to html", - ], -} -ORDER = list(STEPS) - - -def run_command(cmd: str, log_file: Path) -> None: - with log_file.open("a", encoding="utf-8") as handle: - handle.write(f"\n$ {cmd}\n") - handle.flush() - subprocess.run( - cmd, cwd=ROOT, shell=True, check=True, stdout=handle, stderr=subprocess.STDOUT - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Run CRPTO standalone pipeline.") - parser.add_argument( - "--run-tag", default=f"crpto-e2e-{datetime.now(UTC).strftime('%Y%m%d-%H%M%S')}" - ) - parser.add_argument("--from-step", choices=ORDER, default=ORDER[0]) - parser.add_argument("--until-step", choices=ORDER, default=ORDER[-1]) - parser.add_argument("--resume", action="store_true") - args = parser.parse_args() - - start = ORDER.index(args.from_step) - end = ORDER.index(args.until_step) - if start > end: - raise SystemExit("--from-step must come before --until-step") - - run_dir = LOG_ROOT / args.run_tag - run_dir.mkdir(parents=True, exist_ok=True) - status = {"run_tag": args.run_tag, "started_at_utc": datetime.now(UTC).isoformat(), "steps": {}} - - for step in ORDER[start : end + 1]: - marker = run_dir / f"{step}.done" - if args.resume and marker.exists(): - status["steps"][step] = "skipped_done" - continue - log_file = run_dir / f"{step}.log" - status["steps"][step] = "running" - (run_dir / "status.json").write_text(json.dumps(status, indent=2) + "\n", encoding="utf-8") - for cmd in STEPS[step]: - run_command(cmd, log_file) - marker.write_text(datetime.now(UTC).isoformat() + "\n", encoding="utf-8") - status["steps"][step] = "completed" - - status["completed_at_utc"] = datetime.now(UTC).isoformat() - (run_dir / "status.json").write_text(json.dumps(status, indent=2) + "\n", encoding="utf-8") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_fairness_audit.py b/scripts/run_fairness_audit.py deleted file mode 100644 index 03c9b2f..0000000 --- a/scripts/run_fairness_audit.py +++ /dev/null @@ -1,1046 +0,0 @@ -"""Run fairness audit across multiple protected attributes. - -Computes demographic parity, equalized odds, and disparate impact -for each attribute defined in the fairness policy config. -Supports policy `outcome_mode`: -- `default`: fairness over predicted default events. -- `approval`: fairness over favorable credit decision (approved loans). - -Usage: - uv run python scripts/run_fairness_audit.py - uv run python scripts/run_fairness_audit.py --config configs/fairness_policy.yaml -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from fairlearn.metrics import ( - MetricFrame, - demographic_parity_difference, - equalized_odds_difference, - selection_rate, -) -from loguru import logger -from sklearn.metrics import accuracy_score - -from src.evaluation.fairness import ( - build_intersectional_groups, - fairness_report_from_binary, - fairness_threshold_frontier, -) -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.threshold_semantics import write_threshold_semantics - -SCHEMA_VERSION = "2026-03-06.1" -SHAP_STATUS_PATH = Path("models/shap_fairness_status.json") -SHAP_SAMPLE_SIZE = 10_000 - - -def _as_float(value: Any) -> float: - """Convert scalar config/data values to float with a narrow error surface.""" - return float(value) - - -def _load_config(config_path: str) -> dict: - """Load fairness policy YAML config.""" - with open(config_path) as f: - return yaml.safe_load(f) - - -def _build_groups_dict( - df: pd.DataFrame, - attributes: list[dict], -) -> dict[str, np.ndarray]: - """Build groups dict from config attribute definitions.""" - groups_dict: dict[str, np.ndarray] = {} - - for attr in attributes: - name = attr["name"] - col = attr["column"] - - if col not in df.columns: - logger.warning(f"Column '{col}' not found in data, skipping attribute '{name}'") - continue - - if attr.get("binning") == "quartile": - groups_dict[name] = ( - pd.qcut(df[col], q=4, labels=["Q1", "Q2", "Q3", "Q4"], duplicates="drop") - .astype(str) - .values - ) - else: - groups_dict[name] = df[col].astype(str).values - - return groups_dict - - -def _resolve_prediction_threshold(cfg: dict, policy: dict) -> tuple[float, str]: - """Resolve prediction threshold from artifact when configured.""" - fallback_threshold = float(policy.get("prediction_threshold", 0.5)) - threshold_cfg = cfg.get("threshold_policy", {}) or {} - if not bool(threshold_cfg.get("use_artifact", True)): - return fallback_threshold, "policy_default" - - artifact_path = Path(threshold_cfg.get("artifact_path", "models/decision_threshold.json")) - if not artifact_path.exists(): - return fallback_threshold, "policy_default_missing_artifact" - - key = str(threshold_cfg.get("selected_threshold_key", "selected_threshold")) - try: - payload = json.loads(artifact_path.read_text(encoding="utf-8")) - resolved = float(payload.get(key, fallback_threshold)) - return resolved, "artifact" - except Exception: - return fallback_threshold, "policy_default_artifact_error" - - -def _resolve_outcome_mode(policy: dict) -> str: - raw = str(policy.get("outcome_mode", "default")).strip().lower() - if raw in {"approval", "approve", "good", "non_default"}: - return "approval" - return "default" - - -def _resolve_frontier_thresholds(primary_threshold: float, cfg: dict) -> list[float]: - frontier_cfg = cfg.get("threshold_frontier", {}) or {} - if not bool(frontier_cfg.get("enabled", True)): - return [float(primary_threshold)] - - explicit = frontier_cfg.get("thresholds", []) or [] - if explicit: - values = [float(x) for x in explicit] - else: - radius = float(frontier_cfg.get("window_radius", 0.10)) - step = float(frontier_cfg.get("step", 0.05)) - low = max(0.01, float(primary_threshold) - radius) - high = min(0.99, float(primary_threshold) + radius) - values = np.arange(low, high + step * 0.5, step).tolist() - values.append(float(primary_threshold)) - clipped = [min(max(float(x), 0.01), 0.99) for x in values] - return sorted({round(x, 4) for x in clipped}) - - -def _decision_policy_cfg(cfg: dict) -> dict: - return cfg.get("decision_policy", {}) or {} - - -def _select_threshold_from_frontier( - frontier: pd.DataFrame, - *, - y_pred_proba_eval: np.ndarray, -) -> dict[str, float | int]: - if frontier.empty: - return { - "selected_threshold": 0.5, - "n_passed": 0, - "worst_eo_gap": 1.0, - "approval_rate": 0.0, - } - - rows: list[dict[str, float | int]] = [] - for threshold, grp in frontier.groupby("threshold", observed=True): - threshold_value = _as_float(threshold) - rows.append( - { - "threshold": threshold_value, - "n_passed": int(grp["passed_all"].sum()), - "worst_eo_gap": float(grp["eo_gap"].max()), - "approval_rate": float( - (np.asarray(y_pred_proba_eval, dtype=float) >= threshold_value).mean() - ), - } - ) - - ranking = pd.DataFrame(rows).sort_values( - ["n_passed", "worst_eo_gap", "approval_rate"], - ascending=[False, True, False], - ) - selected = ranking.iloc[0].to_dict() - selected["selected_threshold"] = float(selected["threshold"]) - return selected - - -def _load_decision_policy_artifact(path: Path) -> dict: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - return payload if isinstance(payload, dict) else {} - except Exception: - return {} - - -def _apply_decision_policy( - *, - y_pred_proba: np.ndarray, - groups_all: dict[str, np.ndarray], - decision_policy: dict, - default_threshold: float, -) -> np.ndarray: - y_pred_proba = np.asarray(y_pred_proba, dtype=float) - thresholds = np.full(len(y_pred_proba), float(default_threshold), dtype=float) - overrides = decision_policy.get("overrides", []) if isinstance(decision_policy, dict) else [] - if not isinstance(overrides, list): - overrides = [] - - for override in overrides: - if not isinstance(override, dict): - continue - attribute = str(override.get("attribute", "")).strip() - group = str(override.get("group", "")).strip() - threshold = float(override.get("threshold", default_threshold)) - labels = groups_all.get(attribute) - if labels is None: - continue - mask = pd.Series(labels).astype(str).eq(group).to_numpy() - thresholds[mask] = threshold - - return (y_pred_proba >= thresholds).astype(float) - - -def _model_feature_names(model: Any) -> list[str]: - return list(getattr(model, "feature_names_", None) or []) - - -def _available_features(feature_names: list[str], data: pd.DataFrame) -> list[str]: - return [feature for feature in feature_names if feature in data.columns] - - -def _sample_shap_frame( - frame: pd.DataFrame, - *, - shap_sample_size: int, - random_state: int, -) -> tuple[pd.DataFrame, np.ndarray]: - rng = np.random.default_rng(random_state) - n_rows = len(frame) - if n_rows > shap_sample_size: - sample_idx = np.sort(rng.choice(n_rows, size=shap_sample_size, replace=False)) - else: - sample_idx = np.arange(n_rows) - return frame.iloc[sample_idx].reset_index(drop=True), sample_idx - - -def _cat_features_from_model(model: Any, frame: pd.DataFrame) -> list[str]: - try: - feature_names = _model_feature_names(model) - cat_idx = model.get_cat_feature_indices() - return [ - feature_names[idx] - for idx in cat_idx - if idx < len(feature_names) and feature_names[idx] in frame.columns - ] - except Exception: - return [] - - -def _text_categorical_columns(frame: pd.DataFrame, known_cats: set[str]) -> list[str]: - detected: list[str] = [] - for col in frame.columns: - if col in known_cats or pd.api.types.is_numeric_dtype(frame[col]): - continue - probe = frame[col].dropna().head(5) - if probe.empty: - continue - try: - pd.to_numeric(probe, errors="raise") - except (ValueError, TypeError): - detected.append(str(col)) - known_cats.add(str(col)) - return detected - - -def _catboost_cat_feature_names(model: Any, frame: pd.DataFrame) -> list[str]: - cat_names = _cat_features_from_model(model, frame) - cat_set = set(cat_names) - cat_names.extend(_text_categorical_columns(frame, cat_set)) - return cat_names - - -def _prepare_catboost_shap_frame(frame: pd.DataFrame, cat_feature_names: list[str]) -> pd.DataFrame: - out = frame.copy() - cat_set = set(cat_feature_names) - for col in list(out.columns): - if not out[col].isna().any(): - continue - if col in cat_set: - out[col] = out[col].astype(object).fillna("missing").astype(str) - elif pd.api.types.is_numeric_dtype(out[col]): - out[col] = out[col].fillna(0.0) - return out - - -def _catboost_shap_matrix( - model: Any, frame: pd.DataFrame, cat_feature_names: list[str] -) -> np.ndarray: - from catboost import Pool as CatPool - - pool = CatPool(frame, cat_features=cat_feature_names or None) - shap_raw = model.get_feature_importance(pool, type="ShapValues") - return np.abs(np.asarray(shap_raw[:, :-1], dtype=float)) - - -def _top_shap_features( - mean_abs_shap: np.ndarray, - feature_names: list[str], - *, - limit: int, -) -> list[dict[str, object]]: - top_idx = np.argsort(mean_abs_shap)[::-1][:limit] - return [ - {"feature": feature_names[idx], "mean_abs_shap": float(mean_abs_shap[idx])} - for idx in top_idx - ] - - -def _pairwise_shap_diffs( - group_shap: dict[str, np.ndarray], - feature_names: list[str], - *, - limit: int, -) -> list[dict[str, object]]: - pairwise_diffs: list[dict[str, object]] = [] - groups_with_shap = list(group_shap.keys()) - for i, group_a in enumerate(groups_with_shap): - for group_b in groups_with_shap[i + 1 :]: - diff = np.abs(group_shap[group_a] - group_shap[group_b]) - top_idx = np.argsort(diff)[::-1][:limit] - pairwise_diffs.append( - { - "group_a": group_a, - "group_b": group_b, - "top_driving_features": [ - { - "feature": feature_names[idx], - "shap_diff": float(diff[idx]), - } - for idx in top_idx - ], - } - ) - return pairwise_diffs - - -def _attribute_shap_result( - *, - attribute: str, - labels: np.ndarray, - sample_idx: np.ndarray, - shap_matrix: np.ndarray, - feature_names: list[str], - min_group_size: int = 10, -) -> dict[str, object]: - group_labels = pd.Series(labels).iloc[sample_idx].reset_index(drop=True).astype(str) - unique_groups = sorted(group_labels.unique()) - group_shap: dict[str, np.ndarray] = {} - group_top5: dict[str, list[dict[str, object]]] = {} - - for group in unique_groups: - mask = group_labels.eq(group).to_numpy() - if mask.sum() < min_group_size: - continue - mean_abs_shap = shap_matrix[mask].mean(axis=0) - group_shap[str(group)] = mean_abs_shap - group_top5[str(group)] = _top_shap_features(mean_abs_shap, feature_names, limit=5) - - groups_with_shap = list(group_shap.keys()) - return { - "attribute": attribute, - "groups_analyzed": groups_with_shap, - "top5_per_group": group_top5, - "pairwise_feature_diffs": _pairwise_shap_diffs( - group_shap, - feature_names, - limit=3, - ), - } - - -def _shap_attribute_results( - groups_dict: dict[str, np.ndarray], - *, - sample_idx: np.ndarray, - shap_matrix: np.ndarray, - feature_names: list[str], -) -> list[dict[str, object]]: - attribute_results: list[dict[str, object]] = [] - for attribute, labels in groups_dict.items(): - if "__x__" in attribute: - continue - result = _attribute_shap_result( - attribute=attribute, - labels=labels, - sample_idx=sample_idx, - shap_matrix=shap_matrix, - feature_names=feature_names, - ) - attribute_results.append(result) - groups_analyzed = result.get("groups_analyzed", []) - n_groups = len(groups_analyzed) if isinstance(groups_analyzed, list) else 0 - logger.info(f"SHAP per-group: {attribute} ({n_groups} groups)") - return attribute_results - - -def _shap_result_payload( - *, - model_path: Path, - sample_size: int, - n_features: int, - attribute_results: list[dict[str, object]], -) -> dict[str, object]: - return { - "schema_version": SCHEMA_VERSION, - "model_path": str(model_path), - "shap_sample_size": sample_size, - "n_features": n_features, - "attributes": attribute_results, - "interpretation": ( - "For each protected attribute, top-5 features by mean |SHAP| per group. " - "Pairwise diffs show which features drive SHAP disparities between groups. " - "Features like dti/loan_amnt are legitimate credit risk factors; " - "home_ownership may proxy for race in US ECOA context." - ), - } - - -def _compute_shap_per_group( - data: pd.DataFrame, - groups_dict: dict[str, np.ndarray], - model_path: str | Path = "models/pd_canonical.cbm", - shap_sample_size: int = SHAP_SAMPLE_SIZE, - random_state: int = 42, -) -> dict[str, object] | None: - """Compute per-group SHAP analysis to identify which features drive disparities. - - For each protected attribute, computes mean |SHAP| per group and the - top-5 features per group. Also computes pairwise group differences - |mean_SHAP_A - mean_SHAP_B| to identify the features responsible for - any observed fairness gaps. - - Args: - data: Test feature DataFrame (test_fe.parquet, n=276K rows). - groups_dict: Base attribute groups from fairness audit. - model_path: Path to the trained CatBoost model (.cbm). - shap_sample_size: Max rows to use for SHAP (performance cap). - random_state: Random seed for sampling. - - Returns: - Dict with per-attribute SHAP analysis, or None on failure. - """ - try: - from catboost import CatBoostClassifier - except ImportError as e: - logger.warning(f"SHAP per-group analysis skipped — missing dependency: {e}") - return None - - model_path = Path(model_path) - if not model_path.exists(): - logger.warning(f"SHAP per-group analysis skipped — model not found: {model_path}") - return None - - try: - model = CatBoostClassifier() - model.load_model(str(model_path)) - feature_names = _model_feature_names(model) - except Exception as e: - logger.warning(f"SHAP per-group analysis skipped — model load error: {e}") - return None - - available_features = _available_features(feature_names, data) - if not available_features: - logger.warning("SHAP per-group analysis skipped — no model features found in test data") - return None - - x_sample, sample_idx = _sample_shap_frame( - data[available_features].copy(), - shap_sample_size=shap_sample_size, - random_state=random_state, - ) - - logger.info( - f"Computing SHAP values on {len(x_sample):,} rows, {len(available_features)} features" - ) - try: - # Use CatBoost's native SHAP via Pool + get_feature_importance — avoids the shap - # library's cat/NaN handling issues entirely. model.get_cat_feature_indices() works - # on .cbm-loaded models without needing the sklearn feature_names_ attribute. - cat_feature_names = _catboost_cat_feature_names(model, x_sample) - x_sample = _prepare_catboost_shap_frame(x_sample, cat_feature_names) - shap_matrix = _catboost_shap_matrix(model, x_sample, cat_feature_names) - except Exception as e: - logger.warning(f"SHAP per-group analysis skipped — SHAP computation error: {e}") - return None - - attribute_results = _shap_attribute_results( - groups_dict, - sample_idx=sample_idx, - shap_matrix=shap_matrix, - feature_names=available_features, - ) - return _shap_result_payload( - model_path=model_path, - sample_size=len(x_sample), - n_features=len(available_features), - attribute_results=attribute_results, - ) - - -def _bootstrap_base_indices( - n_rows: int, - *, - bootstrap_max_rows: int, - rng: np.random.Generator, -) -> np.ndarray: - if bootstrap_max_rows > 0 and n_rows > bootstrap_max_rows: - return np.sort(rng.choice(n_rows, size=bootstrap_max_rows, replace=False)) - return np.arange(n_rows) - - -def _fairlearn_group_rows( - *, - attribute: str, - y_true: np.ndarray, - y_pred: np.ndarray, - sensitive: pd.Series, -) -> list[dict[str, object]]: - metric_frame = MetricFrame( - metrics={"selection_rate": selection_rate, "accuracy": accuracy_score}, - y_true=y_true, - y_pred=y_pred, - sensitive_features=sensitive, - ) - by_group = metric_frame.by_group.reset_index() - by_group.columns = ["group", *[str(col) for col in by_group.columns[1:]]] - rows = by_group.to_dict(orient="records") - for row in rows: - row["attribute"] = attribute - return rows - - -def _bootstrap_fairlearn_gaps( - *, - boot_sensitive_base: pd.Series, - boot_true_base: np.ndarray, - boot_pred_base: np.ndarray, - rng: np.random.Generator, - n_boot: int, -) -> tuple[list[float], list[float]]: - dpd_boot: list[float] = [] - eo_boot: list[float] = [] - for _ in range(max(n_boot, 0)): - idx = rng.integers(0, len(boot_true_base), len(boot_true_base)) - boot_sensitive = boot_sensitive_base.iloc[idx] - boot_true = boot_true_base[idx] - boot_pred = boot_pred_base[idx] - dpd_boot.append( - float( - demographic_parity_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - eo_boot.append( - float( - equalized_odds_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - return dpd_boot, eo_boot - - -def _fairlearn_summary_row( - *, - attribute: str, - y_true: np.ndarray, - y_pred: np.ndarray, - sensitive: pd.Series, - bootstrap_idx: np.ndarray, - rng: np.random.Generator, - n_boot: int, -) -> dict[str, object]: - dpd = float( - demographic_parity_difference( - y_true=y_true, - y_pred=y_pred, - sensitive_features=sensitive, - ) - ) - eo = float( - equalized_odds_difference( - y_true=y_true, - y_pred=y_pred, - sensitive_features=sensitive, - ) - ) - dpd_boot, eo_boot = _bootstrap_fairlearn_gaps( - boot_sensitive_base=sensitive.iloc[bootstrap_idx].reset_index(drop=True), - boot_true_base=y_true[bootstrap_idx], - boot_pred_base=y_pred[bootstrap_idx], - rng=rng, - n_boot=n_boot, - ) - return { - "attribute": attribute, - "demographic_parity_difference": dpd, - "equalized_odds_difference": eo, - "dpd_ci_low": float(np.quantile(dpd_boot, 0.025)) if dpd_boot else None, - "dpd_ci_high": float(np.quantile(dpd_boot, 0.975)) if dpd_boot else None, - "eo_ci_low": float(np.quantile(eo_boot, 0.025)) if eo_boot else None, - "eo_ci_high": float(np.quantile(eo_boot, 0.975)) if eo_boot else None, - } - - -def _fairlearn_sidecar_rows( - *, - groups_all: dict[str, np.ndarray], - y_true: np.ndarray, - y_pred: np.ndarray, - bootstrap_idx: np.ndarray, - rng: np.random.Generator, - n_boot: int, -) -> tuple[list[dict[str, object]], list[dict[str, object]]]: - group_rows: list[dict[str, object]] = [] - summary_rows: list[dict[str, object]] = [] - for attribute, labels in groups_all.items(): - sensitive = pd.Series(labels).astype(str).reset_index(drop=True) - group_rows.extend( - _fairlearn_group_rows( - attribute=attribute, - y_true=y_true, - y_pred=y_pred, - sensitive=sensitive, - ) - ) - summary_rows.append( - _fairlearn_summary_row( - attribute=attribute, - y_true=y_true, - y_pred=y_pred, - sensitive=sensitive, - bootstrap_idx=bootstrap_idx, - rng=rng, - n_boot=n_boot, - ) - ) - return group_rows, summary_rows - - -def _write_fairlearn_sidecar( - *, - sidecar_cfg: dict, - groups_all: dict[str, np.ndarray], - y_true_eval: np.ndarray, - y_pred_binary: np.ndarray, - status_path: Path, - primary_threshold: float, - outcome_mode: str, - resolved_run_tag: str, -) -> None: - rng = np.random.default_rng(int(sidecar_cfg.get("bootstrap_random_state", 42))) - n_boot = int(sidecar_cfg.get("bootstrap_samples", 200)) - bootstrap_max_rows = int(sidecar_cfg.get("bootstrap_max_rows", 50_000)) - y_true_arr = np.asarray(y_true_eval, dtype=float) - y_pred_arr = np.asarray(y_pred_binary, dtype=float) - bootstrap_idx = _bootstrap_base_indices( - len(y_true_arr), - bootstrap_max_rows=bootstrap_max_rows, - rng=rng, - ) - group_rows, summary_rows = _fairlearn_sidecar_rows( - groups_all=groups_all, - y_true=y_true_arr, - y_pred=y_pred_arr, - bootstrap_idx=bootstrap_idx, - rng=rng, - n_boot=n_boot, - ) - - sidecar_path = Path(sidecar_cfg.get("status_json", "models/fairlearn_fairness_status.json")) - group_metrics_path = Path( - sidecar_cfg.get("group_metrics_parquet", "data/processed/fairlearn_group_metrics.parquet") - ) - group_metrics_path.parent.mkdir(parents=True, exist_ok=True) - sidecar_path.parent.mkdir(parents=True, exist_ok=True) - pd.DataFrame(group_rows).to_parquet(group_metrics_path, index=False) - sidecar_payload = { - "primary_status_path": str(status_path), - "group_metrics_path": str(group_metrics_path), - "n_attributes": len(summary_rows), - "attributes": summary_rows, - "bootstrap_samples": n_boot, - "bootstrap_rows_used": len(bootstrap_idx), - "bootstrap_max_rows": bootstrap_max_rows, - "prediction_threshold": float(primary_threshold), - "outcome_mode": outcome_mode, - **build_artifact_metadata( - schema_version=f"{SCHEMA_VERSION}-fairlearn", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - sidecar_path.write_text(json.dumps(sidecar_payload, indent=2, default=str), encoding="utf-8") - logger.info(f"Saved fairlearn sidecar status: {sidecar_path}") - - -def _with_attribute_type(frame: pd.DataFrame) -> pd.DataFrame: - if frame.empty: - return frame - out = frame.copy() - out["attribute_type"] = np.where( - out["attribute"].astype(str).str.contains("__x__"), - "intersectional", - "base", - ) - return out - - -def _primary_frontier(frontier: pd.DataFrame, primary_threshold: float) -> pd.DataFrame: - if frontier.empty: - return pd.DataFrame() - return frontier.loc[np.isclose(frontier["threshold"].astype(float), primary_threshold)] - - -def _worst_primary_attribute(primary_frontier: pd.DataFrame) -> str: - if primary_frontier.empty: - return "" - return str( - primary_frontier.sort_values( - by=["passed_all", "eo_gap", "dpd", "dir"], - ascending=[True, False, False, True], - ).iloc[0]["attribute"] - ) - - -def _decision_override_count(decision_policy: Any) -> int: - overrides = decision_policy.get("overrides", []) if isinstance(decision_policy, dict) else [] - return len(overrides) if isinstance(overrides, list) else 0 - - -def _decision_global_threshold(decision_policy: Any, primary_threshold: float) -> float: - if isinstance(decision_policy, dict): - return _as_float(decision_policy.get("global_threshold", primary_threshold)) - return float(primary_threshold) - - -def _fairness_status_payload( - *, - report: pd.DataFrame, - frontier: pd.DataFrame, - frontier_path: Path, - frontier_thresholds: list[float], - primary_threshold: float, - threshold_source: str, - outcome_mode: str, - policy: dict, - decision_policy: Any, - decision_policy_path: Path, - config_path: str, - resolved_run_tag: str, -) -> dict[str, object]: - primary_frontier = _primary_frontier(frontier, primary_threshold) - return { - "overall_pass": bool(report["passed_all"].all()), - "n_attributes": len(report), - "n_base_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "base").sum() - ), - "n_intersectional_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "intersectional").sum() - ), - "n_passed": int(report["passed_all"].sum()), - "attributes": report.to_dict(orient="records"), - "prediction_threshold": float(primary_threshold), - "primary_threshold": float(primary_threshold), - "prediction_threshold_source": threshold_source, - "outcome_mode": outcome_mode, - "thresholds": { - "dpd": policy["dpd_threshold"], - "eo_gap": policy["eo_gap_threshold"], - "dir": policy["dir_threshold"], - }, - "threshold_frontier": { - "path": str(frontier_path), - "thresholds": frontier_thresholds, - "worst_primary_attribute": _worst_primary_attribute(primary_frontier), - "selected_threshold": float(primary_threshold), - "all_primary_pass": bool( - primary_frontier.get("passed_all", pd.Series(dtype=bool)).all() - ) - if not primary_frontier.empty - else True, - }, - "decision_policy": { - "path": str(decision_policy_path), - "global_threshold": float(primary_threshold), - "n_overrides": _decision_override_count(decision_policy), - }, - "policy_config": str(config_path), - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - - -def _write_json_payload(path: Path, payload: dict[str, object], *, label: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") - logger.info(f"Saved {label}: {path}") - - -def _write_shap_status( - *, - shap_result: dict[str, object] | None, - resolved_run_tag: str, - primary_threshold: float, - outcome_mode: str, - path: Path = SHAP_STATUS_PATH, -) -> None: - if shap_result is None: - return - shap_result["generated_at_utc"] = str( - __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() - ) - shap_result["run_tag"] = resolved_run_tag - shap_result["prediction_threshold"] = float(primary_threshold) - shap_result["outcome_mode"] = outcome_mode - _write_json_payload(path, shap_result, label="SHAP per-group fairness analysis") - - -def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None = None) -> None: - """Run the fairness audit pipeline.""" - cfg = _load_config(config_path) - policy = cfg["policy"] - artifacts = cfg["artifacts"] - output = cfg["output"] - - # Load test predictions and test data - pred_path = Path(artifacts["test_predictions_path"]) - data_path = Path(artifacts["test_data_path"]) - - if not pred_path.exists(): - raise FileNotFoundError(f"Missing test predictions: {pred_path}") - if not data_path.exists(): - raise FileNotFoundError(f"Missing test data: {data_path}") - - preds = pd.read_parquet(pred_path) - data = pd.read_parquet(data_path) - - # Extract y_true and y_pred_proba - y_true_col = "default_flag" - y_proba_col = "y_pred_proba" if "y_pred_proba" in preds.columns else "pd_calibrated" - - if y_true_col not in data.columns: - raise KeyError(f"Missing target column '{y_true_col}' in test data") - if y_proba_col not in preds.columns: - raise KeyError( - f"Missing probability column in predictions. Available: {list(preds.columns)}" - ) - - # Align lengths (both should be OOT test set) - n = min(len(preds), len(data)) - y_true = data[y_true_col].values[:n] - y_proba = preds[y_proba_col].values[:n] - - logger.info(f"Loaded {n} observations for fairness audit") - - # Build groups from attributes config - groups_dict = _build_groups_dict(data.iloc[:n], cfg["attributes"]) - - if not groups_dict: - logger.error("No valid attributes found for fairness audit") - return - - threshold, threshold_source = _resolve_prediction_threshold(cfg, policy) - outcome_mode = _resolve_outcome_mode(policy) - if outcome_mode == "approval": - # Fairness in credit decisions should be audited on favorable outcome (approval). - y_true_eval = 1.0 - y_true - y_proba_eval = 1.0 - y_proba - else: - y_true_eval = y_true - y_proba_eval = y_proba - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - - intersectional_cfg = cfg.get("intersectional", {}) or {} - intersectional_groups = ( - build_intersectional_groups( - groups_dict, - max_order=int(intersectional_cfg.get("max_order", 2)), - min_group_size=int(intersectional_cfg.get("min_group_size", 300)), - ) - if bool(intersectional_cfg.get("enabled", True)) - else {} - ) - groups_all = dict(groups_dict) - groups_all.update(intersectional_groups) - - frontier_thresholds = _resolve_frontier_thresholds(float(threshold), cfg) - frontier = fairness_threshold_frontier( - y_true=y_true_eval, - y_pred_proba=y_proba_eval, - groups_dict=groups_all, - thresholds=frontier_thresholds, - primary_threshold=float(threshold), - dpd_threshold=policy["dpd_threshold"], - eo_gap_threshold=policy["eo_gap_threshold"], - dir_threshold=policy["dir_threshold"], - ) - frontier = _with_attribute_type(frontier) - frontier_path = Path( - output.get("frontier_parquet", "data/processed/fairness_threshold_frontier.parquet") - ) - frontier_path.parent.mkdir(parents=True, exist_ok=True) - frontier.to_parquet(frontier_path, index=False) - logger.info(f"Saved fairness threshold frontier: {frontier_path}") - - decision_policy_cfg = _decision_policy_cfg(cfg) - decision_policy_path = Path( - decision_policy_cfg.get("artifact_path", "models/fairness_decision_policy.json") - ) - auto_select = bool(decision_policy_cfg.get("auto_select", False)) - decision_policy = _load_decision_policy_artifact(decision_policy_path) - selected_threshold_info = _select_threshold_from_frontier( - frontier, y_pred_proba_eval=y_proba_eval - ) - primary_threshold = float( - selected_threshold_info.get("selected_threshold", float(threshold)) - if auto_select - else float(threshold) - ) - - if auto_select: - decision_policy = { - "global_threshold": primary_threshold, - "overrides": decision_policy.get("overrides", []) - if isinstance(decision_policy, dict) - else [], - "selection": { - "source": "fairness_frontier_auto_select", - "n_passed": int(selected_threshold_info.get("n_passed", 0)), - "worst_eo_gap": float(selected_threshold_info.get("worst_eo_gap", 0.0)), - "approval_rate": float(selected_threshold_info.get("approval_rate", 0.0)), - }, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - decision_policy_path.parent.mkdir(parents=True, exist_ok=True) - decision_policy_path.write_text( - json.dumps(decision_policy, indent=2, default=str), - encoding="utf-8", - ) - threshold_source = "decision_policy_artifact_auto_selected" - elif decision_policy: - primary_threshold = float(decision_policy.get("global_threshold", threshold)) - threshold_source = "decision_policy_artifact" - - y_pred_binary = _apply_decision_policy( - y_pred_proba=y_proba_eval, - groups_all=groups_all, - decision_policy=decision_policy, - default_threshold=primary_threshold, - ) - report = fairness_report_from_binary( - y_true=y_true_eval, - y_pred_binary=y_pred_binary, - groups_dict=groups_all, - dpd_threshold=policy["dpd_threshold"], - eo_gap_threshold=policy["eo_gap_threshold"], - dir_threshold=policy["dir_threshold"], - ) - report = _with_attribute_type(report) - - audit_path = Path(output["audit_parquet"]) - audit_path.parent.mkdir(parents=True, exist_ok=True) - report.to_parquet(audit_path, index=False) - logger.info(f"Saved fairness audit: {audit_path}") - - status = _fairness_status_payload( - report=report, - frontier=frontier, - frontier_path=frontier_path, - frontier_thresholds=frontier_thresholds, - primary_threshold=float(primary_threshold), - threshold_source=threshold_source, - outcome_mode=outcome_mode, - policy=policy, - decision_policy=decision_policy, - decision_policy_path=decision_policy_path, - config_path=config_path, - resolved_run_tag=resolved_run_tag, - ) - - status_path = Path(output["status_json"]) - _write_json_payload(status_path, status, label="fairness status") - - sidecar_cfg = cfg.get("fairlearn_sidecar", {}) or {} - if bool(sidecar_cfg.get("enabled", True)): - _write_fairlearn_sidecar( - sidecar_cfg=sidecar_cfg, - groups_all=groups_all, - y_true_eval=y_true_eval, - y_pred_binary=y_pred_binary, - status_path=status_path, - primary_threshold=float(primary_threshold), - outcome_mode=outcome_mode, - resolved_run_tag=resolved_run_tag, - ) - - _write_shap_status( - shap_result=_compute_shap_per_group( - data=data.iloc[:n], - groups_dict=groups_dict, - ), - resolved_run_tag=resolved_run_tag, - primary_threshold=float(primary_threshold), - outcome_mode=outcome_mode, - ) - - write_threshold_semantics( - fairness_primary_threshold=float(primary_threshold), - decision_policy_global_threshold=_decision_global_threshold( - decision_policy, - float(primary_threshold), - ), - source_artifacts={ - "fairness_status": str(status_path), - "fairness_decision_policy": str(decision_policy_path), - "fairness_frontier": str(frontier_path), - }, - run_tag=resolved_run_tag, - extra={ - "fairness_threshold_source": threshold_source, - "outcome_mode": outcome_mode, - }, - path=output.get("threshold_semantics_json", "models/threshold_semantics.json"), - ) - - overall_pass = bool(status["overall_pass"]) - pass_label = "PASS" if overall_pass else "FAIL" - logger.info( - f"Fairness audit: {pass_label} ({status['n_passed']}/{status['n_attributes']} attributes)" - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run fairness audit") - parser.add_argument("--config", default="configs/fairness_policy.yaml") - parser.add_argument("--run-tag", default=None) - args = parser.parse_args() - main(config_path=args.config, run_tag=args.run_tag) diff --git a/scripts/run_gpu_replay.py b/scripts/run_gpu_replay.py deleted file mode 100644 index e7b3ce7..0000000 --- a/scripts/run_gpu_replay.py +++ /dev/null @@ -1,559 +0,0 @@ -"""Replay GPU-eligible pipeline stages against the current workspace artifacts. - -This script is meant to run immediately after a completed CPU baseline so the -workspace still contains the exact upstream artifacts produced by that run. -It reruns only the heavy stages where GPU backends are meaningful. -""" - -from __future__ import annotations - -import argparse -import csv -import json -import os -import shlex -import subprocess -import time -from contextlib import suppress -from datetime import UTC, datetime -from pathlib import Path -from threading import Event, Thread -from typing import Any - -STAGE_ORDER = [ - "pd", - "lgd_ead", - "portfolio", - "tradeoff", - "policy_selection", - "ab", - "cate_portfolio", - "ifrs9_mc", -] -RAPIDS_STAGES = {"portfolio", "tradeoff", "policy_selection", "ab", "cate_portfolio", "ifrs9_mc"} - -PROFILE_CONFIGS: dict[str, dict[str, int]] = { - "mega64": { - "portfolio_candidates": 100_000, - "tradeoff_candidates": 60_000, - "ab_candidates": 100_000, - "cate_candidates": 100_000, - }, - "mega64plus": { - "portfolio_candidates": 150_000, - "tradeoff_candidates": 80_000, - "ab_candidates": 150_000, - "cate_candidates": 150_000, - }, - "rapids_final": { - "portfolio_candidates": 0, - "tradeoff_candidates": 150_000, - "ab_candidates": 150_000, - "cate_candidates": 0, - "ifrs9_mc_scenarios": 8192, - "ifrs9_mc_chunk_size": 256, - }, -} - - -def _utc_now() -> str: - return datetime.now(tz=UTC).isoformat() - - -def _normalize_stages(raw: str) -> list[str]: - parts = [p.strip() for p in str(raw).split(",") if p.strip()] - if not parts or parts == ["all"]: - return list(STAGE_ORDER) - unknown = sorted(set(parts) - set(STAGE_ORDER)) - if unknown: - raise ValueError(f"Unknown GPU replay stages: {', '.join(unknown)}") - return [stage for stage in STAGE_ORDER if stage in parts] - - -def _resolve_rapids_python() -> Path: - proc = subprocess.run( - ["conda", "info", "--envs", "--json"], - capture_output=True, - text=True, - check=False, - ) - if proc.returncode != 0: - raise RuntimeError(f"Unable to resolve RAPIDS env path: {proc.stderr.strip()}") - payload = json.loads(proc.stdout or "{}") - envs = [Path(p) for p in payload.get("envs", [])] - for env_path in envs: - if env_path.name == "rapids": - python_path = env_path / "bin" / "python" - if python_path.exists(): - return python_path - raise RuntimeError("Could not find a usable python binary for conda env 'rapids'.") - - -def build_stage_commands( - *, - run_tag: str, - profile: str, - pd_config: str, - optimization_config: str, - rapids_python: str | None = None, -) -> dict[str, str]: - profile_cfg = PROFILE_CONFIGS[profile] - rapids_python = rapids_python or "python" - return { - "pd": ( - "uv run python -u scripts/train_pd_model.py " - f"--config {shlex.quote(pd_config)} --sample_size 0" - ), - "lgd_ead": ( - "uv run python -u scripts/train_lgd_ead.py --sample_size 0 " - f"--run-tag {shlex.quote(run_tag)} --catboost_backend gpu" - ), - "portfolio": ( - f"{shlex.quote(rapids_python)} -u -m scripts.optimize_portfolio " - f"--config {shlex.quote(optimization_config)} " - f"--max_candidates {profile_cfg['portfolio_candidates']} --solver_backend cuopt" - ), - "tradeoff": ( - f"{shlex.quote(rapids_python)} -u -m scripts.optimize_portfolio_tradeoff " - f"--config {shlex.quote(optimization_config)} " - f"--max_candidates {profile_cfg['tradeoff_candidates']} " - "--grid-profile night --solver_backend cuopt" - ), - "policy_selection": ( - f"{shlex.quote(rapids_python)} -u -m scripts.select_economic_portfolio_policy " - f"--config {shlex.quote(optimization_config)} " - f"--run-tag {shlex.quote(run_tag)} --solver_backend cuopt" - ), - "ab": ( - f"{shlex.quote(rapids_python)} -u -m scripts.simulate_ab_test --max_portfolio_pd 0.18 " - f"--max_candidates {profile_cfg['ab_candidates']} " - "--n_boot 5000 --seed 42 --no_regression_tolerance_pct 0.05 " - f"--run-tag {shlex.quote(run_tag)} --solver_backend cuopt " - "--policy_selector explicit_champion_only" - ), - "cate_portfolio": ( - f"{shlex.quote(rapids_python)} -u -m scripts.optimize_cate_portfolio " - f"--max_candidates {profile_cfg['cate_candidates']} --solver_backend cuopt" - ), - "ifrs9_mc": ( - f"{shlex.quote(rapids_python)} -u scripts/run_ifrs9_monte_carlo_gpu.py " - f"--n-scenarios {int(profile_cfg.get('ifrs9_mc_scenarios', 8192))} " - f"--chunk-size {int(profile_cfg.get('ifrs9_mc_chunk_size', 256))}" - ), - } - - -def validate_rapids_env( - *, - selected_stages: list[str], - rapids_python: Path | None = None, -) -> dict[str, Any]: - needs_rapids = any(stage in RAPIDS_STAGES for stage in selected_stages) - if not needs_rapids: - return {"checked": False, "needs_rapids": False} - if rapids_python is None: - rapids_python = _resolve_rapids_python() - - proc = subprocess.run( - [ - str(rapids_python), - "-c", - "import json, cuopt, cudf, cupy, pyomo, loguru; " - "print(json.dumps({'python': __import__('sys').executable, 'ok': True}))", - ], - capture_output=True, - text=True, - ) - stdout = (proc.stdout or "").strip() - stderr = (proc.stderr or "").strip() - if proc.returncode != 0: - raise RuntimeError( - "RAPIDS env validation failed for replay stages " - f"{sorted(RAPIDS_STAGES & set(selected_stages))}: stderr={stderr}" - ) - return { - "checked": True, - "needs_rapids": True, - "missing": [], - "python": str(rapids_python), - "stdout": stdout, - "stderr": stderr, - } - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") - - -class _GpuSampler: - def __init__(self, output_csv: Path, interval_seconds: float = 1.0) -> None: - self.output_csv = output_csv - self.interval_seconds = max(0.25, float(interval_seconds)) - self._stop = Event() - self._thread: Thread | None = None - self.samples: list[dict[str, float | str]] = [] - - def start(self) -> None: - self.output_csv.parent.mkdir(parents=True, exist_ok=True) - self._thread = Thread(target=self._loop, daemon=True) - self._thread.start() - - def stop(self) -> dict[str, Any]: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=5.0) - if not self.samples: - return {"available": False, "sample_count": 0, "csv_path": str(self.output_csv)} - - with self.output_csv.open("w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter( - f, - fieldnames=[ - "timestamp", - "gpu_util", - "memory_used_mb", - "memory_total_mb", - "power_draw_w", - ], - ) - writer.writeheader() - writer.writerows(self.samples) - - gpu_utils = [float(s["gpu_util"]) for s in self.samples] - memory_used = [float(s["memory_used_mb"]) for s in self.samples] - power = [float(s["power_draw_w"]) for s in self.samples] - return { - "available": True, - "sample_count": len(self.samples), - "csv_path": str(self.output_csv), - "peak_gpu_util": max(gpu_utils), - "avg_gpu_util": sum(gpu_utils) / len(gpu_utils), - "peak_memory_used_mb": max(memory_used), - "avg_memory_used_mb": sum(memory_used) / len(memory_used), - "peak_power_draw_w": max(power), - "avg_power_draw_w": sum(power) / len(power), - } - - def _loop(self) -> None: - while not self._stop.is_set(): - proc = subprocess.run( - [ - "nvidia-smi", - "--query-gpu=timestamp,utilization.gpu,memory.used,memory.total,power.draw", - "--format=csv,noheader,nounits", - ], - capture_output=True, - text=True, - check=False, - ) - if proc.returncode == 0: - lines = [line.strip() for line in (proc.stdout or "").splitlines() if line.strip()] - if lines: - first = [part.strip() for part in lines[0].split(",")] - if len(first) >= 5: - with suppress(ValueError): - self.samples.append( - { - "timestamp": first[0], - "gpu_util": float(first[1]), - "memory_used_mb": float(first[2]), - "memory_total_mb": float(first[3]), - "power_draw_w": float(first[4]), - } - ) - self._stop.wait(self.interval_seconds) - - -def _build_stage_env( - *, - base_env: dict[str, str], - run_tag: str, - baseline_run_tag: str, - artifact_root: Path, - stage: str, -) -> dict[str, str]: - env = dict(base_env) - env["PIPELINE_RUN_TAG"] = run_tag - env["GPU_REPLAY_BASELINE_RUN_TAG"] = baseline_run_tag - env["GPU_REPLAY_ARTIFACT_ROOT"] = str(artifact_root) - env["GPU_REPLAY_STAGE"] = stage - return env - - -def _build_rapids_env( - *, - base_env: dict[str, str], - run_tag: str, - baseline_run_tag: str, - artifact_root: Path, - stage: str, -) -> dict[str, str]: - env = _build_stage_env( - base_env=base_env, - run_tag=run_tag, - baseline_run_tag=baseline_run_tag, - artifact_root=artifact_root, - stage=stage, - ) - for key in [ - "VIRTUAL_ENV", - "PYTHONHOME", - "PYTHONPATH", - "UV_PROJECT_ENVIRONMENT", - "PYTHONNOUSERSITE", - ]: - env.pop(key, None) - return env - - -def build_post_replay_commands( - *, - notebook_timeout: int, - notebook_output_dir: str, - notebook_inplace: bool, - include_side_projects: bool, - extract_images_after: bool, -) -> list[tuple[str, str]]: - side_projects_flag = " --include-side-projects" if include_side_projects else "" - inplace_value = "true" if notebook_inplace else "false" - commands: list[tuple[str, str]] = [ - ( - "notebooks", - "uv run python -u scripts/run_all_notebooks.py " - f"--execute-all{side_projects_flag} --timeout {int(notebook_timeout)} " - f"--inplace {inplace_value} --output-dir {shlex.quote(notebook_output_dir)}", - ) - ] - if extract_images_after: - commands.append( - ( - "extract_images", - "uv run python -u scripts/extract_notebook_images.py " - f"--notebook-dir {shlex.quote(str(Path(notebook_output_dir) / 'notebooks'))}", - ) - ) - return commands - - -def main() -> int: - parser = argparse.ArgumentParser( - description="Replay GPU-eligible stages after a CPU baseline run." - ) - parser.add_argument("--baseline-run-tag", required=True) - parser.add_argument("--run-tag", required=True) - parser.add_argument("--profile", choices=sorted(PROFILE_CONFIGS), default="mega64plus") - parser.add_argument("--stages", default="all") - parser.add_argument("--pd-config", default="configs/pd_model.gpu.yaml") - parser.add_argument("--optimization-config", default="configs/optimization.yaml") - parser.add_argument("--run-notebooks-after", action="store_true") - parser.add_argument("--notebook-timeout", type=int, default=3600) - parser.add_argument("--notebook-output-dir", default="reports/notebook_exec") - parser.add_argument("--notebook-inplace", action="store_true", default=True) - parser.add_argument("--no-notebook-inplace", action="store_false", dest="notebook_inplace") - parser.add_argument("--include-side-projects", action="store_true") - parser.add_argument("--extract-images-after", action="store_true") - parser.add_argument("--dry-run", action="store_true") - args = parser.parse_args() - - selected_stages = _normalize_stages(args.stages) - rapids_python = _resolve_rapids_python() - commands = build_stage_commands( - run_tag=args.run_tag, - profile=args.profile, - pd_config=args.pd_config, - optimization_config=args.optimization_config, - rapids_python=str(rapids_python), - ) - rapids_validation = validate_rapids_env( - selected_stages=selected_stages, - rapids_python=rapids_python, - ) - - run_dir = Path("reports/gpu_replay") / args.run_tag - run_dir.mkdir(parents=True, exist_ok=True) - info_path = run_dir / "run_info.json" - summary_path = run_dir / "run_summary.json" - - payload = { - "schema_version": "2026-03-08.1", - "run_tag": args.run_tag, - "baseline_run_tag": args.baseline_run_tag, - "profile": args.profile, - "selected_stages": selected_stages, - "pd_config": args.pd_config, - "optimization_config": args.optimization_config, - "run_notebooks_after": bool(args.run_notebooks_after), - "notebook_timeout": int(args.notebook_timeout), - "notebook_output_dir": args.notebook_output_dir, - "notebook_inplace": bool(args.notebook_inplace), - "include_side_projects": bool(args.include_side_projects), - "extract_images_after": bool(args.extract_images_after), - "started_at_utc": _utc_now(), - "state": "planned" if args.dry_run else "running", - "rapids_env_validation": rapids_validation, - "note": ( - "This replay uses the current workspace artifacts. Run it immediately after the CPU " - "baseline you want to compare against." - ), - } - _write_json(info_path, payload) - - if args.dry_run: - _write_json( - summary_path, - { - **payload, - "state": "dry_run", - "commands": {stage: commands[stage] for stage in selected_stages}, - "post_replay_commands": build_post_replay_commands( - notebook_timeout=args.notebook_timeout, - notebook_output_dir=args.notebook_output_dir, - notebook_inplace=bool(args.notebook_inplace), - include_side_projects=bool(args.include_side_projects), - extract_images_after=bool(args.extract_images_after), - ) - if args.run_notebooks_after - else [], - "ended_at_utc": _utc_now(), - }, - ) - return 0 - - base_env = os.environ.copy() - artifact_root = run_dir / "artifacts" - - stage_results: list[dict[str, Any]] = [] - for stage in selected_stages: - cmd = commands[stage] - log_path = run_dir / f"{stage}.log" - stage_env = ( - _build_rapids_env( - base_env=base_env, - run_tag=args.run_tag, - baseline_run_tag=args.baseline_run_tag, - artifact_root=artifact_root, - stage=stage, - ) - if stage in RAPIDS_STAGES - else _build_stage_env( - base_env=base_env, - run_tag=args.run_tag, - baseline_run_tag=args.baseline_run_tag, - artifact_root=artifact_root, - stage=stage, - ) - ) - started = time.perf_counter() - sampler = _GpuSampler(run_dir / f"{stage}_gpu_metrics.csv") - sampler.start() - with log_path.open("w", encoding="utf-8") as log_file: - log_file.write(f"$ {cmd}\n\n") - log_file.flush() - proc = subprocess.run( - cmd, - shell=True, - stdout=log_file, - stderr=subprocess.STDOUT, - env=stage_env, - text=True, - ) - duration = time.perf_counter() - started - gpu_metrics = sampler.stop() - stage_results.append( - { - "stage": stage, - "command": cmd, - "exit_code": int(proc.returncode), - "duration_seconds": round(duration, 3), - "log_path": str(log_path), - "gpu_metrics": gpu_metrics, - } - ) - if proc.returncode != 0: - _write_json( - summary_path, - { - **payload, - "state": "failed", - "ended_at_utc": _utc_now(), - "stage_results": stage_results, - "failed_stage": stage, - "final_exit_code": int(proc.returncode), - }, - ) - return int(proc.returncode) - - post_results: list[dict[str, Any]] = [] - if args.run_notebooks_after: - for stage, cmd in build_post_replay_commands( - notebook_timeout=args.notebook_timeout, - notebook_output_dir=args.notebook_output_dir, - notebook_inplace=bool(args.notebook_inplace), - include_side_projects=bool(args.include_side_projects), - extract_images_after=bool(args.extract_images_after), - ): - log_path = run_dir / f"{stage}.log" - started = time.perf_counter() - sampler = _GpuSampler(run_dir / f"{stage}_gpu_metrics.csv") - sampler.start() - with log_path.open("w", encoding="utf-8") as log_file: - log_file.write(f"$ {cmd}\n\n") - log_file.flush() - proc = subprocess.run( - cmd, - shell=True, - stdout=log_file, - stderr=subprocess.STDOUT, - env=_build_stage_env( - base_env=base_env, - run_tag=args.run_tag, - baseline_run_tag=args.baseline_run_tag, - artifact_root=artifact_root, - stage=stage, - ), - text=True, - ) - duration = time.perf_counter() - started - gpu_metrics = sampler.stop() - post_results.append( - { - "stage": stage, - "command": cmd, - "exit_code": int(proc.returncode), - "duration_seconds": round(duration, 3), - "log_path": str(log_path), - "gpu_metrics": gpu_metrics, - } - ) - if proc.returncode != 0: - _write_json( - summary_path, - { - **payload, - "state": "failed", - "ended_at_utc": _utc_now(), - "stage_results": stage_results, - "post_replay_results": post_results, - "failed_stage": stage, - "final_exit_code": int(proc.returncode), - }, - ) - return int(proc.returncode) - - _write_json( - summary_path, - { - **payload, - "state": "completed", - "ended_at_utc": _utc_now(), - "stage_results": stage_results, - "post_replay_results": post_results, - "final_exit_code": 0, - }, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_spo_comparison.py b/scripts/run_spo_comparison.py deleted file mode 100644 index 8f46af7..0000000 --- a/scripts/run_spo_comparison.py +++ /dev/null @@ -1,259 +0,0 @@ -"""Run SPO+ decision regret comparison for CRPTO. - -Compares decision regret of: -1. Two-stage classic: predict PD point estimates, optimize portfolio -2. Robust conformal: predict PD intervals, optimize with worst-case -3. SPO+ (if pyepo/torch available): train to minimize decision regret directly - -This script consumes canonical artifacts and produces -``models/spo_comparison_status.json`` for the CRPTO Streamlit page. -""" - -# ruff: noqa: E402 -from __future__ import annotations - -import argparse -import importlib -import json -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from src.utils.artifact_metadata import build_artifact_metadata - -SCHEMA_VERSION = "2026-03-16.1" - - -def _load_canonical_artifacts() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - """Load test predictions, conformal intervals, and portfolio allocation.""" - data_dir = ROOT / "data" / "processed" - - test_preds = pd.read_parquet(data_dir / "test_predictions.parquet") - logger.info("Loaded test_predictions: {} rows", len(test_preds)) - - intervals = pd.read_parquet(data_dir / "conformal_intervals_mondrian.parquet") - logger.info("Loaded conformal_intervals_mondrian: {} rows", len(intervals)) - - robustness = pd.read_parquet(data_dir / "portfolio_robustness_summary.parquet") - logger.info("Loaded portfolio_robustness_summary: {} rows", len(robustness)) - - return test_preds, intervals, robustness - - -def _compute_two_stage_regret( - y_true: np.ndarray, - y_pred_point: np.ndarray, -) -> dict[str, Any]: - """Compute proxy decision regret for two-stage (point estimate) approach. - - Uses a simplified regret metric: mean absolute difference between - the ranking induced by predictions vs true values, weighted by true cost. - """ - pred_order = np.argsort(y_pred_point) - true_order = np.argsort(y_true) - - # Rank correlation as proxy for decision alignment - n = len(y_true) - pred_ranks = np.empty(n, dtype=float) - pred_ranks[pred_order] = np.arange(n, dtype=float) - true_ranks = np.empty(n, dtype=float) - true_ranks[true_order] = np.arange(n, dtype=float) - - rank_regret = float(np.mean(np.abs(pred_ranks - true_ranks)) / n) - mae = float(np.mean(np.abs(y_pred_point - y_true))) - - return { - "approach": "two_stage_point", - "rank_regret": rank_regret, - "mae": mae, - "description": "Point PD estimate, optimize directly on predictions", - } - - -def _compute_robust_regret( - y_true: np.ndarray, - y_pred_point: np.ndarray, - y_pred_high: np.ndarray, -) -> dict[str, Any]: - """Compute proxy decision regret for robust conformal approach. - - The robust approach uses pd_high as worst-case, which is more conservative. - """ - n = len(y_true) - pred_order = np.argsort(y_pred_high) - true_order = np.argsort(y_true) - - pred_ranks = np.empty(n, dtype=float) - pred_ranks[pred_order] = np.arange(n, dtype=float) - true_ranks = np.empty(n, dtype=float) - true_ranks[true_order] = np.arange(n, dtype=float) - - rank_regret = float(np.mean(np.abs(pred_ranks - true_ranks)) / n) - mae_point = float(np.mean(np.abs(y_pred_point - y_true))) - mae_high = float(np.mean(np.abs(y_pred_high - y_true))) - coverage = float(np.mean(y_true <= y_pred_high)) - - return { - "approach": "robust_conformal", - "rank_regret": rank_regret, - "mae_point": mae_point, - "mae_high": mae_high, - "worst_case_coverage": coverage, - "description": "Conformal pd_high as worst-case, optimize conservatively", - } - - -def _try_spo_comparison( - y_true: np.ndarray, - y_pred_point: np.ndarray, -) -> dict[str, object] | None: - """Attempt SPO+ comparison if pyepo and torch are available.""" - try: - importlib.import_module("torch") - importlib.import_module("pyepo.func") - - logger.info("PyEPO and torch available. SPO+ comparison enabled.") - except ImportError: - logger.warning( - "PyEPO or torch not available. " - "SPO+ comparison skipped. Install with: pip install pyepo torch" - ) - return { - "approach": "spo_plus", - "status": "skipped", - "reason": "pyepo or torch not installed", - "description": "SPO+ trains model to minimize decision regret directly", - } - - # SPO+ requires a proper optimization model setup. - # For now, return a placeholder indicating the capability exists. - logger.info( - "SPO+ full training requires optimization model setup. Returning capability assessment." - ) - return { - "approach": "spo_plus", - "status": "capability_available", - "reason": "pyepo and torch installed; full training requires optmodel setup", - "description": "SPO+ trains neural net with decision-focused loss (Elmachtoub & Grigas 2022)", - "reference_code": "scripts/run_spo_real.py (src/optimization/spo_integration.py removed 2026-06, see CHANGELOG)", - } - - -def _compute_portfolio_economics( - robustness: pd.DataFrame, -) -> dict[str, object]: - """Extract portfolio economics from robustness summary.""" - if robustness.empty: - return {"status": "no_data"} - - result: dict[str, object] = {} - for _, row in robustness.iterrows(): - tol = float(row.get("risk_tolerance", 0)) - result[f"tol_{tol:.2f}"] = { - "risk_tolerance": tol, - "robust_return": float(row.get("best_robust_return", 0)), - "nonrobust_return": float(row.get("baseline_nonrobust_return", 0)), - "price_of_robustness": float(row.get("price_of_robustness", 0)), - "price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0)), - } - return result - - -def main() -> int: - """Run SPO+ decision regret comparison.""" - parser = argparse.ArgumentParser(description="SPO+ decision regret comparison") - parser.add_argument( - "--run-tag", - default=f"spo-comparison-{datetime.now(UTC).strftime('%Y-%m-%d-%H%M%S')}", - ) - parser.add_argument("--sample-size", type=int, default=50_000) - args = parser.parse_args() - - run_tag = str(args.run_tag).strip() - sample_size = int(args.sample_size) - - logger.info("Starting SPO+ comparison | run_tag={}", run_tag) - - test_preds, intervals, robustness = _load_canonical_artifacts() - - # Align predictions with true labels - target_col = "default_flag" if "default_flag" in test_preds.columns else "y_true" - y_true = test_preds[target_col].to_numpy(dtype=float) - y_pred_col = "y_prob_final" if "y_prob_final" in test_preds.columns else "y_prob_cb_tuned" - y_pred_point = test_preds[y_pred_col].to_numpy(dtype=float) - - # Get conformal upper bound - if "pd_high_90" in intervals.columns: - y_pred_high = intervals["pd_high_90"].to_numpy(dtype=float) - elif "y_pred" in intervals.columns: - y_pred_high = intervals["y_pred"].to_numpy(dtype=float) + 0.1 # fallback - else: - y_pred_high = y_pred_point + 0.1 - - # Align sizes - n = min(len(y_true), len(y_pred_high), sample_size) - y_true = y_true[:n] - y_pred_point = y_pred_point[:n] - y_pred_high = y_pred_high[:n] - - logger.info("Using {} samples for comparison", n) - - # Compute regret metrics - two_stage = _compute_two_stage_regret(y_true, y_pred_point) - robust = _compute_robust_regret(y_true, y_pred_point, y_pred_high) - spo = _try_spo_comparison(y_true, y_pred_point) - economics = _compute_portfolio_economics(robustness) - two_stage_rank_regret = float(two_stage["rank_regret"]) - robust_rank_regret = float(robust["rank_regret"]) - - # Build output - output = { - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=run_tag, - ), - "sample_size": n, - "prediction_column": y_pred_col, - "two_stage": two_stage, - "robust_conformal": robust, - "spo_plus": spo, - "portfolio_economics": economics, - "summary": { - "two_stage_rank_regret": two_stage_rank_regret, - "robust_rank_regret": robust_rank_regret, - "robust_vs_two_stage_pct": float( - (two_stage_rank_regret - robust_rank_regret) - / max(two_stage_rank_regret, 1e-9) - * 100 - ), - }, - } - - out_path = ROOT / "models" / "spo_comparison_status.json" - out_path.write_text( - json.dumps(output, indent=2, ensure_ascii=False, default=str) + "\n", - encoding="utf-8", - ) - logger.info("Written SPO+ comparison to {}", out_path) - - logger.info( - "Two-stage rank regret: {:.6f} | Robust rank regret: {:.6f} | Improvement: {:.1f}%", - two_stage["rank_regret"], - robust["rank_regret"], - output["summary"]["robust_vs_two_stage_pct"], - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_spo_real.py b/scripts/run_spo_real.py deleted file mode 100644 index e4adfef..0000000 --- a/scripts/run_spo_real.py +++ /dev/null @@ -1,699 +0,0 @@ -"""SPO+ Real Training for CRPTO — v2. - -Implements Smart Predict-then-Optimize (Elmachtoub & Grigas 2022) using PyEPO -with OR-Tools as the LP backend for credit portfolio selection. - -Fixes from v1: - Fix 1: Point-wise permutation-equivariant MLP (each loan processed independently). - Fix 2: Calibrated PD (CatBoost + Venn-Abers) as true cost proxy, not binary default_flag. - Fix 3: Multi-seed evaluation → mean±std + Wilcoxon signed-rank test vs two-stage. - Fix 4: Conformal robust optimization as 3rd method (pd_high_90 worst-case PD cost). - Fix 5: Default n_items=100 for richer combinatorial decision space. - -Usage: - uv run python scripts/run_spo_real.py - uv run python scripts/run_spo_real.py --n-items 100 --budget 30 --epochs 50 --seeds 5 -""" - -from __future__ import annotations - -import argparse -import importlib -import json -import pickle -import time -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger -from ortools.linear_solver import pywraplp -from scipy import stats - -from src.models.conformal import apply_probability_calibrator -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag - -SCHEMA_VERSION = "2026-03-17.2" -LGD = 0.40 -RANDOM_SEED = 42 - - -def _require_optional_module(module_name: str) -> Any: - """Import an optional SPO dependency with an explicit experiment-level error.""" - try: - return importlib.import_module(module_name) - except ImportError as exc: - raise RuntimeError( - "SPO+ is an optional experiment. Install the `spo` extras before " - f"running this script; missing module: {module_name}." - ) from exc - - -pyepo: Any = _require_optional_module("pyepo") -torch: Any = _require_optional_module("torch") -nn: Any = _require_optional_module("torch.nn") -OptModelBase: Any = _require_optional_module("pyepo.model.opt").optModel - -NUMERIC_FEATURES = [ - "loan_amnt", - "int_rate", - "annual_inc", - "dti", - "fico_range_low", - "open_acc", - "revol_bal", - "revol_util", - "total_acc", - "installment", - "emp_length", - "pub_rec", - "delinq_2yrs", - "inq_last_6mths", - "mths_since_last_delinq", -] - - -# ── 1. Portfolio LP optModel ──────────────────────────────────────────────── - - -class CreditPortfolioLP(OptModelBase): - """Portfolio selection LP: select exactly `budget` of `n_items` loans to minimize - expected loss c^T x = sum(x_i * (PD_i * LGD - r_i)). - - Args: - n_items: Loans per problem instance. - budget: Loans to select (equality constraint). - """ - - def __init__(self, n_items: int, budget: int) -> None: - self.n_items = n_items - self.budget = budget - self.modelSense = pyepo.EPO.MINIMIZE - super().__init__() - - def _getModel(self) -> tuple: - solver = pywraplp.Solver.CreateSolver("GLOP") - solver.SuppressOutput() - x = {i: solver.NumVar(0.0, 1.0, f"x{i}") for i in range(self.n_items)} - ct = solver.Constraint(float(self.budget), float(self.budget)) - for i in range(self.n_items): - ct.SetCoefficient(x[i], 1.0) - return solver, x - - def setObj(self, c: np.ndarray | Any) -> None: - if isinstance(c, torch.Tensor): - c = c.detach().cpu().numpy() - c = np.asarray(c, dtype=float) - obj = self._model.Objective() - obj.Clear() - for i in range(self.n_items): - obj.SetCoefficient(self.x[i], float(c[i])) - obj.SetMinimization() - - def solve(self) -> tuple[list[float], float]: - status = self._model.Solve() - if status not in (pywraplp.Solver.OPTIMAL, pywraplp.Solver.FEASIBLE): - logger.warning("LP solver failed (status={}), greedy fallback", status) - sol = [0.0] * self.n_items - for i in range(self.budget): - sol[i] = 1.0 - return sol, 0.0 - sol = [self.x[i].solution_value() for i in range(self.n_items)] - return sol, self._model.Objective().Value() - - def copy(self) -> CreditPortfolioLP: - return CreditPortfolioLP(self.n_items, self.budget) - - -# ── 2. Fix 1: Point-wise permutation-equivariant MLP ──────────────────────── - - -class PDPredictorMLP(nn.Module): - """Point-wise MLP that processes each loan independently. - - Input: (B, n_items * n_features) — flattened instance features (optDataset compat). - Internally: reshapes to (B * n_items, n_features), processes each loan with shared weights. - Output: (B, n_items) — predicted cost per loan. - - Permutation-equivariant: shuffling loans within an instance consistently - shuffles predictions. Much lower input dimensionality than flat MLP. - """ - - def __init__(self, n_features: int, n_items: int) -> None: - super().__init__() - self.n_features = n_features - self.n_items = n_items - self.net = nn.Sequential( - nn.Linear(n_features, 64), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(64, 32), - nn.ReLU(), - nn.Linear(32, 1), - ) - - def forward(self, x: Any) -> Any: - B = x.shape[0] - # (B, n_items * n_features) → (B * n_items, n_features) - x_per_loan = x.view(B * self.n_items, self.n_features) - out = self.net(x_per_loan) # (B * n_items, 1) - return out.view(B, self.n_items) - - -# ── 3. Fix 2: Calibrated PD as true cost ──────────────────────────────────── - - -def _load_pd_artifacts() -> tuple | None: - """Load CatBoost model, calibrator, and feature contract. Returns None if unavailable.""" - paths = [ - Path("models/pd_canonical.cbm"), - Path("models/pd_canonical_calibrator.pkl"), - Path("models/pd_model_contract.json"), - ] - if not all(p.exists() for p in paths): - return None - try: - from catboost import CatBoostClassifier - - model = CatBoostClassifier() - model.load_model(str(paths[0])) - with open(paths[1], "rb") as f: - calibrator = pickle.load(f) - with open(paths[2]) as f: - contract = json.load(f) - return ( - model, - calibrator, - contract["feature_names"], - contract.get("categorical_features", []), - ) - except Exception as e: - logger.warning("PD artifacts unavailable: {} — using binary default_flag costs", e) - return None - - -def _predict_calibrated_costs( - df: pd.DataFrame, - model: Any, - calibrator: object, - feature_names: list[str], - cat_feats: list[str], -) -> np.ndarray: - """Predict calibrated PD and return cost_i = pd_i * LGD - int_rate_i.""" - avail = [f for f in feature_names if f in df.columns] - X = df[avail].copy() - for col in avail: - if col in cat_feats: - X[col] = X[col].astype(str).fillna("UNKNOWN") - else: - X[col] = pd.to_numeric(X[col], errors="coerce").fillna(0.0) - raw_pd = model.predict_proba(X)[:, 1] - cal_pd = apply_probability_calibrator(calibrator, raw_pd) - int_rate = pd.to_numeric(df["int_rate"], errors="coerce").fillna(12.0).values / 100.0 - return (cal_pd * LGD - int_rate).astype(np.float32) - - -def _binary_costs(df: pd.DataFrame) -> np.ndarray: - int_rate = pd.to_numeric(df["int_rate"], errors="coerce").fillna(12.0).values / 100.0 - return (df["default_flag"].astype(float).values * LGD - int_rate).astype(np.float32) - - -# ── 4. Data loading (done once, outside seed loop) ────────────────────────── - - -def _prep_features( - df: pd.DataFrame, - avail: list[str], - mu: np.ndarray | None = None, - sigma: np.ndarray | None = None, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Normalize features. Returns (X, mu, sigma).""" - X = df[avail].copy() - for col in avail: - X[col] = pd.to_numeric(X[col], errors="coerce") - X = X.fillna(X.median()) - X_arr = X.values.astype(np.float32) - if mu is None: - mu = X_arr.mean(axis=0) - sigma = X_arr.std(axis=0) + 1e-8 - if sigma is None: - sigma = X_arr.std(axis=0) + 1e-8 - return ((X_arr - mu) / sigma).astype(np.float32), mu, sigma - - -def _load_conformal_pd_high() -> np.ndarray | None: - """Load pd_high_90 from conformal intervals (aligned with test_fe rows).""" - path = Path("data/processed/conformal_intervals_mondrian.parquet") - if not path.exists(): - return None - df = pd.read_parquet(path, columns=["pd_high_90"]) - return df["pd_high_90"].values.astype(np.float32) - - -def _load_all_data(avail: list[str]) -> tuple: - """Load and pre-process all loan-level data. Returns arrays for train and test.""" - train = pd.read_parquet("data/processed/train_fe.parquet") - test = pd.read_parquet("data/processed/test_fe.parquet") - logger.info("Loaded: train {:,} | test {:,} rows", len(train), len(test)) - - X_tr, mu, sigma = _prep_features(train, avail) - X_te, _, _ = _prep_features(test, avail, mu=mu, sigma=sigma) - - pd_arts = _load_pd_artifacts() - if pd_arts is not None: - cb_model, calibrator, feat_names, cat_feats = pd_arts - logger.info( - "Predicting calibrated PD on {:,} train + {:,} test loans...", len(train), len(test) - ) - t0 = time.time() - c_tr = _predict_calibrated_costs(train, cb_model, calibrator, feat_names, cat_feats) - c_te = _predict_calibrated_costs(test, cb_model, calibrator, feat_names, cat_feats) - logger.info( - " Done in {:.1f}s | cost range train [{:.4f}, {:.4f}]", - time.time() - t0, - c_tr.min(), - c_tr.max(), - ) - use_calibrated = True - else: - c_tr = _binary_costs(train) - c_te = _binary_costs(test) - logger.warning("Using binary default_flag costs (fallback)") - use_calibrated = False - - # Fix 3: Two-stage baseline — Ridge regression fitted on full training data - from sklearn.linear_model import Ridge - - logger.info("Fitting Ridge regression for two-stage baseline...") - ridge = Ridge(alpha=1.0) - ridge.fit(X_tr, c_tr) - c_ts_te = ridge.predict(X_te).astype(np.float32) - logger.info(" Two-stage Ridge: train R²={:.4f}", ridge.score(X_tr, c_tr)) - - # Fix 4: Conformal robust costs for test set - pd_high = _load_conformal_pd_high() - if pd_high is not None and len(pd_high) == len(test): - int_rate_te = pd.to_numeric(test["int_rate"], errors="coerce").fillna(12.0).values / 100.0 - c_robust_te = (pd_high * LGD - int_rate_te).astype(np.float32) - logger.info( - "Conformal robust costs loaded (pd_high_90 ≤ 0.20 for {:,} loans)", - (pd_high < 0.20).sum(), - ) - else: - c_robust_te = None - logger.warning("Conformal intervals not available — skipping conformal robust method") - - return X_tr, c_tr, X_te, c_te, c_ts_te, c_robust_te, use_calibrated - - -# ── 5. Instance sampling ───────────────────────────────────────────────────── - - -def _sample_instances( - X_all: np.ndarray, - c_all: np.ndarray, - n_items: int, - n_inst: int, - rng: np.random.RandomState, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Sample n_inst problem instances of n_items loans each. - - Returns: - feats: (n_inst, n_items, n_features) — per-loan features per instance - costs: (n_inst, n_items) — per-loan costs per instance - indices: (n_inst, n_items) — row indices into source arrays - """ - n = len(X_all) - feats, costs, indices = [], [], [] - for _ in range(n_inst): - idx = rng.choice(n, size=n_items, replace=False) - feats.append(X_all[idx]) - costs.append(c_all[idx]) - indices.append(idx) - return np.stack(feats), np.stack(costs), np.array(indices) - - -def _index_costs(c_all: np.ndarray, indices: np.ndarray) -> np.ndarray: - """Look up pre-computed costs for sampled indices: (n_inst, n_items).""" - return np.array([c_all[idx] for idx in indices]) - - -# ── 6. SPO+ training ────────────────────────────────────────────────────────── - - -def _train_spo( - X_inst_train: np.ndarray, - c_inst_train: np.ndarray, - optmodel: CreditPortfolioLP, - n_features: int, - n_items: int, - epochs: int, - lr: float, - batch_size: int, - seed: int, -) -> tuple[PDPredictorMLP, list[float]]: - """Train SPO+ MLP using optDataset pre-solved instances. - - Args: - X_inst_train: (n_inst, n_items, n_features) — instance features. - c_inst_train: (n_inst, n_items) — true costs per instance. - """ - optDataset = _require_optional_module("pyepo.data.dataset").optDataset - SPOPlus = _require_optional_module("pyepo.func").SPOPlus - DataLoader = _require_optional_module("torch.utils.data").DataLoader - - n_input = n_items * n_features - X_flat = X_inst_train.reshape(len(X_inst_train), n_input) - - logger.info(" Pre-solving {} instances with LP...", len(X_flat)) - t_pre = time.time() - dataset = optDataset(optmodel, X_flat, c_inst_train) - logger.info(" optDataset built in {:.1f}s", time.time() - t_pre) - - np.random.seed(seed) - torch.manual_seed(seed) - - model = PDPredictorMLP(n_features=n_features, n_items=n_items) - spo_loss_fn = SPOPlus(optmodel, processes=1) - optimizer = torch.optim.Adam(model.parameters(), lr=lr) - loader = DataLoader(dataset, batch_size=batch_size, shuffle=True) - - losses: list[float] = [] - for epoch in range(epochs): - epoch_loss, n_batches = 0.0, 0 - for feats_b, costs_b, sols_b, objs_b in loader: - optimizer.zero_grad() - c_hat = model(feats_b.float()) # (B, n_items) via point-wise reshape - loss = spo_loss_fn(c_hat, costs_b.float(), sols_b.float(), objs_b.float()) - loss.backward() - optimizer.step() - epoch_loss += float(loss.item()) - n_batches += 1 - avg = epoch_loss / max(n_batches, 1) - losses.append(avg) - if (epoch + 1) % 10 == 0 or epoch == 0: - logger.info(" Epoch {:3d}/{} | SPO+ loss = {:.6f}", epoch + 1, epochs, avg) - - return model, losses - - -# ── 7. Evaluation ───────────────────────────────────────────────────────────── - - -def _compute_true_optima(c_true: np.ndarray, optmodel: CreditPortfolioLP) -> list[tuple]: - """Pre-compute optimal LP decisions for all instances (cached for all methods).""" - optima = [] - for i in range(len(c_true)): - optmodel.setObj(c_true[i]) - x_star, _ = optmodel.solve() - optima.append((np.array(x_star), float(np.dot(c_true[i], x_star)))) - return optima - - -def _compute_regret( - c_pred: np.ndarray, - c_true: np.ndarray, - optmodel: CreditPortfolioLP, - true_optima: list[tuple], -) -> np.ndarray: - """Per-instance regret using pre-computed optima (no redundant LP solves).""" - regrets = [] - for i in range(len(c_pred)): - _, true_opt = true_optima[i] - optmodel.setObj(c_pred[i]) - x_pred, _ = optmodel.solve() - regrets.append(float(np.dot(c_true[i], x_pred)) - true_opt) - return np.array(regrets) - - -# ── 8. Main ─────────────────────────────────────────────────────────────────── - - -def main() -> int: - parser = argparse.ArgumentParser(description="SPO+ real training — v2 (5 fixes)") - parser.add_argument( - "--n-items", type=int, default=100, help="Loans per problem instance (Fix 5: default 100)" - ) - parser.add_argument( - "--budget", type=int, default=30, help="Loans to select per instance (default 30%%)" - ) - parser.add_argument( - "--n-train", type=int, default=800, help="Training instances per seed (default 800)" - ) - parser.add_argument( - "--n-test", type=int, default=200, help="Test instances per seed (default 200)" - ) - parser.add_argument("--epochs", type=int, default=50, help="SPO+ training epochs (default 50)") - parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate (default 0.001)") - parser.add_argument("--batch-size", type=int, default=32, help="Batch size (default 32)") - parser.add_argument( - "--seeds", - type=int, - default=5, - help="Number of random seeds for multi-seed eval (Fix 3: default 5)", - ) - args = parser.parse_args() - - run_tag = resolve_run_tag(None, allow_untracked=True) - logger.info( - "SPO+ v2 | n_items={} budget={} n_train={} n_test={} epochs={} seeds={} run_tag={}", - args.n_items, - args.budget, - args.n_train, - args.n_test, - args.epochs, - args.seeds, - run_tag, - ) - - # ── Load all data once (outside seed loop) ─────────────────────────────── - avail = [f for f in NUMERIC_FEATURES if True] # all checked per-dataset below - train_check = pd.read_parquet("data/processed/train_fe.parquet", columns=NUMERIC_FEATURES[:1]) - test_check = pd.read_parquet("data/processed/test_fe.parquet", columns=NUMERIC_FEATURES[:1]) - del train_check, test_check - - # Load train_fe to get available columns - _tr_cols = pd.read_parquet("data/processed/train_fe.parquet").columns - _te_cols = pd.read_parquet("data/processed/test_fe.parquet").columns - avail = [f for f in NUMERIC_FEATURES if f in _tr_cols and f in _te_cols] - n_features = len(avail) - n_input = args.n_items * n_features - logger.info("Using {} features: {}", n_features, avail) - - t_load = time.time() - X_tr_all, c_tr_all, X_te_all, c_te_all, c_ts_te_all, c_robust_te_all, use_cal = _load_all_data( - avail - ) - logger.info("Data loaded in {:.1f}s | use_calibrated_pd={}", time.time() - t_load, use_cal) - - # ── Multi-seed evaluation loop ─────────────────────────────────────────── - all_regrets: dict[str, list[np.ndarray]] = { - "two_stage": [], - "spo_plus": [], - } - if c_robust_te_all is not None: - all_regrets["conformal_robust"] = [] - - per_seed_means: dict[str, list[float]] = {k: [] for k in all_regrets} - all_spo_losses: list[list[float]] = [] - t_total = time.time() - - for seed_idx in range(args.seeds): - seed = RANDOM_SEED + seed_idx * 1000 - logger.info("=== Seed {}/{} (seed={}) ===", seed_idx + 1, args.seeds, seed) - rng = np.random.RandomState(seed) - - # Sample train instances (for SPO+ training) - X_tr_inst, c_tr_inst, _ = _sample_instances( - X_tr_all, c_tr_all, args.n_items, args.n_train, rng - ) - - # Sample test instances (shared across all methods — paired comparison) - X_te_inst, c_te_inst, idx_te = _sample_instances( - X_te_all, c_te_all, args.n_items, args.n_test, rng - ) - - # Look up pre-computed two-stage and robust costs for these test indices - c_ts_inst = _index_costs(c_ts_te_all, idx_te) # (n_test, n_items) - c_robust_inst = ( - _index_costs(c_robust_te_all, idx_te) if c_robust_te_all is not None else None - ) - - # Compute true optima ONCE (shared cache for all methods) - optmodel_eval = CreditPortfolioLP(n_items=args.n_items, budget=args.budget) - logger.info(" Pre-computing true optima for {} test instances...", args.n_test) - t_opt = time.time() - true_optima = _compute_true_optima(c_te_inst, optmodel_eval) - logger.info(" True optima computed in {:.1f}s", time.time() - t_opt) - - # SPO+ training - optmodel_train = CreditPortfolioLP(n_items=args.n_items, budget=args.budget) - spo_model, spo_losses = _train_spo( - X_tr_inst, - c_tr_inst, - optmodel_train, - n_features=n_features, - n_items=args.n_items, - epochs=args.epochs, - lr=args.lr, - batch_size=args.batch_size, - seed=seed, - ) - all_spo_losses.append(spo_losses) - - # SPO+ predictions for test instances - X_te_flat = X_te_inst.reshape(args.n_test, n_input) - spo_model.eval() - with torch.no_grad(): - c_spo_inst = spo_model(torch.tensor(X_te_flat, dtype=torch.float32)).numpy() - - # Evaluate all methods - t_eval = time.time() - regrets_ts = _compute_regret(c_ts_inst, c_te_inst, optmodel_eval.copy(), true_optima) - regrets_spo = _compute_regret(c_spo_inst, c_te_inst, optmodel_eval.copy(), true_optima) - logger.info( - " Regret — two_stage: {:.4f}±{:.4f} | spo_plus: {:.4f}±{:.4f} (eval {:.1f}s)", - regrets_ts.mean(), - regrets_ts.std(), - regrets_spo.mean(), - regrets_spo.std(), - time.time() - t_eval, - ) - - all_regrets["two_stage"].append(regrets_ts) - all_regrets["spo_plus"].append(regrets_spo) - per_seed_means["two_stage"].append(float(regrets_ts.mean())) - per_seed_means["spo_plus"].append(float(regrets_spo.mean())) - - if c_robust_inst is not None: - regrets_robust = _compute_regret( - c_robust_inst, c_te_inst, optmodel_eval.copy(), true_optima - ) - all_regrets["conformal_robust"].append(regrets_robust) - per_seed_means["conformal_robust"].append(float(regrets_robust.mean())) - logger.info( - " conformal_robust: {:.4f}±{:.4f}", regrets_robust.mean(), regrets_robust.std() - ) - - total_time = time.time() - t_total - logger.info("All seeds done in {:.1f}s", total_time) - - # ── Aggregate across seeds ─────────────────────────────────────────────── - pooled: dict[str, np.ndarray] = {k: np.concatenate(v) for k, v in all_regrets.items()} - agg: dict[str, dict[str, Any]] = {} - for method, arr in pooled.items(): - agg[method] = { - "mean_regret": float(arr.mean()), - "std_regret": float(arr.std()), - "median_regret": float(np.median(arr)), - "per_seed_means": per_seed_means[method], - } - - # Wilcoxon signed-rank test: H1 = two_stage regrets > spo_plus regrets - wil_stat, wil_pval = stats.wilcoxon( - pooled["two_stage"], - pooled["spo_plus"], - alternative="greater", - ) - improvement_pct = ( - (agg["two_stage"]["mean_regret"] - agg["spo_plus"]["mean_regret"]) - / (abs(agg["two_stage"]["mean_regret"]) + 1e-9) - * 100 - ) - - logger.info("=== Final Results ===") - logger.info( - "Two-stage: {:.4f} ± {:.4f}", - agg["two_stage"]["mean_regret"], - agg["two_stage"]["std_regret"], - ) - logger.info( - "SPO+: {:.4f} ± {:.4f}", - agg["spo_plus"]["mean_regret"], - agg["spo_plus"]["std_regret"], - ) - if "conformal_robust" in agg: - logger.info( - "Conformal robust: {:.4f} ± {:.4f}", - agg["conformal_robust"]["mean_regret"], - agg["conformal_robust"]["std_regret"], - ) - logger.info("SPO+ improvement: {:.2f}%", improvement_pct) - logger.info("Wilcoxon p-value: {:.4f} (significant={})", wil_pval, wil_pval < 0.05) - - # ── Save outputs ────────────────────────────────────────────────────────── - # Average loss curve across seeds - avg_losses = np.mean(all_spo_losses, axis=0).tolist() - loss_df = pd.DataFrame( - { - "epoch": range(1, len(avg_losses) + 1), - "spo_loss": avg_losses, - } - ) - loss_path = Path("data/processed/spo_training_loss.parquet") - loss_df.to_parquet(loss_path, index=False) - - metadata = build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=run_tag, - allow_untracked=True, - extra={ - "n_items": args.n_items, - "budget": args.budget, - "n_train_instances": args.n_train, - "n_test_instances": args.n_test, - "epochs": args.epochs, - "n_seeds": args.seeds, - "n_features": n_features, - "n_input": n_input, - "feature_names": avail, - "use_calibrated_pd": use_cal, - }, - ) - status: dict[str, object] = { - **metadata, - "results": { - **agg, - "spo_improvement_vs_ts_pct": improvement_pct, - "wilcoxon_spo_vs_ts": { - "statistic": float(wil_stat), - "pvalue": float(wil_pval), - "alternative": "two_stage > spo_plus", - "significant_at_0.05": bool(wil_pval < 0.05), - }, - "n_paired_observations": int(args.seeds * args.n_test), - }, - "config": { - "n_items": args.n_items, - "budget": args.budget, - "selection_rate": args.budget / args.n_items, - "lgd": LGD, - "lr": args.lr, - "batch_size": args.batch_size, - }, - "fixes_applied": { - "fix1_pointwise_mlp": True, - "fix2_calibrated_pd_costs": use_cal, - "fix3_multi_seed_wilcoxon": True, - "fix4_conformal_robust": "conformal_robust" in agg, - "fix5_larger_n_items": args.n_items >= 100, - }, - "train_time_seconds": total_time, - "note": ( - "SPO+ v2: point-wise MLP + calibrated PD costs + multi-seed Wilcoxon test " - "+ conformal robust as 3rd comparison method. " - "Two-stage uses Ridge regression to predict calibrated cost. " - "SPO+ trains end-to-end to minimize decision regret directly." - ), - } - status_path = Path("models/spo_real_training_status.json") - with open(status_path, "w") as f: - json.dump(status, f, indent=2, default=str) - logger.info("Saved status: {}", status_path) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/run_ty_advisory.py b/scripts/run_ty_advisory.py index ab6736c..5c83a9c 100644 --- a/scripts/run_ty_advisory.py +++ b/scripts/run_ty_advisory.py @@ -9,24 +9,17 @@ from collections.abc import Sequence from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[1] DEFAULT_OUTPUT = ROOT / "reports" / "ci" / "ty-advisory.txt" -TY_REQUIREMENT = "ty==0.0.57" +TY_REQUIREMENT = "ty==0.0.59" SOURCE_ROOTS = ("src", "scripts") -ACTIVE_EXCLUDED_FILES = { - "scripts/generate_conformal_intervals.py", - "scripts/run_cqr_comparison.py", - "scripts/run_crpto_vs_spo_stability.py", - "scripts/run_spo_comparison.py", - "scripts/run_spo_real.py", - "scripts/train_pd_model.py", - "src/optimization/cuopt_adapter.py", -} -ACTIVE_EXPERIMENT_FILES = { - "scripts/experiments/ijds_policy_support.py", - "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py", - "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py", +COMPATIBILITY_SOURCE_FILES = { + "src/data/make_dataset.py", + "src/data/prepare_dataset.py", + "src/optimization/tail_satisficing_objective.py", } SUMMARY_RE = re.compile(r"^Found \d+ diagnostics", flags=re.MULTILINE) @@ -37,22 +30,33 @@ def _relative_posix(path: Path) -> str: def iter_python_files(*, scope: str) -> list[str]: """Return the Python files ty should check for a given advisory scope.""" - files: list[str] = [] - for root_name in SOURCE_ROOTS: - for path in sorted((ROOT / root_name).rglob("*.py")): - rel = _relative_posix(path) - parts = rel.split("/") - if scope == "active": - if parts[:2] == ["scripts", "archive"]: - continue - if parts[:2] == ["scripts", "experiments"] and rel not in (ACTIVE_EXPERIMENT_FILES): - continue - if parts[:2] == ["scripts", "search"] and path.name.startswith("run_"): - continue - if rel in ACTIVE_EXCLUDED_FILES: - continue - files.append(rel) - return files + if scope not in {"active", "full"}: + raise ValueError(f"Unsupported ty scope: {scope}") + full = [ + _relative_posix(path) + for root_name in SOURCE_ROOTS + for path in sorted((ROOT / root_name).rglob("*.py")) + ] + if scope == "full": + return full + + publication = yaml.safe_load( + (ROOT / "configs/crpto_publication_targets.yaml").read_text(encoding="utf-8") + ) + surface = publication["active_scientific_contract"]["active_code_surface"] + active_scripts = { + *surface["paper_pipeline"], + *surface["protocol_entrypoints"], + *surface["support_tools"], + "scripts/__init__.py", + "scripts/experiments/__init__.py", + } + return [ + path + for path in full + if (path.startswith("src/") and path not in COMPATIBILITY_SOURCE_FILES) + or path in active_scripts + ] def build_ty_command(*, uvx: str, files: Sequence[str], fail_on_diagnostics: bool) -> list[str]: @@ -74,8 +78,18 @@ def build_ty_command(*, uvx: str, files: Sequence[str], fail_on_diagnostics: boo return [*command, *files] -def run_ty(scope: str, output: Path, *, fail_on_diagnostics: bool = False) -> int: - """Run pinned ty, persist its report, and optionally enforce diagnostics.""" +def _diagnostic_lines(output: str) -> list[str]: + """Return every concise ty error line.""" + return [line for line in output.splitlines() if ": error[" in line] + + +def run_ty( + scope: str, + output: Path | None, + *, + fail_on_diagnostics: bool = False, +) -> int: + """Run pinned ty, optionally persist its report, and enforce diagnostics.""" uvx = shutil.which("uvx") if uvx is None: raise RuntimeError("uvx is required to run the ty advisory check.") @@ -93,40 +107,62 @@ def run_ty(scope: str, output: Path, *, fail_on_diagnostics: bool = False) -> in capture_output=True, check=False, ) - output.parent.mkdir(parents=True, exist_ok=True) + raw_output = f"{result.stdout}{result.stderr}" + diagnostics = _diagnostic_lines(raw_output) + effective_return_code = result.returncode report = ( f"# ty advisory report\n" f"requirement: {TY_REQUIREMENT}\n" f"scope: {scope}\n" f"blocking: {str(fail_on_diagnostics).lower()}\n" f"files_checked: {len(files)}\n" - f"return_code: {result.returncode}\n" + f"raw_return_code: {result.returncode}\n" + f"effective_return_code: {effective_return_code}\n" + f"diagnostics: {len(diagnostics)}\n" f"\n" - f"{result.stdout}{result.stderr}" + f"{raw_output}" ) - output.write_text(report, encoding="utf-8") + if output is not None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(report, encoding="utf-8") + elif raw_output.strip(): + print(raw_output.rstrip()) summary = SUMMARY_RE.search(report) - if summary: + if diagnostics: + print(f"Found {len(diagnostics)} diagnostics") + elif summary: print(summary.group(0)) - elif result.returncode == 0: + elif effective_return_code == 0: print("ty advisory clean") else: - print(f"ty failed with return code {result.returncode}") - print(f"Full report: {output.relative_to(ROOT)}") - return result.returncode if fail_on_diagnostics else 0 + print(f"ty failed with return code {effective_return_code}") + if output is not None: + try: + display_path = output.relative_to(ROOT) + except ValueError: + display_path = output + print(f"Full report: {display_path}") + return effective_return_code if fail_on_diagnostics else 0 def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--scope", choices=["active", "full"], default="active") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--no-report", + action="store_true", + help="Do not write a report file; emit diagnostics to the terminal only.", + ) parser.add_argument( "--fail-on-diagnostics", action="store_true", help="Return ty's nonzero status when diagnostics are present.", ) args = parser.parse_args() - output = args.output if args.output.is_absolute() else ROOT / args.output + output = None + if not args.no_report: + output = args.output if args.output.is_absolute() else ROOT / args.output return run_ty( scope=args.scope, output=output, diff --git a/scripts/search/__init__.py b/scripts/search/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/search/build_pool93_body_allocation_audit.py b/scripts/search/build_pool93_body_allocation_audit.py deleted file mode 100644 index 2d4d365..0000000 --- a/scripts/search/build_pool93_body_allocation_audit.py +++ /dev/null @@ -1,339 +0,0 @@ -"""Export the pool93 body-point funded set and compact grade audit. - -This is a paper-facing sidecar, not a new portfolio search. It re-solves the -selected finite-grid body policy from the consolidated frontier at alpha=0.01 -and writes row-level funded allocations plus a small grade-bucket table. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 -from scripts.validate_alpha_gamma_bound import ( # noqa: E402 - DEFAULT_LGD, - DEFAULT_MAX_CONCENTRATION, - DEFAULT_TIME_LIMIT, - _compute_effective_pd_vector, - _compute_intervals_at_alpha, - _load_aligned_dataset, -) -from src.optimization.certificate_semantics import ( # noqa: E402 - compute_funded_certificate_metrics, -) -from src.optimization.portfolio_model import ( # noqa: E402 - optimize_portfolio_allocation, - solution_allocation_vector, -) -from src.utils.script_helpers import resolve_repo_artifact_path # noqa: E402 - -DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" -DEFAULT_BODY_ROLE = "body/default balanced return-bound point" - - -def _default_frontier_path(consolidated_tag: str) -> Path: - return ( - ROOT - / "models/experiments/champion_reopen" - / consolidated_tag - / "portfolio/pool93_ijds_consolidated_frontier.json" - ) - - -def _manifest_path(run_tag: str) -> Path: - return ( - ROOT - / "models/experiments/champion_reopen" - / run_tag - / "portfolio/pool93_ijds_local_refinement_manifest.json" - ) - - -def _load_role_row(frontier_path: Path, role: str) -> dict[str, Any]: - payload = json.loads(frontier_path.read_text(encoding="utf-8")) - matches = [row for row in payload.get("rows", []) if str(row.get("role")) == role] - if not matches: - raise ValueError(f"Role {role!r} not found in {frontier_path}") - return dict(matches[0]) - - -def _policy_from_row(row: dict[str, Any]) -> dict[str, Any]: - semantic = row.get("semantic_policy_key") - policy: dict[str, Any] = ( - json.loads(semantic) if isinstance(semantic, str) and semantic.strip() else {} - ) - for key in [ - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - ]: - if key in row: - policy[key] = row[key] - policy.setdefault("delta_cap_quantile", 1.0) - policy.setdefault("tail_focus_quantile", 1.0) - policy.setdefault("min_budget_utilization", 0.0) - policy.setdefault("pd_cap_slack_penalty", 0.0) - policy.setdefault("solver_backend", "highspy") - return policy - - -def _grade_bucket(series: pd.Series) -> pd.Series: - grade = series.fillna("unknown").astype(str).str.upper().str[:1] - buckets = np.select( - [ - grade.isin(["A", "B"]), - grade.eq("C"), - grade.eq("D"), - grade.isin(["E", "F", "G"]), - ], - ["A-B", "C", "D", "E-G"], - default="unknown", - ) - return pd.Series(buckets, index=series.index, name="grade_bucket") - - -def _format_tex_table(summary: pd.DataFrame) -> str: - lines = [ - "\\begin{tabular}{lrrrrr}", - "\\toprule", - "Grade bucket & Funded rows & Exposure share & Default rate & $V$ contribution & Mean $u_i(0.01)$ \\\\", - "\\midrule", - ] - for row in summary.to_dict("records"): - lines.append( - f"{row['grade_bucket']} & {row['funded_rows']:,.0f} & " - f"{row['exposure_share']:.2%} & {row['default_rate']:.2%} & " - f"{row['v_contribution']:.5f} & {row['mean_pd_high_alpha01']:.5f} \\\\" - ) - lines.extend(["\\bottomrule", "\\end{tabular}", ""]) - return "\n".join(lines) - - -def build_audit( - *, - frontier_path: Path, - role: str, - alpha: float, - output_dir: Path, - report_table_dir: Path, - threads: int, - solver_backend: str | None, -) -> dict[str, Any]: - row = _load_role_row(frontier_path, role) - policy = _policy_from_row(row) - if solver_backend: - policy["solver_backend"] = solver_backend - - manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) - conformal_intervals_path = str( - resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) - ) - aligned = _load_aligned_dataset( - conformal_intervals_path=conformal_intervals_path, - max_candidates=int(manifest.get("max_candidates", 0) or 0), - random_state=int(manifest.get("random_state", 42)), - ) - pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) - effective_pd = _compute_effective_pd_vector(aligned, pd_point, pd_high, policy) - int_rates = ( - _parse_percent_series(aligned["int_rate"]) - if "int_rate" in aligned.columns - else np.full(len(aligned), 0.12) - ) - loan_amounts = ( - pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in aligned.columns - else np.ones(len(aligned), dtype=float) - ) - y_true = ( - pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0).to_numpy(dtype=float) - if "y_true" in aligned.columns - else pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=float) - ) - default_flag = ( - pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) - if "default_flag" in aligned.columns - else y_true.astype(int) - ) - - solution = optimize_portfolio_allocation( - loans=aligned, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=np.full(len(aligned), DEFAULT_LGD, dtype=float), - int_rates=int_rates, - total_budget=float(manifest.get("budget", 1_000_000.0)), - max_concentration=DEFAULT_MAX_CONCENTRATION, - max_portfolio_pd=float(policy["risk_tolerance"]), - robust=True, - uncertainty_aversion=float(policy["uncertainty_aversion"]), - min_budget_utilization=float(policy["min_budget_utilization"]), - pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), - pd_constraint_override=effective_pd, - time_limit=DEFAULT_TIME_LIMIT, - threads=max(1, int(threads)), - solver_backend=str(policy["solver_backend"]), - ) - alloc = solution_allocation_vector(solution, len(aligned)) - - exposure = alloc * loan_amounts - total_allocated = float(exposure.sum()) - weights = exposure / max(total_allocated, 1e-6) - funded = alloc > 0.01 - miscoverage = (y_true > pd_high).astype(float) - certificate = compute_funded_certificate_metrics( - weights, - outcomes=y_true, - pd_point=pd_point, - pd_high=pd_high, - pd_effective=effective_pd, - alpha=alpha, - risk_tolerance=float(policy["risk_tolerance"]), - pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), - ) - realized_return = np.where( - funded & (default_flag.astype(int) == 1), - exposure * (-DEFAULT_LGD), - np.where(funded, exposure * int_rates, 0.0), - ) - funded_df = aligned.loc[funded].copy() - funded_idx = np.flatnonzero(funded) - funded_df["allocation"] = alloc[funded] - funded_df["funded_exposure"] = exposure[funded] - funded_df["funded_weight"] = weights[funded] - funded_df["pd_point_alpha01"] = pd_point[funded] - funded_df["pd_low_alpha01"] = pd_low[funded] - funded_df["pd_high_alpha01"] = pd_high[funded] - funded_df["effective_pd"] = effective_pd[funded] - funded_df["miscoverage_alpha01"] = miscoverage[funded] - funded_df["realized_return"] = realized_return[funded] - funded_df["grade_bucket"] = ( - _grade_bucket(funded_df["grade"]) if "grade" in funded_df else "unknown" - ) - funded_df["source_row_position"] = funded_idx - - summary = ( - funded_df.groupby("grade_bucket", dropna=False) - .agg( - funded_rows=("allocation", "size"), - exposure=("funded_exposure", "sum"), - default_rate=("default_flag", "mean"), - weighted_default_rate=("funded_weight", lambda s: float(np.sum(s * y_true[s.index]))), - v_contribution=("funded_weight", lambda s: float(np.sum(s * miscoverage[s.index]))), - mean_pd_high_alpha01=("pd_high_alpha01", "mean"), - realized_return=("realized_return", "sum"), - ) - .reset_index() - ) - summary["exposure_share"] = summary["exposure"] / max(total_allocated, 1e-6) - order = pd.Categorical( - summary["grade_bucket"], ["A-B", "C", "D", "E-G", "unknown"], ordered=True - ) - summary = summary.assign(_order=order).sort_values("_order").drop(columns="_order") - - output_dir.mkdir(parents=True, exist_ok=True) - report_table_dir.mkdir(parents=True, exist_ok=True) - funded_path = output_dir / "pool93_body_allocation_alpha01.parquet" - summary_path = output_dir / "pool93_body_allocation_alpha01_grade_summary.parquet" - json_path = output_dir / "pool93_body_allocation_alpha01_audit.json" - csv_path = report_table_dir / "crpto_tableA36_pool93_body_funded_grade_audit.csv" - tex_path = report_table_dir / "crpto_tableA36_pool93_body_funded_grade_audit.tex" - funded_df.to_parquet(funded_path, index=False) - summary.to_parquet(summary_path, index=False) - summary.to_csv(csv_path, index=False, float_format="%.9f") - tex_path.write_text(_format_tex_table(summary), encoding="utf-8") - - payload = { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "frontier_path": str(frontier_path), - "role": role, - "frontier_row": row, - "manifest_path": str(_manifest_path(str(row["run_tag"]))), - "conformal_intervals_path": conformal_intervals_path, - "alpha": float(alpha), - "policy": policy, - "solver": { - "status": str(solution.get("solver_status", "unknown")), - "backend": str(solution.get("solver_backend", policy["solver_backend"])), - "threads": int(threads), - }, - "metrics": { - "n_funded": int(funded.sum()), - "total_allocated": round(total_allocated, 6), - "realized_return": round(float(realized_return.sum()), 6), - "Gamma_CP": round(certificate.gamma_cp, 6), - "Gamma_internalized": round(certificate.gamma_internalized, 6), - "Gamma_residual": round(certificate.gamma_residual, 6), - "V": round(certificate.weighted_miscoverage, 6), - "weighted_coverage_funded": round(certificate.weighted_coverage, 6), - "weighted_pd_true": round(certificate.weighted_outcome, 6), - "endpoint_budget": round(certificate.endpoint_budget, 9), - "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), - "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), - "markov_cap": round(certificate.markov_loss_cap, 9), - "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 6), - }, - "outputs": { - "funded_rows": str(funded_path), - "grade_summary_parquet": str(summary_path), - "grade_summary_csv": str(csv_path), - "grade_summary_tex": str(tex_path), - }, - } - json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - return payload - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--consolidated-tag", default=DEFAULT_CONSOLIDATED_TAG) - parser.add_argument("--frontier", default="") - parser.add_argument("--role", default=DEFAULT_BODY_ROLE) - parser.add_argument("--alpha", type=float, default=0.01) - parser.add_argument("--threads", type=int, default=1) - parser.add_argument("--solver-backend", default="") - parser.add_argument("--output-dir", default="") - parser.add_argument("--report-table-dir", default=str(ROOT / "reports/crpto/tables")) - args = parser.parse_args(argv) - - frontier_path = ( - Path(args.frontier) if args.frontier else _default_frontier_path(args.consolidated_tag) - ) - output_dir = ( - Path(args.output_dir) - if args.output_dir - else ROOT - / "data/processed/experiments/champion_reopen" - / args.consolidated_tag - / "portfolio" - ) - payload = build_audit( - frontier_path=frontier_path, - role=str(args.role), - alpha=float(args.alpha), - output_dir=output_dir, - report_table_dir=Path(args.report_table_dir), - threads=int(args.threads), - solver_backend=str(args.solver_backend).strip() or None, - ) - print(json.dumps(payload["metrics"], indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_ijds_claim_governance.py b/scripts/search/build_pool93_ijds_claim_governance.py deleted file mode 100644 index bfb51b4..0000000 --- a/scripts/search/build_pool93_ijds_claim_governance.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Build a claim-governance sidecar for pool93 IJDS local refinement.""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.search.run_pool93_ijds_local_refinement import ( # noqa: E402 - DEFAULT_ALPHA_GRID, - _claim_summary, -) -from src.utils.pipeline_runtime import atomic_write_json # noqa: E402 - -DEFAULT_RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-local-refine-stage1" -PAPER_FACING_KEY_RENAMES = { - "champion_return_reference": "declared_return_floor", - "champion_return_surplus": "return_floor_surplus", - "n_all_alpha_passers_above_champion": "n_all_alpha_passers_above_return_floor", - "best_gamma_cp_above_champion_claim": "best_gamma_cp_return_floor_claim", - "best_weighted_miscoverage_above_champion_claim": "best_weighted_miscoverage_return_floor_claim", - "min_gamma_cp_above_champion": "min_gamma_cp_above_return_floor", - "min_v_above_champion": "min_v_above_return_floor", -} - -PAPER_FACING_TEXT_RENAMES = { - "above-champion": "above-return-floor", - "above champion": "above return floor", - "champion_return_surplus": "return_floor_surplus", - "champion-level return": "declared-return-floor return", - "positive champion_return_surplus for replacement claims": ( - "nonnegative return_floor_surplus for declared-return-floor claims" - ), - "preserving champion-level return": "preserving the declared return floor", -} - - -def _load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def _default_paths(run_tag: str) -> dict[str, Path]: - data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" - model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" - return { - "leaderboard": data_dir / "pool93_ijds_local_refinement_leaderboard.parquet", - "bound_eval": data_dir / "pool93_ijds_local_refinement_bound_eval.parquet", - "manifest": model_dir / "pool93_ijds_local_refinement_manifest.json", - "status": model_dir / "runtime_status.json", - "output": model_dir / "pool93_ijds_claim_governance.json", - } - - -def _paper_facing_value(value: Any) -> Any: - if isinstance(value, dict): - return { - PAPER_FACING_KEY_RENAMES.get(str(key), str(key)): _paper_facing_value(item) - for key, item in value.items() - } - if isinstance(value, list): - return [_paper_facing_value(item) for item in value] - if isinstance(value, str): - text = value - for old, new in PAPER_FACING_TEXT_RENAMES.items(): - text = text.replace(old, new) - for old, new in PAPER_FACING_KEY_RENAMES.items(): - text = text.replace(old, new) - return text - return value - - -def _claim_hierarchy(summary: dict[str, Any], status: dict[str, Any]) -> dict[str, Any]: - state = str(status.get("state", "unknown")) - pct = float(status.get("pct_complete", 0.0) or 0.0) - complete = state == "completed" or pct >= 1.0 - return { - "status": "final" if complete else "partial_do_not_promote_yet", - "paper_body_candidate": "balanced_return_bound_claim", - "appendix_frontier_candidates": [ - "max_return_claim", - "best_gamma_cp_return_floor_claim", - "best_weighted_miscoverage_return_floor_claim", - ], - "do_not_claim": [ - "universal alpha robustness beyond the finite alpha grid", - "continuous-region robustness beyond evaluated policies", - "nominal funded-set alpha coverage when V(alpha) exceeds alpha", - "prospective live-selection validity from retrospective OOT selection", - ], - "promotion_gate": [ - "run state completed", - "selected claim passes all alpha levels in finite_grid_policy.alpha_grid", - "zero violation at alpha=0.01", - "nonnegative return_floor_surplus for declared-return-floor claims", - "manuscript labels max-return point as an economic frontier endpoint if it is used", - ], - "current_counts": { - "n_policies": summary.get("n_policies"), - "n_all_alpha_passers": summary.get("n_all_alpha_passers"), - "n_all_alpha_passers_above_return_floor": summary.get( - "n_all_alpha_passers_above_return_floor" - ), - }, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--run-tag", default=DEFAULT_RUN_TAG) - parser.add_argument("--leaderboard", default="") - parser.add_argument("--bound-eval", default="") - parser.add_argument("--manifest", default="") - parser.add_argument("--status", default="") - parser.add_argument("--output", default="") - args = parser.parse_args(argv) - - paths = _default_paths(str(args.run_tag)) - leaderboard_path = Path(args.leaderboard) if args.leaderboard else paths["leaderboard"] - bound_eval_path = Path(args.bound_eval) if args.bound_eval else paths["bound_eval"] - manifest_path = Path(args.manifest) if args.manifest else paths["manifest"] - status_path = Path(args.status) if args.status else paths["status"] - output_path = Path(args.output) if args.output else paths["output"] - - leaderboard = pd.read_parquet(leaderboard_path) - bound_eval = pd.read_parquet(bound_eval_path) - manifest = _load_json(manifest_path) - status = _load_json(status_path) - alpha_grid = [float(value) for value in manifest.get("alpha_grid", DEFAULT_ALPHA_GRID)] - - raw_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) - summary = _paper_facing_value(raw_summary) - paper_status = _paper_facing_value(status) - payload = { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": str(args.run_tag), - "source_paths": { - "leaderboard": str(leaderboard_path), - "bound_eval": str(bound_eval_path), - "manifest": str(manifest_path), - "status": str(status_path), - }, - "runtime_status": paper_status, - "claim_summary": summary, - "claim_hierarchy": _claim_hierarchy(summary, paper_status), - } - atomic_write_json(output_path, payload) - print(output_path) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_ijds_consolidated_frontier.py b/scripts/search/build_pool93_ijds_consolidated_frontier.py deleted file mode 100644 index 7b2d9ad..0000000 --- a/scripts/search/build_pool93_ijds_consolidated_frontier.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Build a consolidated IJDS frontier across pool93 refinement runs.""" - -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -from src.optimization.certificate_semantics import add_policy_aware_bound_columns - -ROOT = Path(__file__).resolve().parents[2] -DEFAULT_OUTPUT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" -DEFAULT_RUN_TAGS = [ - "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", - "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", - "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", - "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", -] -DEFAULT_CAPS = [0.30, 0.32, 0.345, 0.36, 0.45] -DEFAULT_BODY_MARKOV_THRESHOLD = 0.35 - - -def _leaderboard_path(run_tag: str) -> Path: - return ( - ROOT - / "data/processed/experiments/champion_reopen" - / run_tag - / "portfolio/pool93_ijds_local_refinement_leaderboard.parquet" - ) - - -def _bound_eval_path(run_tag: str) -> Path: - return ( - ROOT - / "data/processed/experiments/champion_reopen" - / run_tag - / "portfolio/pool93_ijds_local_refinement_bound_eval.parquet" - ) - - -def _output_path(output_tag: str) -> Path: - return ( - ROOT - / "models/experiments/champion_reopen" - / output_tag - / "portfolio/pool93_ijds_consolidated_frontier.json" - ) - - -def _short_run_label(run_tag: str) -> str: - if run_tag.endswith("__pool93__ijds-claim-expanded-refine"): - return "expanded" - if run_tag.endswith("__pool93__ijds-claim-micro-refine"): - return "micro" - if run_tag.endswith("__pool93__ijds-claim-micro-ext"): - return "micro_ext" - if run_tag.endswith("__pool93__ijds-claim-bound-closure"): - return "bound_closure" - if run_tag.endswith("__pool93__ijds-claim-bound-floor-closure"): - return "bound_floor" - if run_tag.endswith("__pool93__ijds-claim-bound-terminal"): - return "bound_terminal" - return run_tag - - -def _row_payload(row: pd.Series, role: str) -> dict[str, Any]: - return { - "role": role, - "run_label": str(row["run_label"]), - "run_tag": str(row["run_tag"]), - "local_candidate_id": int(row["local_candidate_id"]), - "family": str(row["local_family"]), - "anchor_rank": int(row["anchor_rank"]), - "source_reason": str(row["source_reason"]), - "risk_tolerance": round(float(row["risk_tolerance"]), 6), - "policy_mode": str(row["policy_mode"]), - "gamma": round(float(row["gamma"]), 6), - "delta_cap_quantile": round(float(row["delta_cap_quantile"]), 6), - "tail_focus_quantile": round(float(row["tail_focus_quantile"]), 6), - "uncertainty_aversion": round(float(row["uncertainty_aversion"]), 6), - "return": round(float(row["alpha01_realized_total_return"]), 6), - "return_floor_surplus": round(float(row["return_floor_surplus"]), 6), - "Gamma_CP": round(float(row["alpha01_gamma_cp"]), 6), - "Gamma_internalized": round(float(row["alpha01_gamma_internalized"]), 6), - "Gamma_residual": round(float(row["alpha01_gamma_residual"]), 6), - "V": round(float(row["alpha01_weighted_miscoverage_V"]), 6), - "endpoint_budget": round(float(row["alpha01_endpoint_budget"]), 9), - "endpoint_budget_upper": round(float(row["alpha01_endpoint_budget_upper"]), 9), - "Markov_threshold": round(float(row["alpha01_markov_loss_threshold"]), 9), - "Markov_cap": round(float(row["alpha01_markov_loss_cap"]), 9), - "alpha_pass": f"{int(row['alpha_exact_pass_count'])}/{int(row['alpha_exact_check_count'])}", - "n_funded_mean": round(float(row["n_funded_mean"]), 3), - "semantic_policy_key": str(row["semantic_policy_key"]), - } - - -def _eligible(df: pd.DataFrame) -> pd.DataFrame: - return df[ - (df["alpha_exact_pass_count"] == df["alpha_exact_check_count"]) - & (df["return_floor_surplus"] >= 0) - ].copy() - - -def _score_body_candidate(eligible: pd.DataFrame) -> pd.Series: - work = eligible.copy() - score_specs = { - "return_score": ("alpha01_realized_total_return", False), - "bound_score": ("alpha01_markov_loss_threshold", True), - "v_score": ("alpha01_weighted_miscoverage_V", True), - } - for score_col, (metric_col, inverse) in score_specs.items(): - lo = float(work[metric_col].min()) - hi = float(work[metric_col].max()) - if hi == lo: - work[score_col] = 1.0 - elif inverse: - work[score_col] = (hi - work[metric_col]) / (hi - lo) - else: - work[score_col] = (work[metric_col] - lo) / (hi - lo) - work["ijds_balanced_score"] = ( - 0.40 * work["return_score"] + 0.40 * work["bound_score"] + 0.20 * work["v_score"] - ) - return work.sort_values( - [ - "ijds_balanced_score", - "alpha01_realized_total_return", - "alpha01_markov_loss_threshold", - ], - ascending=[False, False, True], - ).iloc[0] - - -def _body_candidate(eligible: pd.DataFrame, *, markov_threshold: float) -> pd.Series: - """Select the paper-body point from the exact finite-grid frontier. - - The body point is intentionally not the global max-return endpoint and not - the minimum-bound endpoint. It is the highest-return policy below a declared - exact Markov-loss threshold, which matches the paper-facing return-bound - claim and remains valid for linear, capped and tail-focused policies. - """ - under_threshold = _best_under_threshold(eligible, markov_threshold) - if under_threshold is not None: - return under_threshold - return _score_body_candidate(eligible) - - -def _best_under_threshold(eligible: pd.DataFrame, threshold: float) -> pd.Series | None: - candidates = eligible[eligible["alpha01_markov_loss_threshold"] <= threshold] - if candidates.empty: - return None - return candidates.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], - ascending=[False, True], - ).iloc[0] - - -def _append_row(rows: list[dict[str, Any]], row: pd.Series | None, role: str) -> None: - if row is None: - return - payload = _row_payload(row, role) - key = (payload["role"], payload["semantic_policy_key"]) - existing = {(item["role"], item["semantic_policy_key"]) for item in rows} - if key not in existing: - rows.append(payload) - - -def _load_leaderboards(run_tags: list[str]) -> pd.DataFrame: - frames: list[pd.DataFrame] = [] - for run_tag in run_tags: - path = _leaderboard_path(run_tag) - bound_path = _bound_eval_path(run_tag) - if not path.exists(): - raise FileNotFoundError(path) - if not bound_path.exists(): - raise FileNotFoundError(bound_path) - frame = pd.read_parquet(path) - bound_eval = pd.read_parquet(bound_path) - alpha01 = add_policy_aware_bound_columns( - bound_eval[np.isclose(bound_eval["alpha"], 0.01)].copy() - ) - alpha01_columns = { - "gamma_internalized": "alpha01_gamma_internalized", - "gamma_residual": "alpha01_gamma_residual", - "weighted_pd_constraint_used": "alpha01_weighted_pd_constraint_used", - "weighted_pd_high": "alpha01_weighted_pd_high", - "weighted_pd_point": "alpha01_weighted_pd_point", - "endpoint_budget": "alpha01_endpoint_budget", - "endpoint_budget_upper": "alpha01_endpoint_budget_upper", - "markov_loss_threshold": "alpha01_markov_loss_threshold", - "markov_loss_cap": "alpha01_markov_loss_cap", - } - alpha01 = alpha01[["local_candidate_id", "semantic_policy_key", *alpha01_columns]].rename( - columns=alpha01_columns - ) - legacy_columns = { - "alpha01_endpoint_budget_upper": "legacy_alpha01_endpoint_budget_upper", - "alpha01_markov_loss_cap": "legacy_alpha01_markov_loss_cap", - } - frame = frame.rename( - columns={ - source: target - for source, target in legacy_columns.items() - if source in frame.columns - } - ) - stale_columns = [ - column - for column in alpha01_columns.values() - if column in frame.columns and column not in legacy_columns.values() - ] - if stale_columns: - frame = frame.drop(columns=stale_columns) - frame = frame.merge( - alpha01, - on=["local_candidate_id", "semantic_policy_key"], - how="left", - validate="one_to_one", - ) - if frame["alpha01_markov_loss_threshold"].isna().any(): - raise ValueError(f"Missing alpha=0.01 certificate rows for {run_tag}") - frame["run_tag"] = run_tag - frame["run_label"] = _short_run_label(run_tag) - frames.append(frame) - return pd.concat(frames, ignore_index=True) - - -def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[str, Any]: - raw = _load_leaderboards(run_tags) - deduped = ( - raw.sort_values( - [ - "semantic_policy_key", - "alpha_exact_pass_count", - "alpha01_realized_total_return", - "alpha01_markov_loss_threshold", - ], - ascending=[True, False, False, True], - ) - .drop_duplicates("semantic_policy_key", keep="first") - .reset_index(drop=True) - ) - eligible = _eligible(deduped) - - body = _body_candidate(eligible, markov_threshold=DEFAULT_BODY_MARKOV_THRESHOLD) - semantics_audit: dict[str, Any] = { - "status": "not_available", - "selection_metric": "alpha01_markov_loss_threshold", - } - if "legacy_alpha01_markov_loss_cap" in deduped.columns: - legacy = pd.to_numeric(deduped["legacy_alpha01_markov_loss_cap"], errors="coerce") - exact = pd.to_numeric(deduped["alpha01_markov_loss_threshold"], errors="coerce") - delta = exact - legacy - tolerance = 1e-5 - legacy_eligible = eligible[eligible["legacy_alpha01_markov_loss_cap"] <= 0.35] - legacy_body = legacy_eligible.sort_values( - ["alpha01_realized_total_return", "legacy_alpha01_markov_loss_cap"], - ascending=[False, True], - ).iloc[0] - semantics_audit = { - "status": "corrected_from_existing_exact_bound_evaluations", - "selection_metric": "alpha01_markov_loss_threshold", - "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", - "material_difference_tolerance": tolerance, - "materially_changed_policies": int((delta.abs() > tolerance).sum()), - "materially_understated_policies": int((delta > tolerance).sum()), - "maximum_legacy_understatement": round(float(delta.max()), 9), - "legacy_under_0_50_excluded_by_exact_threshold": int( - ((legacy <= 0.50) & (exact > 0.50)).sum() - ), - "body_selection_unchanged": bool( - str(legacy_body["semantic_policy_key"]) == str(body["semantic_policy_key"]) - ), - "affected_policy_modes": sorted( - deduped.loc[delta > tolerance, "policy_mode"].astype(str).unique().tolist() - ), - } - rows: list[dict[str, Any]] = [] - _append_row( - rows, - eligible.sort_values("alpha01_markov_loss_threshold").iloc[0], - "minimum Markov-threshold endpoint", - ) - _append_row(rows, body, "body/default balanced return-bound point") - _append_row( - rows, - eligible.sort_values( - ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], - ascending=[True, False], - ).iloc[0], - "lowest realized V return-bound point", - ) - _append_row( - rows, - eligible.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], - ascending=[False, True], - ).iloc[0], - "max-return economic endpoint", - ) - for cap in caps: - _append_row( - rows, - _best_under_threshold(eligible, cap), - f"highest return under threshold<={cap:g}", - ) - - by_run = [] - for run_tag, run_df in raw.groupby("run_tag", sort=False): - by_run.append( - { - "run_label": _short_run_label(str(run_tag)), - "run_tag": str(run_tag), - "rows": int(len(run_df)), - "all_alpha_passers": int( - (run_df["alpha_exact_pass_count"] == run_df["alpha_exact_check_count"]).sum() - ), - "all_alpha_pass_rate": round( - float( - ( - run_df["alpha_exact_pass_count"] == run_df["alpha_exact_check_count"] - ).mean() - ), - 9, - ), - "best_return": round(float(run_df["alpha01_realized_total_return"].max()), 6), - "min_markov_threshold": round( - float(run_df["alpha01_markov_loss_threshold"].min()), 9 - ), - } - ) - - return { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "source_run_tags": run_tags, - "selection_rule": { - "eligible": "all-alpha pass and nonnegative return_floor_surplus", - "dedupe_key": "semantic_policy_key", - "dedupe_semantics": ( - "duplicate semantic policies across refinement runs have identical metrics; " - "one representative row is retained for the consolidated table" - ), - "body_selection": ( - f"highest realized return among eligible finite-grid policies with " - f"exact Markov_threshold <= {DEFAULT_BODY_MARKOV_THRESHOLD:g}; " - "falls back to the legacy " - "balanced normalized return/bound/V score only if no eligible policy " - "exists under that declared threshold" - ), - "bound_semantics": ( - "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); " - "Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). " - "The exact threshold drives selection." - ), - "caps": caps, - "role_semantics": "finite-grid frontier roles, not continuous optima", - }, - "counts": { - "raw_rows": int(len(raw)), - "deduped_semantic_policies": int(len(deduped)), - "duplicate_rows_removed": int(len(raw) - len(deduped)), - "eligible_all_alpha_return_floor_policies": int(len(eligible)), - "nonpass_or_below_floor_policies": int(len(deduped) - len(eligible)), - }, - "certificate_semantics_audit": semantics_audit, - "by_run": by_run, - "rows": rows, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--output-tag", default=DEFAULT_OUTPUT_TAG) - parser.add_argument("--output", default="") - parser.add_argument("--run-tag", action="append", dest="run_tags", default=[]) - parser.add_argument("--caps", default=",".join(str(cap) for cap in DEFAULT_CAPS)) - args = parser.parse_args(argv) - - run_tags = [str(tag) for tag in args.run_tags] or DEFAULT_RUN_TAGS - caps = [float(part.strip()) for part in str(args.caps).split(",") if part.strip()] - payload = build_consolidated_frontier(run_tags, caps) - output = Path(args.output) if args.output else _output_path(str(args.output_tag)) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(payload, indent=2), encoding="utf-8") - print(output) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_ijds_consolidated_governance.py b/scripts/search/build_pool93_ijds_consolidated_governance.py deleted file mode 100644 index 2d06995..0000000 --- a/scripts/search/build_pool93_ijds_consolidated_governance.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Build the paper-facing governance sidecar for the consolidated pool93 frontier.""" - -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[2] -DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" -DEFAULT_BODY_ROLE = "body/default balanced return-bound point" - - -def _default_paths(consolidated_tag: str) -> tuple[Path, Path]: - portfolio_dir = ROOT / "models/experiments/champion_reopen" / consolidated_tag / "portfolio" - return ( - portfolio_dir / "pool93_ijds_consolidated_frontier.json", - portfolio_dir / "pool93_ijds_consolidated_governance.json", - ) - - -def _find_role(rows: list[dict[str, Any]], role: str) -> dict[str, Any]: - for row in rows: - if str(row.get("role")) == role: - return dict(row) - raise ValueError(f"Role {role!r} not found in consolidated frontier.") - - -def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: - frontier = json.loads(frontier_path.read_text(encoding="utf-8")) - rows = list(frontier.get("rows", [])) - body = _find_role(rows, body_role) - strict_threshold = _find_role(rows, "highest return under threshold<=0.345") - low_threshold = _find_role(rows, "minimum Markov-threshold endpoint") - max_return = _find_role(rows, "max-return economic endpoint") - return { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "source_frontier_path": str(frontier_path), - "source_run_tags": frontier.get("source_run_tags", []), - "counts": frontier.get("counts", {}), - "selection_rule": frontier.get("selection_rule", {}), - "certificate_semantics_audit": frontier.get("certificate_semantics_audit", {}), - "claim_hierarchy": { - "status": "final", - "paper_body_candidate": body_role, - "paper_body_claim": ( - "The selected pool93 body point is the highest-return eligible " - "finite-grid policy under the declared exact Markov-threshold ceiling " - "and passes all eight predeclared alpha checks." - ), - "appendix_frontier_candidates": [ - "minimum Markov-threshold endpoint", - "highest return under threshold<=0.345", - "highest return under threshold<=0.36", - "max-return economic endpoint", - ], - "do_not_claim": [ - "continuous-region optimality beyond the evaluated finite policy grid", - "nominal funded-set alpha coverage when V(alpha) exceeds alpha", - "prospective live-selection validity from retrospective OOT selection", - "pool93-specific row-level tail/CVaR dominance unless regenerated from the promoted allocation", - ], - "promotion_gate": [ - "consolidated frontier generated from completed exact runs", - "semantic-policy deduplication applied", - "policy-aware endpoint decomposition applied to every alpha=0.01 row", - "selected body point passes 8/8 alpha checks", - "zero realized risk-tolerance excess at alpha=0.01", - "return exceeds declared return floor", - "A35 frontier, A36 funded-set grade audit, and A40 matched baseline are regenerated from retained artifacts", - ], - }, - "selected_candidates": { - "paper_body": body, - "strict_threshold_leq_0_345": strict_threshold, - "minimum_markov_threshold_endpoint": low_threshold, - "max_return_economic_endpoint": max_return, - }, - "paper_artifacts": { - "frontier_table_csv": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", - "frontier_table_tex": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", - "funded_grade_audit_csv": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", - "funded_grade_audit_tex": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", - "point_baseline_csv": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", - "point_baseline_tex": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", - "point_baseline_audit": ( - "models/experiments/champion_reopen/" - "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" - "portfolio/pool93_point_pd_baseline_audit.json" - ), - }, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--consolidated-tag", default=DEFAULT_CONSOLIDATED_TAG) - parser.add_argument("--frontier", default="") - parser.add_argument("--output", default="") - parser.add_argument("--body-role", default=DEFAULT_BODY_ROLE) - args = parser.parse_args(argv) - - default_frontier, default_output = _default_paths(str(args.consolidated_tag)) - frontier_path = Path(args.frontier) if args.frontier else default_frontier - output_path = Path(args.output) if args.output else default_output - payload = build_governance(frontier_path, body_role=str(args.body_role)) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - print(output_path) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_ijds_frontier_claim_table.py b/scripts/search/build_pool93_ijds_frontier_claim_table.py deleted file mode 100644 index 195a0a8..0000000 --- a/scripts/search/build_pool93_ijds_frontier_claim_table.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Build a compact IJDS-facing frontier table from a pool93 refinement run.""" - -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, cast - -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] - - -def _default_paths(run_tag: str) -> tuple[Path, Path, Path]: - data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" - model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" - return ( - data_dir / "pool93_ijds_local_refinement_leaderboard.parquet", - model_dir / "pool93_ijds_local_refinement_claim_summary.json", - model_dir / "pool93_ijds_frontier_claim_table.json", - ) - - -def _row_payload(row: pd.Series, role: str) -> dict[str, object]: - return { - "role": role, - "local_candidate_id": int(row["local_candidate_id"]), - "family": str(row["local_family"]), - "tau": float(row["risk_tolerance"]), - "policy": str(row["policy_mode"]), - "gamma": float(row["gamma"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - "return": round(float(row["alpha01_realized_total_return"]), 6), - "Gamma_CP": round(float(row["alpha01_gamma_cp"]), 6), - "V": round(float(row["alpha01_weighted_miscoverage_V"]), 6), - "endpoint_budget_upper": round(float(row["alpha01_endpoint_budget_upper"]), 9), - "Markov_cap": round(float(row["alpha01_markov_loss_cap"]), 9), - "alpha_pass": f"{int(row['alpha_exact_pass_count'])}/{int(row['alpha_exact_check_count'])}", - "n_funded_mean": float(row["n_funded_mean"]), - } - - -def _first_or_none(df: pd.DataFrame) -> pd.Series | None: - if df.empty: - return None - return df.iloc[0] - - -def _append_unique(rows: list[dict[str, object]], row: pd.Series | None, role: str) -> None: - if row is None: - return - payload = _row_payload(row, role) - key = (payload["local_candidate_id"], payload["role"]) - if key in {(item["local_candidate_id"], item["role"]) for item in rows}: - return - rows.append(payload) - - -def build_frontier_table( - leaderboard: pd.DataFrame, - claim_summary: dict[str, object], - *, - caps: list[float], -) -> dict[str, object]: - eligible = leaderboard[ - (leaderboard["alpha_exact_pass_count"] == leaderboard["alpha_exact_check_count"]) - & (leaderboard["return_floor_surplus"] >= 0) - ].copy() - rows: list[dict[str, object]] = [] - - summary_candidates = { - "bound-tight endpoint": "best_gamma_cp_return_floor_claim", - "body/default balanced return-bound point": "balanced_return_bound_claim", - "lowest realized V return-bound point": "best_weighted_miscoverage_return_floor_claim", - "max-return economic endpoint": "max_return_claim", - } - for role, key in summary_candidates.items(): - candidate = claim_summary.get(key, {}) - if not isinstance(candidate, dict) or "local_candidate_id" not in candidate: - continue - candidate = cast(dict[str, Any], candidate) - candidate_id = int(candidate["local_candidate_id"]) - _append_unique( - rows, - _first_or_none(leaderboard[leaderboard["local_candidate_id"] == candidate_id]), - role, - ) - - for cap in caps: - cap_df = eligible[eligible["alpha01_markov_loss_cap"] <= cap].sort_values( - "alpha01_realized_total_return", - ascending=False, - ) - _append_unique( - rows, - _first_or_none(cap_df), - f"highest return under cap<={cap:g}", - ) - - return { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "selection_rule": { - "eligible": "all-alpha pass and nonnegative return_floor_surplus", - "caps": caps, - "role_semantics": "finite-grid frontier roles, not continuous optima", - }, - "rows": rows, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--run-tag", required=True) - parser.add_argument("--leaderboard", default="") - parser.add_argument("--claim-summary", default="") - parser.add_argument("--output", default="") - parser.add_argument("--caps", default="0.33,0.345,0.36,0.50") - args = parser.parse_args(argv) - - leaderboard_path, claim_summary_path, output_path = _default_paths(str(args.run_tag)) - if args.leaderboard: - leaderboard_path = Path(args.leaderboard) - if args.claim_summary: - claim_summary_path = Path(args.claim_summary) - if args.output: - output_path = Path(args.output) - - caps = [float(part.strip()) for part in str(args.caps).split(",") if part.strip()] - leaderboard = pd.read_parquet(leaderboard_path) - claim_summary = json.loads(claim_summary_path.read_text(encoding="utf-8")) - payload = build_frontier_table(leaderboard, claim_summary, caps=caps) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - print(output_path) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_point_baseline_audit.py b/scripts/search/build_pool93_point_baseline_audit.py deleted file mode 100644 index 78ce7ba..0000000 --- a/scripts/search/build_pool93_point_baseline_audit.py +++ /dev/null @@ -1,372 +0,0 @@ -"""Build a matched point-PD baseline for the selected pool93 decision. - -The audit holds the candidate universe, budget, concentration cap and risk -tolerance fixed. It changes only the decision uncertainty treatment: the -baseline constrains calibrated point PD, while the selected CRPTO allocation -uses its declared effective-PD policy. Outputs are isolated from the frozen -champion and contain no policy search. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from dataclasses import asdict -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 -from scripts.search.build_pool93_body_allocation_audit import ( # noqa: E402 - _load_role_row, - _manifest_path, - _policy_from_row, -) -from scripts.validate_alpha_gamma_bound import ( # noqa: E402 - DEFAULT_LGD, - DEFAULT_MAX_CONCENTRATION, - DEFAULT_TIME_LIMIT, - _compute_intervals_at_alpha, - _load_aligned_dataset, -) -from src.optimization.certificate_semantics import ( # noqa: E402 - FundedCertificateMetrics, - compute_funded_certificate_metrics, -) -from src.optimization.portfolio_model import ( # noqa: E402 - optimize_portfolio_allocation, - solution_allocation_vector, -) -from src.utils.script_helpers import resolve_repo_artifact_path, write_table # noqa: E402 - -DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" -DEFAULT_ROLE = "body/default balanced return-bound point" -DEFAULT_FRONTIER = ( - ROOT - / "models/experiments/champion_reopen" - / DEFAULT_TAG - / "portfolio/pool93_ijds_consolidated_frontier.json" -) -DEFAULT_BODY_ALLOCATION = ( - ROOT - / "data/processed/experiments/champion_reopen" - / "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" - / "portfolio/pool93_body_allocation_alpha01.parquet" -) -DEFAULT_OUTPUT_DIR = ROOT / "models/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" -DEFAULT_DATA_OUTPUT_DIR = ( - ROOT / "data/processed/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" -) -DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG - - -def _realized_return( - allocation: np.ndarray, - loan_amounts: np.ndarray, - int_rates: np.ndarray, - default_flag: np.ndarray, -) -> float: - exposure = allocation * loan_amounts - funded = allocation > 0.01 - contributions = np.where( - funded & (default_flag.astype(int) == 1), - -DEFAULT_LGD * exposure, - np.where(funded, int_rates * exposure, 0.0), - ) - return float(contributions.sum()) - - -def _economic_metrics( - *, - allocation: np.ndarray, - loan_amounts: np.ndarray, - int_rates: np.ndarray, - pd_point: np.ndarray, -) -> dict[str, float]: - exposure = allocation * loan_amounts - return { - "total_allocated": float(exposure.sum()), - "expected_return_gross": float(np.sum(exposure * int_rates)), - "expected_loss_point": float(np.sum(exposure * pd_point * DEFAULT_LGD)), - "expected_return_net_point": float( - np.sum(exposure * int_rates) - np.sum(exposure * pd_point * DEFAULT_LGD) - ), - } - - -def _comparison_table( - point: dict[str, Any], - selected: dict[str, Any], -) -> pd.DataFrame: - point_return = float(point["realized_return"]) - rows = [] - for label, payload in ( - ("Point-PD two-stage LP", point), - ("Selected CRPTO", selected), - ): - rows.append( - { - "policy": label, - "realized_return": float(payload["realized_return"]), - "return_cost_vs_point_pct": 100.0 - * (point_return - float(payload["realized_return"])) - / point_return, - "n_funded": int(payload["certificate"]["n_funded"]), - "weighted_default_rate": float(payload["certificate"]["weighted_outcome"]), - "V_alpha01": float(payload["certificate"]["weighted_miscoverage"]), - "Gamma_CP_alpha01": float(payload["certificate"]["gamma_cp"]), - "endpoint_budget_alpha01": float(payload["certificate"]["endpoint_budget"]), - "Markov_threshold_alpha01": float(payload["certificate"]["markov_loss_threshold"]), - "expected_return_net_point": float(payload["expected_return_net_point"]), - } - ) - return pd.DataFrame(rows) - - -def _format_comparison_tex(table: pd.DataFrame) -> str: - """Render the compact comparison used in the paper and supplement.""" - lines = [ - "\\begin{tabular}{lrrrrr}", - "\\toprule", - ( - "Policy & Realized return & Weighted default & " - "$\\Gamma_{\\mathrm{CP}}$ & $B_u$ & Markov threshold \\\\" - ), - "\\midrule", - ] - for row in table.to_dict("records"): - lines.append( - f"{row['policy']} & \\${float(row['realized_return']):,.2f} & " - f"{float(row['weighted_default_rate']):.6f} & " - f"{float(row['Gamma_CP_alpha01']):.6f} & " - f"{float(row['endpoint_budget_alpha01']):.6f} & " - f"{float(row['Markov_threshold_alpha01']):.6f} \\\\" - ) - lines.extend(["\\bottomrule", "\\end{tabular}", ""]) - return "\n".join(lines) - - -def _certificate_payload( - certificate: FundedCertificateMetrics, - *, - realized_return: float, - economic: dict[str, float], - solver_status: str, -) -> dict[str, Any]: - return { - "solver_status": solver_status, - "realized_return": realized_return, - **economic, - "certificate": asdict(certificate), - } - - -def build_audit( - *, - frontier_path: Path, - body_allocation_path: Path, - role: str, - alpha: float, - output_dir: Path, - data_output_dir: Path, - table_dir: Path, - threads: int, -) -> dict[str, Any]: - row = _load_role_row(frontier_path, role) - policy = _policy_from_row(row) - manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) - interval_path = resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) - aligned = _load_aligned_dataset( - conformal_intervals_path=str(interval_path), - max_candidates=int(manifest.get("max_candidates", 0) or 0), - random_state=int(manifest.get("random_state", 42)), - ) - pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) - y_true = pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0.0).to_numpy(float) - default_flag = ( - pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) - ) - loan_amounts = pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(float) - int_rates = _parse_percent_series(aligned["int_rate"]) - risk_tolerance = float(policy["risk_tolerance"]) - solution = optimize_portfolio_allocation( - loans=aligned, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=np.full(len(aligned), DEFAULT_LGD, dtype=float), - int_rates=int_rates, - total_budget=float(manifest.get("budget", 1_000_000.0)), - max_concentration=DEFAULT_MAX_CONCENTRATION, - max_portfolio_pd=risk_tolerance, - robust=False, - uncertainty_aversion=0.0, - min_budget_utilization=float(policy["min_budget_utilization"]), - pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), - pd_constraint_override=pd_point, - time_limit=DEFAULT_TIME_LIMIT, - threads=max(1, int(threads)), - solver_backend=str(policy["solver_backend"]), - ) - point_allocation = solution_allocation_vector(solution, len(aligned)) - point_exposure = point_allocation * loan_amounts - point_weights = point_exposure / max(float(point_exposure.sum()), 1e-12) - point_certificate = compute_funded_certificate_metrics( - point_weights, - outcomes=y_true, - pd_point=pd_point, - pd_high=pd_high, - pd_effective=pd_point, - alpha=alpha, - risk_tolerance=risk_tolerance, - pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), - ) - point_payload = _certificate_payload( - point_certificate, - realized_return=_realized_return( - point_allocation, - loan_amounts, - int_rates, - default_flag, - ), - economic=_economic_metrics( - allocation=point_allocation, - loan_amounts=loan_amounts, - int_rates=int_rates, - pd_point=pd_point, - ), - solver_status=str(solution.get("solver_status", "unknown")), - ) - - funded = pd.read_parquet(body_allocation_path) - selected_weights = funded["funded_weight"].to_numpy(float) - selected_point = funded["pd_point_alpha01"].to_numpy(float) - selected_high = funded["pd_high_alpha01"].to_numpy(float) - selected_effective = funded["effective_pd"].to_numpy(float) - selected_outcomes = funded["default_flag"].to_numpy(float) - selected_exposure = funded["funded_exposure"].to_numpy(float) - selected_rates = _parse_percent_series(funded["int_rate"]) - selected_certificate = compute_funded_certificate_metrics( - selected_weights, - outcomes=selected_outcomes, - pd_point=selected_point, - pd_high=selected_high, - pd_effective=selected_effective, - alpha=alpha, - risk_tolerance=risk_tolerance, - ) - selected_payload = _certificate_payload( - selected_certificate, - realized_return=float(funded["realized_return"].sum()), - economic={ - "total_allocated": float(selected_exposure.sum()), - "expected_return_gross": float(np.sum(selected_exposure * selected_rates)), - "expected_loss_point": float(np.sum(selected_exposure * selected_point * DEFAULT_LGD)), - "expected_return_net_point": float( - np.sum(selected_exposure * selected_rates) - - np.sum(selected_exposure * selected_point * DEFAULT_LGD) - ), - }, - solver_status="frozen_selected_allocation", - ) - - table = _comparison_table(point_payload, selected_payload) - table_paths = write_table( - "crpto_tableA40_pool93_point_baseline", - table, - table_dir=table_dir, - root=ROOT, - float_precision=6, - ) - table_paths[1].write_text( - _format_comparison_tex(table), - encoding="utf-8", - newline="", - ) - output_dir.mkdir(parents=True, exist_ok=True) - data_output_dir.mkdir(parents=True, exist_ok=True) - point_rows = aligned.loc[point_allocation > 0.01].copy() - point_rows["allocation"] = point_allocation[point_allocation > 0.01] - point_rows["funded_exposure"] = point_exposure[point_allocation > 0.01] - point_rows["funded_weight"] = point_weights[point_allocation > 0.01] - point_rows_path = data_output_dir / "pool93_point_pd_baseline_alpha01.parquet" - point_rows.to_parquet(point_rows_path, index=False) - - return_cost = float(point_payload["realized_return"]) - float( - selected_payload["realized_return"] - ) - payload = { - "schema_version": "2026-07-09.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": DEFAULT_TAG, - "comparison": "matched point-PD two-stage LP versus selected CRPTO", - "fixed_design": { - "candidate_universe": len(aligned), - "budget": float(manifest.get("budget", 1_000_000.0)), - "risk_tolerance": risk_tolerance, - "max_concentration": DEFAULT_MAX_CONCENTRATION, - "alpha": alpha, - }, - "point_pd_baseline": point_payload, - "selected_crpto": selected_payload, - "contrasts": { - "realized_return_cost": return_cost, - "realized_return_cost_pct": 100.0 - * return_cost - / float(point_payload["realized_return"]), - "weighted_default_rate_reduction": point_certificate.weighted_outcome - - selected_certificate.weighted_outcome, - "weighted_miscoverage_reduction": point_certificate.weighted_miscoverage - - selected_certificate.weighted_miscoverage, - "markov_threshold_reduction": point_certificate.markov_loss_threshold - - selected_certificate.markov_loss_threshold, - }, - "claim_boundary": ( - "Frozen OOT matched-policy audit; it quantifies a return-risk trade-off and " - "does not establish causal, prospective or universal dominance." - ), - "outputs": { - "point_funded_rows": str(point_rows_path), - "table_csv": str(table_paths[0]), - "table_tex": str(table_paths[1]), - }, - } - output_path = output_dir / "pool93_point_pd_baseline_audit.json" - output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8", newline="") - return payload - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) - parser.add_argument("--body-allocation", default=str(DEFAULT_BODY_ALLOCATION)) - parser.add_argument("--role", default=DEFAULT_ROLE) - parser.add_argument("--alpha", type=float, default=0.01) - parser.add_argument("--threads", type=int, default=1) - parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) - parser.add_argument("--data-output-dir", default=str(DEFAULT_DATA_OUTPUT_DIR)) - parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) - args = parser.parse_args(argv) - payload = build_audit( - frontier_path=Path(args.frontier).resolve(), - body_allocation_path=Path(args.body_allocation).resolve(), - role=str(args.role), - alpha=float(args.alpha), - output_dir=Path(args.output_dir).resolve(), - data_output_dir=Path(args.data_output_dir).resolve(), - table_dir=Path(args.table_dir).resolve(), - threads=max(1, int(args.threads)), - ) - print(json.dumps(payload["contrasts"], indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_pool93_tail_risk_audit.py b/scripts/search/build_pool93_tail_risk_audit.py deleted file mode 100644 index b774119..0000000 --- a/scripts/search/build_pool93_tail_risk_audit.py +++ /dev/null @@ -1,479 +0,0 @@ -"""Regenerate pool93 body-point tail-risk and cluster-bound audit tables. - -This sidecar reads the selected pool93 body allocation, not the legacy robust -region. It closes the paper-facing caveat that CVaR/OCE and cluster-bound -diagnostics must be recomputed from the promoted row-level funded set before -being cited as pool93-specific evidence. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 -from src.optimization.tail_satisficing_objective import ( # noqa: E402 - entropic_oce, - funded_loss_rate, - weighted_cvar, - weighted_mean, -) - -DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" -DEFAULT_ALLOCATION_PATH = ( - ROOT - / "data/processed/experiments/champion_reopen" - / DEFAULT_TAG - / "portfolio/pool93_body_allocation_alpha01.parquet" -) -DEFAULT_BODY_AUDIT_PATH = ( - ROOT - / "data/processed/experiments/champion_reopen" - / DEFAULT_TAG - / "portfolio/pool93_body_allocation_alpha01_audit.json" -) -DEFAULT_OUTPUT_DIR = ROOT / "models/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" -DEFAULT_TABLE_DIR = ROOT / "reports/crpto/tables" - -TABLE_A37_NAME = "crpto_tableA37_pool93_body_tail_risk" -TABLE_A38_NAME = "crpto_tableA38_pool93_body_cluster_bound_audit" -TABLE_A39_NAME = "crpto_tableA39_pool93_body_bootstrap_metrics" -OCE_THETA = 5.0 -CVAR_LEVELS = (0.90, 0.95, 0.99) -BOOTSTRAP_DRAWS = 5000 -BOOTSTRAP_SEED = 20260702 - - -def _read_json(path: Path) -> dict[str, Any]: - if not path.is_file(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def _write_table(table_dir: Path, name: str, frame: pd.DataFrame) -> list[str]: - table_dir.mkdir(parents=True, exist_ok=True) - csv_path = table_dir / f"{name}.csv" - tex_path = table_dir / f"{name}.tex" - csv_path.write_text( - frame.to_csv(index=False, lineterminator="\n", float_format="%.9f"), - encoding="utf-8", - newline="", - ) - tex_path.write_text( - frame.to_latex(index=False, escape=True, float_format=lambda value: f"{value:.6f}"), - encoding="utf-8", - newline="", - ) - return [str(csv_path), str(tex_path)] - - -def _load_allocation(path: Path) -> pd.DataFrame: - if not path.is_file(): - raise FileNotFoundError(f"pool93 allocation not found: {path}") - funded = pd.read_parquet(path) - required = { - "funded_exposure", - "funded_weight", - "default_flag", - "int_rate", - "pd_point_alpha01", - "pd_high_alpha01", - "miscoverage_alpha01", - } - missing = sorted(required.difference(funded.columns)) - if missing: - raise ValueError(f"pool93 allocation missing columns: {missing}") - return funded.copy() - - -def _prepare_allocation(funded: pd.DataFrame) -> pd.DataFrame: - prepared = funded.copy() - prepared["funded_exposure"] = pd.to_numeric( - prepared["funded_exposure"], errors="coerce" - ).fillna(0.0) - total_exposure = float(prepared["funded_exposure"].sum()) - if total_exposure <= 0.0: - raise ValueError("funded exposure must be positive") - prepared["funded_weight"] = prepared["funded_exposure"] / total_exposure - prepared["default_flag"] = pd.to_numeric(prepared["default_flag"], errors="coerce").fillna(0.0) - prepared["miscoverage_alpha01"] = pd.to_numeric( - prepared["miscoverage_alpha01"], errors="coerce" - ).fillna(0.0) - prepared["pd_point_alpha01"] = pd.to_numeric( - prepared["pd_point_alpha01"], errors="coerce" - ).fillna(0.0) - prepared["pd_high_alpha01"] = pd.to_numeric( - prepared["pd_high_alpha01"], errors="coerce" - ).fillna(0.0) - prepared["int_rate_decimal"] = _parse_percent_series(prepared["int_rate"]) - if "grade_bucket" not in prepared.columns: - grade = prepared.get("grade", pd.Series("unknown", index=prepared.index)) - prepared["grade_bucket"] = grade.fillna("unknown").astype(str).str.upper().str[:1] - prepared["issue_period"] = _issue_period(prepared) - return prepared - - -def _issue_period(frame: pd.DataFrame) -> pd.Series: - if "issue_d" not in frame.columns: - return pd.Series("unknown", index=frame.index, dtype="object") - parsed = pd.to_datetime(frame["issue_d"], errors="coerce") - return parsed.dt.to_period("Q").astype("string").fillna("unknown") - - -def _policy_metrics(body_audit: dict[str, Any]) -> dict[str, float]: - metrics = body_audit.get("metrics", {}) - if not isinstance(metrics, dict): - return {} - return { - key: float(metrics[key]) - for key in [ - "realized_return", - "Gamma_CP", - "V", - "endpoint_budget_upper", - "markov_cap", - "empirical_coverage_funded", - ] - if key in metrics - } - - -def build_tail_risk_table( - funded: pd.DataFrame, - *, - body_audit: dict[str, Any], - lgds: tuple[float, ...], -) -> pd.DataFrame: - prepared = _prepare_allocation(funded) - exposure = prepared["funded_exposure"].to_numpy(dtype=float) - int_rates = prepared["int_rate_decimal"].to_numpy(dtype=float) - default_flag = prepared["default_flag"].to_numpy(dtype=float) - pd_high = prepared["pd_high_alpha01"].to_numpy(dtype=float) - weights = prepared["funded_weight"].to_numpy(dtype=float) - total_exposure = float(exposure.sum()) - policy = _policy_metrics(body_audit) - - rows: list[dict[str, Any]] = [] - for lgd in lgds: - realized_loss = funded_loss_rate(default_flag, int_rates, lgd=float(lgd)) - decision_loss = pd_high * float(lgd) - (1.0 - pd_high) * int_rates - row: dict[str, Any] = { - "paper_role": "pool93_body_default", - "lgd": float(lgd), - "funded_rows": int(len(prepared)), - "total_allocated": total_exposure, - "weighted_default_rate": float(np.sum(weights * default_flag)), - "mean_realized_loss_rate": weighted_mean(realized_loss, exposure), - "funded_set_repriced_return": -weighted_mean(realized_loss, exposure) * total_exposure, - "decision_time_mean_loss_rate": weighted_mean(decision_loss, exposure), - "decision_time_oce_theta5": entropic_oce(decision_loss, exposure, theta=OCE_THETA), - "realized_oce_theta5": entropic_oce(realized_loss, exposure, theta=OCE_THETA), - } - for cvar_level in CVAR_LEVELS: - pct = int(cvar_level * 100) - row[f"decision_time_cvar{pct}_loss_rate"] = weighted_cvar( - decision_loss, exposure, tail=cvar_level - ) - row[f"realized_cvar{pct}_loss_rate"] = weighted_cvar( - realized_loss, exposure, tail=cvar_level - ) - row.update( - { - "alpha01_weighted_miscoverage_V": policy.get( - "V", float(np.sum(weights * prepared["miscoverage_alpha01"].to_numpy())) - ), - "alpha01_gamma_cp": policy.get( - "Gamma_CP", - float( - np.sum( - weights - * np.clip( - prepared["pd_high_alpha01"].to_numpy(dtype=float) - - prepared["pd_point_alpha01"].to_numpy(dtype=float), - 0.0, - 1.0, - ) - ) - ), - ), - "endpoint_budget_upper_alpha01": policy.get("endpoint_budget_upper", np.nan), - "markov_cap_alpha01": policy.get("markov_cap", np.nan), - "funded_empirical_coverage": policy.get( - "empirical_coverage_funded", - float(1.0 - prepared["miscoverage_alpha01"].mean()), - ), - } - ) - rows.append(row) - return pd.DataFrame(rows) - - -def _cluster_series(frame: pd.DataFrame, partition: str) -> pd.Series: - if partition == "period": - return frame["issue_period"].astype(str) - if partition == "grade_bucket": - return frame["grade_bucket"].astype(str) - if partition == "period_grade": - return frame["issue_period"].astype(str) + "|" + frame["grade_bucket"].astype(str) - if partition == "score_vintage": - if "temporal_segment" in frame.columns: - return frame["temporal_segment"].fillna("unknown").astype(str) - return frame["issue_period"].astype(str) - raise ValueError(f"unsupported partition: {partition}") - - -def build_cluster_bound_table( - funded: pd.DataFrame, - *, - body_audit: dict[str, Any], - alpha: float, - delta: float, -) -> pd.DataFrame: - prepared = _prepare_allocation(funded) - weights = prepared["funded_weight"].to_numpy(dtype=float) - miscoverage = prepared["miscoverage_alpha01"].to_numpy(dtype=float) - empirical_v = float(np.sum(weights * miscoverage)) - policy = _policy_metrics(body_audit) - empirical_v = policy.get("V", empirical_v) - markov_threshold = float(np.sqrt(alpha)) - s2_tightening_threshold = float(2.0 * (markov_threshold - alpha) ** 2 / np.log(1.0 / delta)) - - rows: list[dict[str, Any]] = [] - for partition in ("period", "grade_bucket", "period_grade", "score_vintage"): - clusters = _cluster_series(prepared, partition) - exposure_share = prepared.groupby(clusters)["funded_weight"].sum() - sum_w2 = float(np.sum(np.square(exposure_share.to_numpy(dtype=float)))) - threshold = float(alpha + np.sqrt(0.5 * sum_w2 * np.log(1.0 / delta))) - rows.append( - { - "cluster_type": partition, - "n_clusters": int(exposure_share.size), - "max_cluster_exposure_share": float(exposure_share.max()), - "sum_cluster_exposure_sq": sum_w2, - "alpha": float(alpha), - "delta": float(delta), - "empirical_weighted_miscoverage_V": empirical_v, - "markov_threshold": markov_threshold, - "cluster_hoeffding_threshold": threshold, - "sum_w2_tightening_threshold": s2_tightening_threshold, - "cluster_bound_tighter_than_markov": bool(threshold < markov_threshold), - "paper_role": "pool93_body_cluster_sensitivity", - } - ) - return pd.DataFrame(rows) - - -def _pool93_metric_snapshot( - frame: pd.DataFrame, - *, - total_exposure: float, - lgd: float, -) -> dict[str, float]: - weights = frame["funded_exposure"].to_numpy(dtype=float) - weights = weights / max(float(weights.sum()), 1e-12) - exposure = weights * float(total_exposure) - default_flag = frame["default_flag"].to_numpy(dtype=float) - miscoverage = frame["miscoverage_alpha01"].to_numpy(dtype=float) - int_rates = frame["int_rate_decimal"].to_numpy(dtype=float) - pd_point = frame["pd_point_alpha01"].to_numpy(dtype=float) - pd_high = frame["pd_high_alpha01"].to_numpy(dtype=float) - realized_loss = funded_loss_rate(default_flag, int_rates, lgd=float(lgd)) - decision_loss = pd_high * float(lgd) - (1.0 - pd_high) * int_rates - gamma_cp = float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))) - return { - "funded_set_repriced_return_lgd45": -weighted_mean(realized_loss, exposure) - * float(total_exposure), - "weighted_default_rate": float(np.sum(weights * default_flag)), - "weighted_miscoverage_V": float(np.sum(weights * miscoverage)), - "alpha01_gamma_cp": gamma_cp, - "realized_cvar95_loss_rate": weighted_cvar(realized_loss, exposure, tail=0.95), - "decision_time_cvar95_loss_rate": weighted_cvar(decision_loss, exposure, tail=0.95), - "realized_oce_theta5": entropic_oce(realized_loss, exposure, theta=OCE_THETA), - "n_default_loans": float(np.sum(default_flag)), - "n_miscovered_loans": float(np.sum(miscoverage)), - } - - -def build_bootstrap_table( - funded: pd.DataFrame, - *, - body_audit: dict[str, Any], - n_draws: int, - seed: int, - lgd: float, -) -> pd.DataFrame: - prepared = _prepare_allocation(funded) - total_exposure = float(prepared["funded_exposure"].sum()) - observed = _pool93_metric_snapshot(prepared, total_exposure=total_exposure, lgd=lgd) - policy = _policy_metrics(body_audit) - if "realized_return" in policy: - observed["funded_set_repriced_return_lgd45"] = policy["realized_return"] - if "V" in policy: - observed["weighted_miscoverage_V"] = policy["V"] - if "Gamma_CP" in policy: - observed["alpha01_gamma_cp"] = policy["Gamma_CP"] - - rng = np.random.default_rng(int(seed)) - n_rows = len(prepared) - draws: list[dict[str, float]] = [] - for _ in range(int(n_draws)): - sample_index = rng.integers(0, n_rows, size=n_rows) - sample = prepared.iloc[sample_index].reset_index(drop=True) - draws.append(_pool93_metric_snapshot(sample, total_exposure=total_exposure, lgd=lgd)) - - draw_frame = pd.DataFrame(draws) - rows: list[dict[str, Any]] = [] - note = "Funded-loan contribution bootstrap; solver input uncertainty is not resampled." - for metric in draw_frame.columns: - values = draw_frame[metric] - rows.append( - { - "metric": metric, - "observed": float(observed[metric]), - "boot_mean": float(values.mean()), - "boot_p025": float(values.quantile(0.025)), - "boot_p50": float(values.quantile(0.50)), - "boot_p975": float(values.quantile(0.975)), - "n_draws": int(n_draws), - "seed": int(seed), - "note": note, - } - ) - return pd.DataFrame(rows) - - -def build_outputs( - *, - allocation_path: Path, - body_audit_path: Path, - table_dir: Path, - output_dir: Path, - lgds: tuple[float, ...], - alpha: float, - delta: float, - bootstrap_draws: int, - bootstrap_seed: int, -) -> dict[str, Any]: - funded = _load_allocation(allocation_path) - body_audit = _read_json(body_audit_path) - tail_table = build_tail_risk_table(funded, body_audit=body_audit, lgds=lgds) - cluster_table = build_cluster_bound_table( - funded, body_audit=body_audit, alpha=alpha, delta=delta - ) - bootstrap_table = build_bootstrap_table( - funded, - body_audit=body_audit, - n_draws=bootstrap_draws, - seed=bootstrap_seed, - lgd=0.45, - ) - outputs = { - "tail_risk": _write_table(table_dir, TABLE_A37_NAME, tail_table), - "cluster_bound": _write_table(table_dir, TABLE_A38_NAME, cluster_table), - "bootstrap": _write_table(table_dir, TABLE_A39_NAME, bootstrap_table), - } - payload = { - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "allocation_path": str(allocation_path), - "body_audit_path": str(body_audit_path), - "alpha": float(alpha), - "delta": float(delta), - "lgds": list(lgds), - "bootstrap_draws": int(bootstrap_draws), - "bootstrap_seed": int(bootstrap_seed), - "tables": outputs, - "headline": { - "baseline_lgd_return": float( - tail_table.loc[tail_table["lgd"].eq(0.45), "funded_set_repriced_return"].iloc[0] - ) - if tail_table["lgd"].eq(0.45).any() - else float(tail_table["funded_set_repriced_return"].iloc[0]), - "baseline_lgd_realized_cvar95": float( - tail_table.loc[tail_table["lgd"].eq(0.45), "realized_cvar95_loss_rate"].iloc[0] - ) - if tail_table["lgd"].eq(0.45).any() - else float(tail_table["realized_cvar95_loss_rate"].iloc[0]), - "baseline_lgd_decision_time_cvar95": float( - tail_table.loc[tail_table["lgd"].eq(0.45), "decision_time_cvar95_loss_rate"].iloc[0] - ) - if tail_table["lgd"].eq(0.45).any() - else float(tail_table["decision_time_cvar95_loss_rate"].iloc[0]), - "min_cluster_hoeffding_threshold": float( - cluster_table["cluster_hoeffding_threshold"].min() - ), - "markov_threshold": float(np.sqrt(alpha)), - "any_cluster_tighter_than_markov": bool( - cluster_table["cluster_bound_tighter_than_markov"].any() - ), - "bootstrap_return_lgd45_p025": float( - bootstrap_table.loc[ - bootstrap_table["metric"].eq("funded_set_repriced_return_lgd45"), - "boot_p025", - ].iloc[0] - ), - "bootstrap_return_lgd45_p975": float( - bootstrap_table.loc[ - bootstrap_table["metric"].eq("funded_set_repriced_return_lgd45"), - "boot_p975", - ].iloc[0] - ), - }, - } - output_dir.mkdir(parents=True, exist_ok=True) - audit_path = output_dir / "pool93_body_tail_risk_audit.json" - audit_path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - newline="", - ) - payload["audit_path"] = str(audit_path) - return payload - - -def _parse_lgds(raw: str) -> tuple[float, ...]: - values = tuple(float(part.strip()) for part in raw.split(",") if part.strip()) - if not values: - raise ValueError("at least one LGD value is required") - return values - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--allocation", default=str(DEFAULT_ALLOCATION_PATH)) - parser.add_argument("--body-audit", default=str(DEFAULT_BODY_AUDIT_PATH)) - parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) - parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) - parser.add_argument("--lgds", default="0.35,0.45,0.60") - parser.add_argument("--alpha", type=float, default=0.01) - parser.add_argument("--delta", type=float, default=0.10) - parser.add_argument("--bootstrap-draws", type=int, default=BOOTSTRAP_DRAWS) - parser.add_argument("--bootstrap-seed", type=int, default=BOOTSTRAP_SEED) - args = parser.parse_args(argv) - - payload = build_outputs( - allocation_path=Path(args.allocation), - body_audit_path=Path(args.body_audit), - table_dir=Path(args.table_dir), - output_dir=Path(args.output_dir), - lgds=_parse_lgds(str(args.lgds)), - alpha=float(args.alpha), - delta=float(args.delta), - bootstrap_draws=int(args.bootstrap_draws), - bootstrap_seed=int(args.bootstrap_seed), - ) - print(json.dumps(payload["headline"], indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/build_portfolio_exact_priority_context.py b/scripts/search/build_portfolio_exact_priority_context.py deleted file mode 100644 index eb97621..0000000 --- a/scripts/search/build_portfolio_exact_priority_context.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Build a priority execution context for exact portfolio reranking. - -The exact certificate remains full-universe: this script only changes the order -in which the frozen shortlist is evaluated and removes redundant seeds when the -evaluation universe is not sampled. -""" - -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] - - -def _resolve(path_like: object) -> Path: - path = Path(str(path_like)) - return path if path.is_absolute() else ROOT / path - - -def _reason(row: pd.Series, *, champion_return: float) -> str: - bucket = str(row.get("shortlist_bucket", "")) - realized = float(row.get("realized_total_return", 0.0)) - ab_pass = bool(row.get("ab_pass_all", False)) - if realized > champion_return and ab_pass and bucket == "forced_incumbent_neighbors": - return "above_champion_forced_incumbent_region" - if realized > champion_return and ab_pass: - return "above_champion_ab_pass" - if bucket == "forced_incumbent_neighbors": - return "forced_incumbent_region" - if bucket == "incumbent_region": - return "incumbent_region" - return "remaining_shortlist" - - -def _tier(row: pd.Series, *, champion_return: float) -> int: - reason = _reason(row, champion_return=champion_return) - return { - "above_champion_forced_incumbent_region": 0, - "above_champion_ab_pass": 1, - "forced_incumbent_region": 2, - "incumbent_region": 3, - "remaining_shortlist": 4, - }[reason] - - -def build_priority_context( - *, - context_path: Path, - output_context_path: Path | None = None, - champion_return: float, -) -> dict[str, Any]: - context_path = context_path.resolve() - context = json.loads(context_path.read_text(encoding="utf-8")) - shortlist_path = _resolve(context["shortlist_path"]) - shortlist = pd.read_parquet(shortlist_path) - if shortlist.empty: - raise ValueError(f"Cannot prioritize empty shortlist: {shortlist_path}") - - prioritized = shortlist.copy() - prioritized["exact_priority_reason"] = prioritized.apply( - _reason, - axis=1, - champion_return=float(champion_return), - ) - prioritized["exact_priority_tier"] = prioritized.apply( - _tier, - axis=1, - champion_return=float(champion_return), - ) - prioritized["exact_priority_return_surplus"] = pd.to_numeric( - prioritized["realized_total_return"], errors="coerce" - ).fillna(0.0) - float(champion_return) - prioritized = prioritized.sort_values( - by=[ - "exact_priority_tier", - "realized_total_return", - "candidate_rank", - ], - ascending=[True, False, True], - kind="mergesort", - ).reset_index(drop=True) - prioritized["exact_priority_order"] = np.arange(1, len(prioritized) + 1) - - priority_shortlist_path = shortlist_path.with_name(shortlist_path.stem + "_priority.parquet") - prioritized.to_parquet(priority_shortlist_path, index=False) - - requested_random_states = list(context.get("exact_random_states", context["random_states"])) - exact_max_candidates = int(context.get("exact_max_candidates", context["max_candidates"])) - effective_random_states = list(requested_random_states) - full_universe_seed_deduped = False - if exact_max_candidates <= 0 and len(effective_random_states) > 1: - effective_random_states = [int(effective_random_states[0])] - full_universe_seed_deduped = True - - priority_context = dict(context) - priority_context["shortlist_path"] = str(priority_shortlist_path) - priority_context["shortlist_priority_source_path"] = str(shortlist_path) - priority_context["exact_priority_context_source_path"] = str(context_path) - priority_context["exact_priority_generated_at_utc"] = datetime.now(tz=UTC).isoformat() - priority_context["exact_priority_champion_return"] = float(champion_return) - priority_context["exact_priority_strategy"] = [ - "above_champion_forced_incumbent_region", - "above_champion_ab_pass", - "forced_incumbent_region", - "incumbent_region", - "remaining_shortlist", - ] - priority_context["requested_exact_random_states"] = requested_random_states - priority_context["exact_random_states"] = effective_random_states - priority_context["full_universe_seed_deduped"] = bool(full_universe_seed_deduped) - priority_context.setdefault("selection_policy", {}) - priority_context["selection_policy"]["exact_execution_order"] = "claim_priority_return_first" - priority_context["selection_policy"]["full_universe_seed_deduped"] = bool( - full_universe_seed_deduped - ) - - if output_context_path is None: - output_context_path = context_path.with_name(context_path.stem + "_priority.json") - output_context_path = output_context_path.resolve() - output_context_path.write_text( - json.dumps(priority_context, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - return { - "context_path": str(output_context_path), - "priority_shortlist_path": str(priority_shortlist_path), - "n_shortlist": int(len(prioritized)), - "n_above_champion": int( - (prioritized["realized_total_return"] > float(champion_return)).sum() - ), - "requested_exact_random_states": requested_random_states, - "effective_exact_random_states": effective_random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), - "top_candidates": prioritized[ - [ - "exact_priority_order", - "candidate_rank", - "exact_priority_reason", - "risk_tolerance", - "policy_mode", - "gamma", - "uncertainty_aversion", - "realized_total_return", - ] - ] - .head(12) - .to_dict(orient="records"), - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--context-path", required=True) - parser.add_argument("--output-context-path", default="") - parser.add_argument("--champion-return", type=float, default=170_464.54) - args = parser.parse_args() - - payload = build_priority_context( - context_path=Path(args.context_path), - output_context_path=Path(args.output_context_path) if args.output_context_path else None, - champion_return=float(args.champion_return), - ) - print(json.dumps(payload, indent=2, ensure_ascii=False)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/export_pool93_policy_aware_frontier.py b/scripts/search/export_pool93_policy_aware_frontier.py deleted file mode 100644 index 17e505b..0000000 --- a/scripts/search/export_pool93_policy_aware_frontier.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Export the policy-aware pool93 frontier as CSV and LaTeX.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -import pandas as pd - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" -DEFAULT_FRONTIER = ( - ROOT - / "models/experiments/champion_reopen" - / DEFAULT_TAG - / "portfolio/pool93_ijds_consolidated_frontier.json" -) -DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG -DEFAULT_TABLE_NAME = "crpto_tableA35_pool93_ijds_frontier_policy_aware_v2" - -ROLE_ORDER = ( - "minimum Markov-threshold endpoint", - "lowest realized V return-bound point", - "highest return under threshold<=0.3", - "highest return under threshold<=0.32", - "highest return under threshold<=0.345", - "body/default balanced return-bound point", - "highest return under threshold<=0.36", - "highest return under threshold<=0.45", - "max-return economic endpoint", -) -ROLE_LABELS = { - "minimum Markov-threshold endpoint": "Minimum Markov-threshold endpoint", - "lowest realized V return-bound point": "Low-threshold balanced endpoint", - "highest return under threshold<=0.3": "Highest return under threshold <= 0.30", - "highest return under threshold<=0.32": "Highest return under threshold <= 0.32", - "highest return under threshold<=0.345": "Highest return under threshold <= 0.345", - "body/default balanced return-bound point": "Body/default balanced point", - "highest return under threshold<=0.36": "Highest return under threshold <= 0.36", - "highest return under threshold<=0.45": "Highest return under threshold <= 0.45", - "max-return economic endpoint": "Max-return economic endpoint", -} - - -def build_table(frontier: dict[str, Any]) -> pd.DataFrame: - """Build the compact publication table in its declared role order.""" - rows_by_role = {str(row["role"]): dict(row) for row in frontier.get("rows", [])} - missing = [role for role in ROLE_ORDER if role not in rows_by_role] - if missing: - raise ValueError(f"Policy-aware frontier is missing roles: {missing}") - rows = [] - for role in ROLE_ORDER: - row = rows_by_role[role] - rows.append( - { - "role": ROLE_LABELS[role], - "source_run": str(row["run_label"]), - "candidate_id": int(row["local_candidate_id"]), - "policy_family": str(row["family"]), - "risk_tolerance": float(row["risk_tolerance"]), - "policy_mode": str(row["policy_mode"]), - "gamma": float(row["gamma"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - "realized_return": float(row["return"]), - "return_floor_surplus": float(row["return_floor_surplus"]), - "Gamma_CP_alpha01": float(row["Gamma_CP"]), - "Gamma_residual_alpha01": float(row["Gamma_residual"]), - "V_alpha01": float(row["V"]), - "endpoint_budget_alpha01": float(row["endpoint_budget"]), - "endpoint_budget_upper_alpha01": float(row["endpoint_budget_upper"]), - "Markov_threshold_alpha01": float(row["Markov_threshold"]), - "Markov_cap_alpha01": float(row["Markov_cap"]), - "alpha_grid_pass": str(row["alpha_pass"]), - "n_funded_mean": float(row["n_funded_mean"]), - } - ) - return pd.DataFrame(rows) - - -def _format_tex(table: pd.DataFrame) -> str: - lines = [ - "\\begin{tabular}{llrrrrrr}", - "\\toprule", - ( - "Role & Source & Return & $\\Gamma_{\\mathrm{CP}}$ & " - "$\\Gamma_{\\mathrm{res}}$ & $V$ & Markov threshold & Pass \\\\" - ), - "\\midrule", - ] - for row in table.to_dict("records"): - role = str(row["role"]).replace("<=", "$\\leq$") - source = str(row["source_run"]).replace("_", " ") - lines.append( - f"{role} & {source} & {float(row['realized_return']):,.2f} & " - f"{float(row['Gamma_CP_alpha01']):.6f} & " - f"{float(row['Gamma_residual_alpha01']):.6f} & " - f"{float(row['V_alpha01']):.6f} & " - f"{float(row['Markov_threshold_alpha01']):.6f} & " - f"{row['alpha_grid_pass']} \\\\" - ) - lines.extend(["\\bottomrule", "\\end{tabular}", ""]) - return "\n".join(lines) - - -def write_frontier_table(table: pd.DataFrame, *, table_dir: Path, table_name: str) -> None: - """Write the full audit CSV and compact publication LaTeX table.""" - table_dir.mkdir(parents=True, exist_ok=True) - csv_path = table_dir / f"{table_name}.csv" - tex_path = table_dir / f"{table_name}.tex" - outputs = { - csv_path: table.to_csv(index=False, lineterminator="\n"), - tex_path: _format_tex(table), - } - for path, content in outputs.items(): - if path.exists() and path.read_text(encoding="utf-8") == content: - continue - path.write_text(content, encoding="utf-8", newline="") - print(f"Wrote {csv_path.relative_to(ROOT).as_posix()}") - print(f"Wrote {tex_path.relative_to(ROOT).as_posix()}") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) - parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) - parser.add_argument("--table-name", default=DEFAULT_TABLE_NAME) - args = parser.parse_args(argv) - frontier = json.loads(Path(args.frontier).read_text(encoding="utf-8")) - write_frontier_table( - build_table(frontier), - table_dir=Path(args.table_dir).resolve(), - table_name=str(args.table_name), - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/run_conformal_reopen_search.py b/scripts/search/run_conformal_reopen_search.py deleted file mode 100644 index 38d3fc4..0000000 --- a/scripts/search/run_conformal_reopen_search.py +++ /dev/null @@ -1,1963 +0,0 @@ -"""Reopen conformal search exhaustively over a fixed upstream PD candidate.""" - -from __future__ import annotations - -import argparse -import json -import os -import pickle -import subprocess -import sys -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import pandas as pd - -from scripts.generate_conformal_intervals import ( - _build_feature_matrix, - _load_model, - _resolve_features, -) -from src.models.calibration import ( - QuadraticLogitCalibrator, - TemperatureScalingCalibrator, - calibrate_beta, - calibrate_isotonic, - calibrate_platt, - evaluate_calibration, -) -from src.models.conformal import apply_probability_calibrator -from src.models.conformal_tuning import split_calibration_for_tuning -from src.models.venn_abers import VennAbersScoreCalibrator -from src.utils.pipeline_topology import load_profile_config - - -@dataclass(frozen=True) -class Phase1Result: - shortlist: pd.DataFrame - aggregated: pd.DataFrame - inner_runs: list[dict[str, Any]] - aggregate_path: str - inner_search_path: str - inner_search_winner: dict[str, Any] - resume_meta: dict[str, Any] | None = None - - -@dataclass(frozen=True) -class Phase1ConfirmationResult: - candidates: list[dict[str, Any]] - frame: pd.DataFrame - best_namespace: str - final_policy: dict[str, Any] - final_sets: dict[str, Any] - final_decision: str - final_namespace: str - - -@dataclass(frozen=True) -class PromotionResult: - final_policy: dict[str, Any] - final_sets: dict[str, Any] - final_decision: str - final_namespace: str - phase2_summary: dict[str, Any] | None - - -@dataclass -class Phase2SearchState: - paths: dict[str, Path] - run_tag: str - upstream_run_tag: str - total_design_runs: int - design_source: str - completed: list[dict[str, Any]] - skipped: list[dict[str, Any]] - failed: list[dict[str, Any]] - - -@dataclass(frozen=True) -class Phase2CalibratorFit: - method: str - path: Path - metrics: dict[str, float] - - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def _utc_now() -> str: - return datetime.now(UTC).isoformat() - - -def _default_run_tag() -> str: - return datetime.now(UTC).strftime("conformal-reopen-%Y-%m-%d-%H%M") - - -def _profile_cfg(profile_name: str) -> dict[str, Any]: - profile = load_profile_config(profile_name) - if not profile: - raise FileNotFoundError(f"Missing conformal reopen profile: {profile_name}") - return profile - - -def _phase1_cfg(profile: dict[str, Any]) -> dict[str, Any]: - return dict((profile.get("search_space", {}) or {}).get("phase1", {}) or {}) - - -def _phase2_cfg(profile: dict[str, Any]) -> dict[str, Any]: - return dict((profile.get("search_space", {}) or {}).get("phase2", {}) or {}) - - -def _sidecar_cfg(profile: dict[str, Any]) -> dict[str, Any]: - return dict(profile.get("sidecar", {}) or {}) - - -def _validation_cfg(profile: dict[str, Any]) -> dict[str, Any]: - return dict(profile.get("validation", {}) or {}) - - -def _run_python(script: str, args: list[str], env: dict[str, str]) -> None: - cmd = [sys.executable, script, *args] - subprocess.run(cmd, cwd=REPO_ROOT, check=True, env=env) - - -def _namespace(run_tag: str, *parts: object) -> str: - suffix = "__".join(str(part).strip().replace("/", "_") for part in parts if str(part).strip()) - return f"{run_tag}__{suffix}" if suffix else run_tag - - -def _load_pickle(path: Path) -> dict[str, Any]: - with open(path, "rb") as handle: - payload = pickle.load(handle) - return dict(payload) if isinstance(payload, dict) else {"payload": payload} - - -def _load_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - - -def _resolve_run_paths(namespace: str) -> dict[str, Path]: - ns = str(namespace).strip().replace("/", "_") - data_dir = REPO_ROOT / "data" / "processed" / "conformal_gap" / ns - models_dir = REPO_ROOT / "models" / "conformal_gap" / ns - data_dir.mkdir(parents=True, exist_ok=True) - models_dir.mkdir(parents=True, exist_ok=True) - return { - "data_dir": data_dir, - "models_dir": models_dir, - "tuning": data_dir / "conformal_mondrian_tuning_90.parquet", - "results": models_dir / "conformal_results_mondrian.pkl", - "policy_status": models_dir / "conformal_policy_status.json", - "selection_status": models_dir / "conformal_variant_selection_status.json", - "set_status": models_dir / "pd_set_prediction_status.json", - } - - -def _reopen_artifact_paths(run_tag: str) -> dict[str, Path]: - run_paths = _resolve_run_paths(run_tag) - data_dir = run_paths["data_dir"] - models_dir = run_paths["models_dir"] - return { - "data_dir": data_dir, - "models_dir": models_dir, - "inner_search": data_dir / "conformal_reopen_inner_search.parquet", - "inner_aggregate": data_dir / "conformal_reopen_inner_aggregate.parquet", - "phase1_shortlist": data_dir / "conformal_reopen_phase1_shortlist.parquet", - "phase1_final_candidates": data_dir / "conformal_reopen_phase1_final_candidates.parquet", - "phase2_search": data_dir / "conformal_reopen_phase2_search.parquet", - "status": models_dir / "conformal_reopen_status.json", - "phase1_progress": models_dir / "conformal_reopen_phase1_progress.json", - "phase2_progress": models_dir / "conformal_reopen_phase2_progress.json", - } - - -class _temporary_env: - def __init__(self, updates: dict[str, str]) -> None: - self._updates = updates - self._previous: dict[str, str | None] = {} - - def __enter__(self) -> None: - for key, value in self._updates.items(): - self._previous[key] = os.environ.get(key) - os.environ[key] = value - - def __exit__(self, exc_type, exc, tb) -> None: - for key, previous in self._previous.items(): - if previous is None: - os.environ.pop(key, None) - else: - os.environ[key] = previous - - -def _normalize_design_row(row: dict[str, Any] | pd.Series) -> dict[str, Any]: - raw = dict(row) - alpha90 = raw.get("alpha_used_90", raw.get("selected_alpha_used", 0.10)) - return { - "partition": str(raw["partition"]), - "partition_probability_source": str(raw.get("partition_probability_source", "raw")), - "n_score_bins": int(float(raw.get("n_score_bins", 10))), - "fallback_mode": str(raw.get("fallback_mode", "grade_then_global")), - "alpha_used_90": float(0.10 if alpha90 is None else alpha90), - "alpha_used_95": float(raw.get("alpha_used_95", 0.05)), - "score_scale_family": str(raw.get("score_scale_family", "none")), - "min_group_size": int(float(raw.get("min_group_size", 100))), - "calibration_fraction": float(raw.get("calibration_fraction", 1.0)), - } - - -def _semantic_candidate_key(row: dict[str, Any] | pd.Series) -> str: - design = _normalize_design_row(row) - partition = str(design["partition"]).strip().lower() - payload: dict[str, Any] = { - "partition": partition, - "fallback_mode": str(design["fallback_mode"]), - "alpha_used_90": float(design["alpha_used_90"]), - "alpha_used_95": float(design["alpha_used_95"]), - "score_scale_family": str(design["score_scale_family"]), - "min_group_size": int(design["min_group_size"]), - "calibration_fraction": float(design["calibration_fraction"]), - } - if partition != "grade": - payload["partition_probability_source"] = str(design["partition_probability_source"]) - payload["n_score_bins"] = int(design["n_score_bins"]) - return json.dumps(payload, sort_keys=True) - - -def _candidate_key(row: pd.Series) -> str: - return json.dumps(_normalize_design_row(row), sort_keys=True) - - -def _aggregate_inner_search(rows: pd.DataFrame) -> pd.DataFrame: - if rows.empty: - raise RuntimeError("Inner conformal search produced no rows.") - work = rows.copy() - if "alpha_used_95" not in work.columns: - work["alpha_used_95"] = 0.05 - work["coverage_gap_abs"] = work["coverage_gap"].abs() - work["candidate_key"] = work.apply(_candidate_key, axis=1) - agg = ( - work.groupby( - [ - "candidate_key", - "partition", - "partition_probability_source", - "n_score_bins", - "fallback_mode", - "alpha_used_90", - "alpha_used_95", - "score_scale_family", - "min_group_size", - "calibration_fraction", - ], - dropna=False, - ) - .agg( - n_runs=("candidate_key", "size"), - global_ok_rate=("global_ok", "mean"), - group_ok_rate=("group_ok", "mean"), - width_ok_rate=("width_ok", "mean"), - pareto_rate=("is_pareto", "mean"), - mean_coverage=("empirical_coverage", "mean"), - mean_abs_coverage_gap=("coverage_gap_abs", "mean"), - mean_avg_interval_width=("avg_interval_width", "mean"), - mean_min_group_coverage=("min_group_coverage", "mean"), - mean_winkler_90=("winkler_90", "mean"), - mean_stability_over_time=("stability_over_time", "mean"), - mean_max_monthly_gap=("max_monthly_gap", "mean"), - ) - .reset_index() - ) - agg = agg.sort_values( - by=[ - "global_ok_rate", - "group_ok_rate", - "width_ok_rate", - "mean_abs_coverage_gap", - "mean_min_group_coverage", - "mean_winkler_90", - "mean_avg_interval_width", - "mean_stability_over_time", - ], - ascending=[False, False, False, True, False, True, True, True], - ).reset_index(drop=True) - agg["selection_rank"] = range(1, len(agg) + 1) - return agg - - -def _dedupe_designs(rows: pd.DataFrame, top_k: int | None = None) -> pd.DataFrame: - records: list[dict[str, Any]] = [] - seen: set[str] = set() - for row in rows.to_dict(orient="records"): - key = _semantic_candidate_key(row) - if key in seen: - continue - seen.add(key) - records.append(dict(row)) - if top_k is not None and len(records) >= int(top_k): - break - out = pd.DataFrame(records) - if out.empty: - return out - out = out.reset_index(drop=True) - out["selection_rank"] = range(1, len(out) + 1) - return out - - -def _design_args(design: dict[str, Any]) -> list[str]: - normalized = _normalize_design_row(design) - return [ - "--partition", - str(normalized["partition"]), - "--partition_candidates", - str(normalized["partition"]), - "--partition_probability_sources", - str(normalized["partition_probability_source"]), - "--n_score_bins_candidates", - str(int(normalized["n_score_bins"])), - "--fallback_modes", - str(normalized["fallback_mode"]), - "--alpha_candidates_90", - str(float(normalized["alpha_used_90"])), - "--alpha_candidates_95", - str(float(normalized.get("alpha_used_95", 0.05))), - "--min_group_sizes", - str(int(normalized["min_group_size"])), - "--score_scale_families", - str(normalized["score_scale_family"]), - "--scaled_scores_options", - "true" if str(normalized["score_scale_family"]).strip().lower() != "none" else "false", - "--calibration_fraction", - str(float(normalized["calibration_fraction"])), - ] - - -def _benchmark_row_to_design(row: dict[str, Any] | pd.Series) -> dict[str, Any]: - raw = dict(row) - return _normalize_design_row( - { - "partition": raw["partition"], - "partition_probability_source": raw.get("partition_probability_source", "raw"), - "n_score_bins": raw.get("n_score_bins", 10), - "fallback_mode": raw.get("fallback_mode", "grade_then_global"), - "alpha_used_90": raw.get("selected_alpha_used", raw.get("alpha", 0.10)), - "alpha_used_95": 0.05, - "score_scale_family": raw.get("score_scale_family", "none"), - "min_group_size": raw.get("min_group_size", 100), - "calibration_fraction": raw.get("calibration_fraction", 1.0), - } - ) - - -def _phase1_inner_artifacts_complete(namespace: str) -> bool: - paths = _resolve_run_paths(namespace) - return bool(paths["tuning"].exists() and paths["results"].exists()) - - -def _phase1_inner_result( - *, - namespace: str, - calibration_fraction: float, - holdout_ratio: float, - random_state: int, - reused_existing: bool, -) -> tuple[pd.DataFrame, dict[str, Any]]: - tuning_df = pd.read_parquet(_resolve_run_paths(namespace)["tuning"]) - tuning_df["calibration_fraction"] = float(calibration_fraction) - tuning_df["holdout_ratio"] = float(holdout_ratio) - tuning_df["random_state"] = int(random_state) - tuning_df["artifact_namespace"] = namespace - result_payload = _load_pickle(_resolve_run_paths(namespace)["results"]) - run_summary = { - "artifact_namespace": namespace, - "calibration_fraction": float(calibration_fraction), - "holdout_ratio": float(holdout_ratio), - "random_state": int(random_state), - "reused_existing": bool(reused_existing), - "metrics_90": result_payload.get("metrics_90", {}), - "tuning_90_best": result_payload.get("tuning_90_best", {}), - "alpha_used_95": result_payload.get("alpha_used_95"), - } - return tuning_df, run_summary - - -def _write_phase1_progress( - *, - path: Path, - run_tag: str, - upstream_run_tag: str, - total: int, - completed: list[dict[str, Any]], - running: list[str], - failed: list[dict[str, Any]], - workers: int, -) -> None: - _write_json( - path, - { - "schema_version": "2026-06-22.1", - "generated_at_utc": _utc_now(), - "run_tag": run_tag, - "upstream_canonical_run_tag": upstream_run_tag, - "phase": "phase1_inner_search", - "workers": int(workers), - "total_inner_runs": int(total), - "completed_inner_runs": len(completed), - "running_inner_runs": list(running), - "failed_inner_runs": failed, - "completed": completed, - }, - ) - - -def _write_phase2_progress( - *, - path: Path, - run_tag: str, - upstream_run_tag: str, - total_design_runs: int, - completed: list[dict[str, Any]], - running: list[dict[str, Any]], - skipped: list[dict[str, Any]], - failed: list[dict[str, Any]], - design_source: str, -) -> None: - _write_json( - path, - { - "schema_version": "2026-06-24.1", - "generated_at_utc": _utc_now(), - "run_tag": run_tag, - "upstream_canonical_run_tag": upstream_run_tag, - "phase": "phase2_calibrator_design_search", - "design_source": design_source, - "total_design_runs": int(total_design_runs), - "completed_design_runs": len(completed), - "running": running, - "skipped": skipped, - "failed": failed, - "completed": completed, - }, - ) - - -def _phase1_inner_specs( - *, - run_tag: str, - calibration_fractions: list[float], - tuning_holdout_ratios: list[float], - inner_random_states: list[int], -) -> list[dict[str, Any]]: - specs: list[dict[str, Any]] = [] - for calibration_fraction in calibration_fractions: - for holdout_ratio in tuning_holdout_ratios: - for random_state in inner_random_states: - namespace = _namespace( - run_tag, - "phase1", - f"calfrac-{float(calibration_fraction):.2f}", - f"holdout-{float(holdout_ratio):.2f}", - f"seed-{int(random_state)}", - ) - specs.append( - { - "namespace": namespace, - "calibration_fraction": float(calibration_fraction), - "holdout_ratio": float(holdout_ratio), - "random_state": int(random_state), - } - ) - return specs - - -def _run_phase1_inner_spec( - *, - spec: dict[str, Any], - env: dict[str, str], - alpha_candidates_90: list[float], - alpha_candidates_95: list[float], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - min_group_sizes: list[int], - score_scale_families: list[str], - resume_completed: bool, -) -> tuple[pd.DataFrame, dict[str, Any]]: - namespace = str(spec["namespace"]) - calibration_fraction = float(spec["calibration_fraction"]) - holdout_ratio = float(spec["holdout_ratio"]) - random_state = int(spec["random_state"]) - if resume_completed and _phase1_inner_artifacts_complete(namespace): - return _phase1_inner_result( - namespace=namespace, - calibration_fraction=calibration_fraction, - holdout_ratio=holdout_ratio, - random_state=random_state, - reused_existing=True, - ) - _run_python( - "scripts/generate_conformal_intervals.py", - [ - "--artifact_namespace", - namespace, - "--evaluation_scope", - "holdout", - "--alpha_candidates_90", - ",".join(str(x) for x in alpha_candidates_90), - "--alpha_candidates_95", - ",".join(str(x) for x in alpha_candidates_95), - "--partition_candidates", - ",".join(str(x) for x in partition_candidates), - "--partition_probability_sources", - ",".join(str(x) for x in partition_probability_sources), - "--n_score_bins_candidates", - ",".join(str(int(x)) for x in n_score_bins_candidates), - "--fallback_modes", - ",".join(str(x) for x in fallback_modes), - "--min_group_sizes", - ",".join(str(int(x)) for x in min_group_sizes), - "--score_scale_families", - ",".join(str(x) for x in score_scale_families), - "--calibration_fraction", - str(calibration_fraction), - "--tuning_holdout_ratio", - str(holdout_ratio), - "--tuning_random_state", - str(random_state), - ], - env, - ) - return _phase1_inner_result( - namespace=namespace, - calibration_fraction=calibration_fraction, - holdout_ratio=holdout_ratio, - random_state=random_state, - reused_existing=False, - ) - - -def _fit_probability_calibrator( - *, - method_key: str, - model: Any, - X_fit: pd.DataFrame, - y_fit: pd.Series, - y_prob_fit_raw: Any, -) -> Any: - y_fit_arr = y_fit.to_numpy(dtype=int) - if method_key == "venn_abers": - return VennAbersScoreCalibrator().fit(y_prob_fit_raw, y_fit_arr) - if method_key == "isotonic": - return calibrate_isotonic(y_fit_arr, y_prob_fit_raw) - if method_key == "platt": - return calibrate_platt(model, X_fit, y_fit) - if method_key == "beta": - return calibrate_beta(y_fit_arr, y_prob_fit_raw) - if method_key == "temperature": - return TemperatureScalingCalibrator().fit(y_prob_fit_raw, y_fit_arr) - if method_key == "quadratic_logit": - return QuadraticLogitCalibrator().fit(y_prob_fit_raw, y_fit_arr) - raise ValueError(f"Unsupported calibrator method: {method_key}") - - -def _fit_calibrator( - *, - method: str, - output_path: Path, - upstream_run_tag: str, - validation_holdout_ratio: float = 0.20, - validation_random_state: int = 42, -) -> tuple[str, dict[str, float]]: - env = os.environ.copy() - env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag - with _temporary_env(env): - model, _ = _load_model() - cal_df = pd.read_parquet(REPO_ROOT / "data" / "processed" / "calibration_fe.parquet") - test_df = pd.read_parquet(REPO_ROOT / "data" / "processed" / "test_fe.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df["default_flag"].astype(int).reset_index(drop=True) - group_cal = cal_df.get("grade", pd.Series(["UNKNOWN"] * len(cal_df))).reset_index(drop=True) - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - idx_fit, idx_eval = split_calibration_for_tuning( - y_cal=y_cal, - group_cal=group_cal, - issue_dates=cal_df.get("issue_d"), - holdout_ratio=float(validation_holdout_ratio), - random_state=int(validation_random_state), - ) - X_cal_fit = X_cal.iloc[idx_fit].reset_index(drop=True) - y_cal_fit = y_cal.iloc[idx_fit].reset_index(drop=True) - y_prob_cal_fit_raw = y_prob_cal_raw[idx_fit] - y_eval = y_cal.iloc[idx_eval].to_numpy(dtype=int) - y_prob_eval_raw = y_prob_cal_raw[idx_eval] - - method_key = str(method).strip().lower() - if method_key in {"platt", "logit"}: - method_key = "platt" - elif method_key in {"quadratic_logit", "quadratic"}: - method_key = "quadratic_logit" - - validation_calibrator = _fit_probability_calibrator( - method_key=method_key, - model=model, - X_fit=X_cal_fit, - y_fit=y_cal_fit, - y_prob_fit_raw=y_prob_cal_fit_raw, - ) - final_calibrator = _fit_probability_calibrator( - method_key=method_key, - model=model, - X_fit=X_cal, - y_fit=y_cal, - y_prob_fit_raw=y_prob_cal_raw, - ) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "wb") as handle: - pickle.dump(final_calibrator, handle) - - calibrated_eval = apply_probability_calibrator(validation_calibrator, y_prob_eval_raw) - metrics = evaluate_calibration(y_eval, calibrated_eval, name=f"{method_key}_cal_holdout") - raw_metrics = evaluate_calibration(y_eval, y_prob_eval_raw, name=f"{method_key}_raw") - for metric_name in ("brier_score", "log_loss", "ece", "adaptive_ece"): - if metric_name in raw_metrics and metric_name in metrics: - metrics[f"raw_{metric_name}"] = float(raw_metrics[metric_name]) - metrics[f"phi_{metric_name}"] = float( - raw_metrics[metric_name] - metrics[metric_name] - ) - metrics["validation_holdout_ratio"] = float(validation_holdout_ratio) - metrics["validation_random_state"] = float(validation_random_state) - metrics["validation_n_fit"] = float(len(idx_fit)) - metrics["validation_n_eval"] = float(len(idx_eval)) - return method_key, {k: float(v) for k, v in metrics.items()} - - -def _acceptance_pass(policy_status: dict[str, Any], validation_cfg: dict[str, Any]) -> bool: - acceptance = dict(validation_cfg.get("acceptance", {}) or {}) - coverage = float(policy_status.get("coverage_90", 0.0)) - min_group_coverage = float(policy_status.get("min_group_coverage_90", 0.0)) - avg_width = float(policy_status.get("avg_width_90", 10.0)) - coverage_gap = abs(coverage - 0.90) - warning_alerts = int(policy_status.get("warning_alerts", 0)) - total_alerts = int(policy_status.get("total_alerts", 0)) - return bool( - policy_status.get("overall_pass", False) - and warning_alerts <= int(acceptance.get("warning_alerts_max", 5)) - and total_alerts <= int(acceptance.get("total_alerts_max", 5)) - and coverage_gap <= float(acceptance.get("coverage_deviation_90_max", 0.03)) - and min_group_coverage >= float(acceptance.get("min_group_coverage_90_min", 0.88)) - and avg_width <= float(acceptance.get("avg_width_90_max", 0.80)) - ) - - -def _policy_reason_code(policy_status: dict[str, Any], acceptance_pass: bool) -> str: - if acceptance_pass: - return "accepted_policy_gate" - if not bool(policy_status.get("overall_pass", False)): - return "policy_overall_fail" - return "acceptance_gate_fail" - - -def _phase2_should_replace_phase1( - *, - phase1_policy: dict[str, Any], - phase2_policy: dict[str, Any], - validation_cfg: dict[str, Any], -) -> tuple[bool, str]: - """Use phase2 as final only when it is a genuine decision improvement.""" - phase1_pass = _acceptance_pass(phase1_policy, validation_cfg) - phase2_pass = _acceptance_pass(phase2_policy, validation_cfg) - if not phase2_pass: - return False, "phase2_acceptance_gate_fail" - if not phase1_pass: - return True, "phase1_failed_phase2_passed" - - phase1_width = float(phase1_policy.get("avg_width_90", 10.0)) - phase2_width = float(phase2_policy.get("avg_width_90", 10.0)) - phase1_gap = abs(float(phase1_policy.get("coverage_90", 0.0)) - 0.90) - phase2_gap = abs(float(phase2_policy.get("coverage_90", 0.0)) - 0.90) - phase1_group = float(phase1_policy.get("min_group_coverage_90", 0.0)) - phase2_group = float(phase2_policy.get("min_group_coverage_90", 0.0)) - - # Keep phase2 conservative: it may replace a passing phase1 only when it is - # no worse on the claim-facing interval diagnostics, within tiny numerical - # tolerance for coverage/group metrics. - width_ok = phase2_width <= phase1_width + 1e-9 - coverage_ok = phase2_gap <= phase1_gap + 0.0025 - group_ok = phase2_group + 0.0025 >= phase1_group - if width_ok and coverage_ok and group_ok: - return True, "phase2_nondominated_vs_phase1" - return False, "phase1_retained_phase2_not_nondominated" - - -def _run_set_prediction_sidecar( - *, - namespace: str, - sidecar_cfg: dict[str, Any], - design: dict[str, Any], - env: dict[str, str], - calibrator_override_path: str | None = None, -) -> None: - methods = sidecar_cfg.get("methods", ["lac", "margin"]) - partitions = sidecar_cfg.get( - "partitions", - ["global", "grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], - ) - partition_probability_sources = list( - sidecar_cfg.get("partition_probability_sources", ["calibrated"]) or ["calibrated"] - ) - calibration_fractions = [ - float(x) - for x in (sidecar_cfg.get("calibration_size_fractions", [0.25, 0.50, 0.75, 1.0]) or []) - ] - args = [ - "--artifact_namespace", - namespace, - "--methods", - ",".join(str(x) for x in methods), - "--partitions", - ",".join(str(x) for x in partitions), - "--partition_probability_source", - str(partition_probability_sources[0]), - "--n_score_bins", - str(int(sidecar_cfg.get("n_score_bins", design["n_score_bins"]))), - "--min_group_size", - str(int(sidecar_cfg.get("min_group_size", design["min_group_size"]))), - "--fallback_mode", - str(sidecar_cfg.get("fallback_mode", design["fallback_mode"])), - "--calibration-size-fractions", - ",".join(str(float(x)) for x in calibration_fractions), - ] - if calibrator_override_path: - args.extend(["--calibrator_override_path", str(calibrator_override_path)]) - _run_python("scripts/benchmark_pd_set_prediction.py", args, env) - - -def _run_phase1_oot_candidate( - *, - run_tag: str, - rank: int, - design: dict[str, Any], - env: dict[str, str], - alpha_candidates_95: list[float], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - sidecar_cfg: dict[str, Any], - calibrator_override_path: str | None = None, - phase_prefix: str = "phase1", -) -> dict[str, Any]: - design = _normalize_design_row(design) - namespace = _namespace(run_tag, phase_prefix, "final", f"rank-{rank}") - interval_args = [ - "--artifact_namespace", - namespace, - "--evaluation_scope", - "test", - "--alpha_candidates_95", - ",".join(str(x) for x in alpha_candidates_95), - *_design_args(design), - ] - if calibrator_override_path: - interval_args.extend(["--calibrator_override_path", str(calibrator_override_path)]) - _run_python("scripts/generate_conformal_intervals.py", interval_args, env) - benchmark_args = [ - "--artifact_namespace", - namespace, - "--selected_config_path", - str(_resolve_run_paths(namespace)["results"]), - "--partition_candidates", - ",".join(str(x) for x in partition_candidates), - "--partition_probability_sources", - ",".join(str(x) for x in partition_probability_sources), - "--n_score_bins_candidates", - ",".join(str(int(x)) for x in n_score_bins_candidates), - "--fallback_modes", - ",".join(str(x) for x in fallback_modes), - "--score_scale_families", - ",".join(str(x) for x in score_scale_families), - "--min_group_size_default", - str(int(design["min_group_size"])), - "--calibration_size_fractions", - ",".join(str(float(x)) for x in calibration_fractions), - "--policy_config_path", - "configs/crpto_conformal_policy.yaml", - ] - if calibrator_override_path: - benchmark_args.extend(["--calibrator_override_path", str(calibrator_override_path)]) - _run_python("scripts/benchmark_conformal_variants.py", benchmark_args, env) - _run_python( - "scripts/validate_conformal_experiment.py", - [ - "--namespace", - namespace, - "--run-tag", - run_tag, - "--base-config", - "configs/crpto_conformal_policy.yaml", - ], - env, - ) - _run_set_prediction_sidecar( - namespace=namespace, - sidecar_cfg=sidecar_cfg, - design=design, - env=env, - calibrator_override_path=calibrator_override_path, - ) - - policy_status = _load_json(_resolve_run_paths(namespace)["policy_status"]) - set_status = _load_json(_resolve_run_paths(namespace)["set_status"]) - selection_status = _load_json(_resolve_run_paths(namespace)["selection_status"]) - return { - **design, - "namespace": namespace, - "policy_status": policy_status, - "set_status": set_status, - "selection_status": selection_status, - } - - -def _phase2_top_designs( - *, - aggregated: pd.DataFrame, - phase1_candidates_frame: pd.DataFrame | None, - top_k: int, -) -> tuple[pd.DataFrame, str]: - """Prefer OOT-confirmed phase-1 designs for phase-2 calibration search.""" - if phase1_candidates_frame is not None and not phase1_candidates_frame.empty: - top = _dedupe_designs(phase1_candidates_frame).head(int(top_k)) - if not top.empty: - top["phase2_design_source"] = "phase1_oot_confirmed" - return top, "phase1_oot_confirmed" - - top = _dedupe_designs(aggregated).head(int(top_k)) - if not top.empty: - top["phase2_design_source"] = "phase1_inner_aggregate" - return top, "phase1_inner_aggregate" - - -def _phase1_candidates_frame( - candidates: list[dict[str, Any]], - validation_cfg: dict[str, Any], -) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for candidate in candidates: - policy_status = dict(candidate["policy_status"]) - set_status = dict(candidate["set_status"]) - acceptance_pass = _acceptance_pass(policy_status, validation_cfg) - selection_status = dict(candidate.get("selection_status", {})) - rows.append( - { - **{ - key: value - for key, value in candidate.items() - if key not in {"policy_status", "set_status", "selection_status"} - }, - "policy_overall_pass": bool(policy_status.get("overall_pass", False)), - "strict_overall_pass": bool(policy_status.get("strict_overall_pass", False)), - "methodological_justification_pass": bool( - policy_status.get("methodological_justification_pass", False) - ), - "coverage_90": float(policy_status.get("coverage_90", 0.0)), - "avg_width_90": float(policy_status.get("avg_width_90", 10.0)), - "min_group_coverage_90": float(policy_status.get("min_group_coverage_90", 0.0)), - "warning_alerts": int(policy_status.get("warning_alerts", 0)), - "total_alerts": int(policy_status.get("total_alerts", 0)), - "acceptance_pass": bool(acceptance_pass), - "decision_reason_code": _policy_reason_code(policy_status, acceptance_pass), - "sidecar_set_coverage": float( - (set_status.get("summary") or {}).get("set_coverage", 0.0) - ), - "sidecar_singleton_rate": float( - (set_status.get("summary") or {}).get("singleton_rate", 0.0) - ), - "local_variant_promotion_pass": bool(selection_status.get("promotion_pass", False)), - "local_selected_variant": str(selection_status.get("selected_variant", "")), - } - ) - frame = pd.DataFrame(rows) - if frame.empty: - return frame - return frame.sort_values( - by=[ - "acceptance_pass", - "policy_overall_pass", - "coverage_90", - "avg_width_90", - "min_group_coverage_90", - ], - ascending=[False, False, False, True, False], - ).reset_index(drop=True) - - -def _extract_rank1_local_winner(source_run_tag: str) -> dict[str, Any] | None: - source_ns = _namespace(source_run_tag, "phase1", "final", "rank-1") - selection_status_path = _resolve_run_paths(source_ns)["selection_status"] - benchmark_path = ( - _resolve_run_paths(source_ns)["data_dir"] / "conformal_variant_benchmark.parquet" - ) - if not selection_status_path.exists() or not benchmark_path.exists(): - return None - selection_status = _load_json(selection_status_path) - if not bool(selection_status.get("promotion_pass", False)): - return None - bench = pd.read_parquet(benchmark_path) - if bench.empty: - return None - promotable = bench.loc[bench["promotion_pass"].fillna(False)].copy() - if promotable.empty: - promotable = bench.head(1).copy() - winner = promotable.iloc[0].to_dict() - return _benchmark_row_to_design(winner) - - -def _build_resume_shortlist( - *, - source_run_tag: str, - top_k_inner: int, -) -> tuple[pd.DataFrame, dict[str, Any]]: - source_paths = _reopen_artifact_paths(source_run_tag) - aggregate = pd.read_parquet(source_paths["inner_aggregate"]) - aggregate_unique = _dedupe_designs(aggregate) - source_seed = _normalize_design_row(aggregate_unique.iloc[0].to_dict()) - source_local_winner = _extract_rank1_local_winner(source_run_tag) - - selected: list[dict[str, Any]] = [] - seen: set[str] = set() - resume_meta = { - "source_seed_design": source_seed, - "source_local_winner_included": False, - "source_local_winner_design": source_local_winner, - } - - if source_local_winner is not None: - local_key = _semantic_candidate_key(source_local_winner) - if local_key != _semantic_candidate_key(source_seed): - selected.append(source_local_winner) - seen.add(local_key) - resume_meta["source_local_winner_included"] = True - - for row in aggregate_unique.to_dict(orient="records"): - design = _normalize_design_row(row) - key = _semantic_candidate_key(design) - if key in seen: - continue - seen.add(key) - selected.append(design) - if len(selected) >= int(top_k_inner): - break - - shortlist = pd.DataFrame(selected).reset_index(drop=True) - if not shortlist.empty: - shortlist["selection_rank"] = range(1, len(shortlist) + 1) - resume_meta["source_aggregate_unique_path"] = str(source_paths["inner_aggregate"]) - return shortlist, resume_meta - - -def _phase2_methods(phase2_cfg: dict[str, Any]) -> list[str]: - return [ - str(method).strip().lower() - for method in phase2_cfg.get("calibrators", ["venn_abers", "isotonic", "platt", "beta"]) - if str(method).strip() - ] - - -def _write_phase2_state_progress( - state: Phase2SearchState, - running: list[dict[str, Any]], -) -> None: - _write_phase2_progress( - path=state.paths["phase2_progress"], - run_tag=state.run_tag, - upstream_run_tag=state.upstream_run_tag, - total_design_runs=state.total_design_runs, - completed=state.completed, - running=running, - skipped=state.skipped, - failed=state.failed, - design_source=state.design_source, - ) - - -def _phase2_baseline_metrics( - *, - calibrator_dir: Path, - upstream_run_tag: str, -) -> dict[str, float] | None: - baseline_path = calibrator_dir / "venn_abers.pkl" - try: - _resolved, baseline_metrics = _fit_calibrator( - method="venn_abers", - output_path=baseline_path, - upstream_run_tag=upstream_run_tag, - ) - except Exception: - return None - return baseline_metrics - - -def _numeric_metric_payload(metrics: dict[str, float]) -> dict[str, float]: - return {key: float(value) for key, value in metrics.items() if isinstance(value, int | float)} - - -def _phase2_metric_blocked( - *, - calibration_metrics: dict[str, float], - baseline_metrics: dict[str, float] | None, - max_metric_degradation: dict[str, Any], -) -> bool: - if baseline_metrics is None: - return False - return any( - float(calibration_metrics.get(metric_name, float("inf"))) - > float(baseline_metrics.get(metric_name, 0.0)) - + float(max_metric_degradation.get(metric_name, 0.0)) - for metric_name in max_metric_degradation - ) - - -def _fit_phase2_calibrator_or_skip( - *, - method_name: str, - calibrator_dir: Path, - state: Phase2SearchState, - baseline_metrics: dict[str, float] | None, - max_metric_degradation: dict[str, Any], -) -> Phase2CalibratorFit | None: - calibrator_path = calibrator_dir / f"{method_name}.pkl" - _write_phase2_state_progress( - state, - running=[{"stage": "fit_calibrator", "calibrator_method": method_name}], - ) - try: - resolved_method, calibration_metrics = _fit_calibrator( - method=method_name, - output_path=calibrator_path, - upstream_run_tag=state.upstream_run_tag, - ) - except Exception as exc: - state.failed.append( - { - "stage": "fit_calibrator", - "calibrator_method": method_name, - "error": repr(exc), - } - ) - _write_phase2_state_progress(state, running=[]) - raise - - if _phase2_metric_blocked( - calibration_metrics=calibration_metrics, - baseline_metrics=baseline_metrics, - max_metric_degradation=max_metric_degradation, - ): - state.skipped.append( - { - "stage": "calibrator_gate", - "calibrator_method": resolved_method, - "reason": "metric_degradation_gate", - "calibration_metrics": _numeric_metric_payload(calibration_metrics), - } - ) - _write_phase2_state_progress(state, running=[]) - return None - - return Phase2CalibratorFit( - method=resolved_method, - path=calibrator_path, - metrics=calibration_metrics, - ) - - -def _phase2_candidate_namespace( - *, - run_tag: str, - calibrator_method: str, - design: dict[str, Any], -) -> str: - return _namespace( - run_tag, - "phase2", - calibrator_method, - f"rank-{int(design.get('selection_rank', 1))}", - ) - - -def _run_phase2_holdout_candidate( - *, - state: Phase2SearchState, - env: dict[str, str], - calibrator_fit: Phase2CalibratorFit, - design: dict[str, Any], - alpha_candidates_95: list[float], - tuning_holdout_ratios: list[float], - inner_random_states: list[int], -) -> dict[str, Any]: - design_norm = _normalize_design_row(design) - selection_rank = int(design.get("selection_rank", 1)) - namespace = _phase2_candidate_namespace( - run_tag=state.run_tag, - calibrator_method=calibrator_fit.method, - design=design, - ) - running_entry = { - "stage": "generate_intervals", - "artifact_namespace": namespace, - "calibrator_method": calibrator_fit.method, - "selection_rank": selection_rank, - } - _write_phase2_state_progress(state, running=[running_entry]) - try: - _run_python( - "scripts/generate_conformal_intervals.py", - [ - "--artifact_namespace", - namespace, - "--evaluation_scope", - "holdout", - "--calibrator_override_path", - str(calibrator_fit.path), - "--tuning_holdout_ratio", - str(float(tuning_holdout_ratios[0])), - "--tuning_random_state", - str(int(inner_random_states[0])), - "--alpha_candidates_95", - ",".join(str(x) for x in alpha_candidates_95), - *_design_args(design_norm), - ], - env, - ) - except Exception as exc: - state.failed.append({**running_entry, "error": repr(exc)}) - _write_phase2_state_progress(state, running=[]) - raise - - payload = _load_pickle(_resolve_run_paths(namespace)["results"]) - metrics_90 = dict(payload.get("metrics_90", {}) or {}) - state.completed.append( - { - "artifact_namespace": namespace, - "calibrator_method": calibrator_fit.method, - "selection_rank": selection_rank, - "coverage_90": float(metrics_90.get("empirical_coverage", 0.0)), - "avg_width_90": float(metrics_90.get("avg_interval_width", 1.0)), - } - ) - _write_phase2_state_progress(state, running=[]) - return { - "artifact_namespace": namespace, - "calibrator_method": calibrator_fit.method, - "phase2_design_source": state.design_source, - "selection_rank": selection_rank, - **design_norm, - "holdout_coverage": float(metrics_90.get("empirical_coverage", 0.0)), - "holdout_width": float(metrics_90.get("avg_interval_width", 1.0)), - "calibrator_ece": float(calibrator_fit.metrics.get("ece", float("inf"))), - "calibrator_adaptive_ece": float(calibrator_fit.metrics.get("adaptive_ece", float("inf"))), - "calibrator_brier": float(calibrator_fit.metrics.get("brier_score", float("inf"))), - "calibrator_log_loss": float(calibrator_fit.metrics.get("log_loss", float("inf"))), - "calibrator_phi_brier": float(calibrator_fit.metrics.get("phi_brier_score", 0.0)), - "calibrator_phi_log_loss": float(calibrator_fit.metrics.get("phi_log_loss", 0.0)), - } - - -def _empty_phase2_frame() -> pd.DataFrame: - return pd.DataFrame( - { - "artifact_namespace": pd.Series(dtype="object"), - "calibrator_method": pd.Series(dtype="object"), - "phase2_design_source": pd.Series(dtype="object"), - "selection_rank": pd.Series(dtype="int64"), - "holdout_coverage": pd.Series(dtype="float64"), - "holdout_width": pd.Series(dtype="float64"), - "calibrator_ece": pd.Series(dtype="float64"), - "calibrator_adaptive_ece": pd.Series(dtype="float64"), - "calibrator_brier": pd.Series(dtype="float64"), - "calibrator_log_loss": pd.Series(dtype="float64"), - "calibrator_phi_brier": pd.Series(dtype="float64"), - "calibrator_phi_log_loss": pd.Series(dtype="float64"), - } - ) - - -def _rank_phase2_candidates(phase2_df: pd.DataFrame) -> pd.DataFrame: - ranked = phase2_df.copy() - ranked["coverage_gap_abs"] = (ranked["holdout_coverage"] - 0.90).abs() - return ranked.sort_values( - by=[ - "coverage_gap_abs", - "holdout_width", - "calibrator_ece", - "calibrator_adaptive_ece", - "calibrator_brier", - "calibrator_phi_brier", - "selection_rank", - ], - ascending=[True, True, True, True, True, False, True], - ).reset_index(drop=True) - - -def _phase2_no_candidate_result( - *, - paths: dict[str, Path], - design_source: str, -) -> tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]]: - return ( - "policy_review_candidate", - {}, - {}, - { - "search_path": str(paths["phase2_search"]), - "best_candidate": {}, - "status": "no_noninferior_calibrator_candidate", - "phase2_design_source": design_source, - }, - ) - - -def _run_phase2_final_candidate( - *, - run_tag: str, - env: dict[str, str], - calibrator_dir: Path, - phase2_best: dict[str, Any], - alpha_candidates_95: list[float], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - sidecar_cfg: dict[str, Any], -) -> dict[str, Any]: - phase2_calibrator_path = calibrator_dir / f"{phase2_best['calibrator_method']}.pkl" - return _run_phase1_oot_candidate( - run_tag=run_tag, - rank=1, - design=phase2_best, - env=env, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - sidecar_cfg=sidecar_cfg, - calibrator_override_path=str(phase2_calibrator_path), - phase_prefix="phase2", - ) - - -def _run_phase2_search( - *, - run_tag: str, - upstream_run_tag: str, - env: dict[str, str], - aggregated: pd.DataFrame, - phase1_candidates_frame: pd.DataFrame | None, - alpha_candidates_95: list[float], - tuning_holdout_ratios: list[float], - inner_random_states: list[int], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - phase2_cfg: dict[str, Any], - sidecar_cfg: dict[str, Any], - validation_cfg: dict[str, Any], -) -> tuple[str, dict[str, Any], dict[str, Any], dict[str, Any] | None]: - paths = _reopen_artifact_paths(run_tag) - models_dir = paths["models_dir"] - calibrator_dir = models_dir / "phase2_calibrators" - top_designs, design_source = _phase2_top_designs( - aggregated=aggregated, - phase1_candidates_frame=phase1_candidates_frame, - top_k=int(phase2_cfg.get("top_k_designs", 3)), - ) - phase2_methods = _phase2_methods(phase2_cfg) - total_design_runs = len(phase2_methods) * len(top_designs) - state = Phase2SearchState( - paths=paths, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - design_source=design_source, - completed=[], - skipped=[], - failed=[], - ) - _write_phase2_state_progress(state, running=[]) - baseline_metrics = _phase2_baseline_metrics( - calibrator_dir=calibrator_dir, - upstream_run_tag=upstream_run_tag, - ) - - max_metric_degradation = dict(phase2_cfg.get("max_metric_degradation", {}) or {}) - calibrator_rows: list[dict[str, Any]] = [] - for method_name in phase2_methods: - calibrator_fit = _fit_phase2_calibrator_or_skip( - method_name=method_name, - calibrator_dir=calibrator_dir, - state=state, - baseline_metrics=baseline_metrics, - max_metric_degradation=max_metric_degradation, - ) - if calibrator_fit is None: - continue - - for design in top_designs.to_dict(orient="records"): - calibrator_rows.append( - _run_phase2_holdout_candidate( - state=state, - env=env, - calibrator_fit=calibrator_fit, - design=design, - alpha_candidates_95=alpha_candidates_95, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - ) - ) - - phase2_df = pd.DataFrame(calibrator_rows) - if phase2_df.empty: - phase2_df = _empty_phase2_frame() - phase2_df.to_parquet(paths["phase2_search"], index=False) - return _phase2_no_candidate_result( - paths=paths, - design_source=design_source, - ) - - phase2_df = _rank_phase2_candidates(phase2_df) - phase2_df.to_parquet(paths["phase2_search"], index=False) - phase2_best = phase2_df.iloc[0].to_dict() - final_candidate = _run_phase2_final_candidate( - run_tag=run_tag, - env=env, - calibrator_dir=calibrator_dir, - phase2_best=phase2_best, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - sidecar_cfg=sidecar_cfg, - ) - final_policy = dict(final_candidate["policy_status"]) - final_sets = dict(final_candidate["set_status"]) - final_decision = ( - "promotable_for_followup" - if _acceptance_pass(final_policy, validation_cfg) - else "policy_review_candidate" - ) - phase2_summary = { - "search_path": str(paths["phase2_search"]), - "best_candidate": phase2_best, - "final_namespace": final_candidate["namespace"], - } - return final_decision, final_policy, final_sets, phase2_summary - - -def _write_consolidated_status( - *, - run_tag: str, - upstream_run_tag: str, - pipeline_profile: str, - mode: str, - inner_search_path: str, - aggregate_path: str, - shortlist_path: str, - phase1_final_path: str, - inner_search_runs: list[dict[str, Any]], - inner_search_winner: dict[str, Any], - best_phase1_namespace: str, - final_policy: dict[str, Any], - final_sets: dict[str, Any], - final_decision: str, - final_namespace: str, - phase2_summary: dict[str, Any] | None, - resume_meta: dict[str, Any] | None, -) -> None: - status = { - "schema_version": "2026-04-05.1", - "generated_at_utc": _utc_now(), - "run_tag": run_tag, - "mode": mode, - "upstream_canonical_run_tag": upstream_run_tag, - "pipeline_profile": pipeline_profile, - "inner_search_winner": inner_search_winner, - "inner_search_path": inner_search_path, - "inner_search_aggregate_path": aggregate_path, - "phase1_shortlist_path": shortlist_path, - "phase1_final_candidates_path": phase1_final_path, - "inner_search_runs": inner_search_runs, - "phase1_oot_namespace": best_phase1_namespace, - "oot_confirmation_result": final_policy, - "sidecar_set_result": final_sets, - "promotion_decision": final_decision, - "policy_review_needed": bool(final_decision == "policy_review_candidate"), - "final_namespace": final_namespace, - "phase2": phase2_summary, - } - if resume_meta is not None: - status["resume_meta"] = resume_meta - _write_json(_reopen_artifact_paths(run_tag)["status"], status) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument("--run-tag", default=_default_run_tag()) - parser.add_argument("--pipeline-profile", default="search_conformal_reopen_exhaustive") - parser.add_argument("--upstream-canonical-run-tag", default="pd-hpo-local-2026-04-03-1325") - parser.add_argument("--phase1-only", action="store_true") - parser.add_argument("--resume-from-run-tag", default=None) - parser.add_argument( - "--phase1-workers", - type=int, - default=None, - help="Parallel workers for independent phase-1 inner conformal runs.", - ) - parser.add_argument( - "--no-resume-completed-inner", - action="store_true", - help="Recompute phase-1 inner namespaces even when tuning/results already exist.", - ) - parser.add_argument( - "--force-phase2", - action="store_true", - help="Evaluate phase-2 calibrator tournament even when phase-1 already passes.", - ) - return parser - - -def _phase1_from_resume( - *, - resume_from_run_tag: str, - top_k_inner: int, - output_paths: dict[str, Path], -) -> Phase1Result: - source_paths = _reopen_artifact_paths(resume_from_run_tag) - if not source_paths["inner_aggregate"].exists(): - raise FileNotFoundError( - f"Resume source missing aggregate artifact: {source_paths['inner_aggregate']}" - ) - shortlist, resume_meta = _build_resume_shortlist( - source_run_tag=resume_from_run_tag, - top_k_inner=top_k_inner, - ) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - aggregated = pd.read_parquet(source_paths["inner_aggregate"]) - return Phase1Result( - shortlist=shortlist, - aggregated=aggregated, - inner_runs=[], - aggregate_path=str(source_paths["inner_aggregate"]), - inner_search_path=str(source_paths["inner_search"]), - inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) - if not aggregated.empty - else {}, - resume_meta=resume_meta, - ) - - -def _run_phase1_inner_search( - *, - run_tag: str, - upstream_run_tag: str, - env: dict[str, str], - output_paths: dict[str, Path], - phase1_workers: int, - top_k_inner: int, - alpha_candidates_90: list[float], - alpha_candidates_95: list[float], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - min_group_sizes: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - tuning_holdout_ratios: list[float], - inner_random_states: list[int], - resume_completed_inner: bool, -) -> Phase1Result: - inner_frames: list[pd.DataFrame] = [] - inner_runs: list[dict[str, Any]] = [] - specs = _phase1_inner_specs( - run_tag=run_tag, - calibration_fractions=calibration_fractions, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - ) - completed_runs: list[dict[str, Any]] = [] - failed_runs: list[dict[str, Any]] = [] - running: set[str] = set() - progress_path = output_paths["phase1_progress"] - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=[], - failed=failed_runs, - workers=phase1_workers, - ) - - def record_success(tuning_df: pd.DataFrame, run_summary: dict[str, Any]) -> None: - inner_frames.append(tuning_df) - inner_runs.append(run_summary) - completed_runs.append( - { - "artifact_namespace": run_summary["artifact_namespace"], - "reused_existing": bool(run_summary.get("reused_existing", False)), - } - ) - - if phase1_workers == 1: - for spec in specs: - running = {str(spec["namespace"])} - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - tuning_df, run_summary = _run_phase1_inner_spec( - spec=spec, - env=env, - alpha_candidates_90=alpha_candidates_90, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - min_group_sizes=min_group_sizes, - score_scale_families=score_scale_families, - resume_completed=resume_completed_inner, - ) - record_success(tuning_df, run_summary) - running = set() - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=[], - failed=failed_runs, - workers=phase1_workers, - ) - else: - with ThreadPoolExecutor(max_workers=phase1_workers) as executor: - future_to_spec = {} - for spec in specs: - namespace = str(spec["namespace"]) - running.add(namespace) - future = executor.submit( - _run_phase1_inner_spec, - spec=spec, - env=env, - alpha_candidates_90=alpha_candidates_90, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - min_group_sizes=min_group_sizes, - score_scale_families=score_scale_families, - resume_completed=resume_completed_inner, - ) - future_to_spec[future] = spec - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - for future in as_completed(future_to_spec): - spec = future_to_spec[future] - namespace = str(spec["namespace"]) - running.discard(namespace) - try: - tuning_df, run_summary = future.result() - except Exception as exc: - failed_runs.append( - { - "artifact_namespace": namespace, - "error": repr(exc), - } - ) - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - raise - record_success(tuning_df, run_summary) - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - - inner_df = pd.concat(inner_frames, ignore_index=True) - inner_df.to_parquet(output_paths["inner_search"], index=False) - aggregated = _aggregate_inner_search(inner_df) - aggregated.to_parquet(output_paths["inner_aggregate"], index=False) - shortlist = _dedupe_designs(aggregated, top_k_inner) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - return Phase1Result( - shortlist=shortlist, - aggregated=aggregated, - inner_runs=inner_runs, - aggregate_path=str(output_paths["inner_aggregate"]), - inner_search_path=str(output_paths["inner_search"]), - inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) - if not aggregated.empty - else {}, - ) - - -def _run_phase1_oot_confirmation( - *, - run_tag: str, - env: dict[str, str], - shortlist: pd.DataFrame, - output_paths: dict[str, Path], - alpha_candidates_95: list[float], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - sidecar_cfg: dict[str, Any], - validation_cfg: dict[str, Any], -) -> Phase1ConfirmationResult: - if shortlist.empty: - raise RuntimeError("No phase1 shortlist candidates available for OOT confirmation.") - - phase1_candidates: list[dict[str, Any]] = [] - for rank, design in enumerate(shortlist.to_dict(orient="records"), start=1): - phase1_candidates.append( - _run_phase1_oot_candidate( - run_tag=run_tag, - rank=rank, - design=design, - env=env, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - sidecar_cfg=sidecar_cfg, - ) - ) - - phase1_df = _phase1_candidates_frame(phase1_candidates, validation_cfg) - phase1_df.to_parquet(output_paths["phase1_final_candidates"], index=False) - - best_phase1_ns = str(phase1_df.iloc[0]["namespace"]) - best_phase1 = next( - candidate for candidate in phase1_candidates if candidate["namespace"] == best_phase1_ns - ) - final_policy = dict(best_phase1["policy_status"]) - final_sets = dict(best_phase1["set_status"]) - final_decision = ( - "promotable_for_followup" - if _acceptance_pass(final_policy, validation_cfg) - else "keep_current_canonical" - ) - return Phase1ConfirmationResult( - candidates=phase1_candidates, - frame=phase1_df, - best_namespace=best_phase1_ns, - final_policy=final_policy, - final_sets=final_sets, - final_decision=final_decision, - final_namespace=str(best_phase1["namespace"]), - ) - - -def _phase2_run_reason(*, force_phase2: bool, phase2_always_evaluate: bool) -> str: - if force_phase2: - return "forced" - if phase2_always_evaluate: - return "always_evaluate" - return "phase1_acceptance_fail" - - -def _maybe_apply_phase2( - *, - run_tag: str, - upstream_run_tag: str, - env: dict[str, str], - aggregated: pd.DataFrame, - phase1: Phase1ConfirmationResult, - phase1_only: bool, - force_phase2: bool, - alpha_candidates_95: list[float], - tuning_holdout_ratios: list[float], - inner_random_states: list[int], - partition_candidates: list[str], - partition_probability_sources: list[str], - n_score_bins_candidates: list[int], - fallback_modes: list[str], - score_scale_families: list[str], - calibration_fractions: list[float], - phase2_cfg: dict[str, Any], - sidecar_cfg: dict[str, Any], - validation_cfg: dict[str, Any], -) -> PromotionResult: - final_policy = dict(phase1.final_policy) - final_sets = dict(phase1.final_sets) - final_decision = str(phase1.final_decision) - final_namespace = str(phase1.final_namespace) - phase2_summary: dict[str, Any] | None = None - - phase2_always_evaluate = bool(phase2_cfg.get("always_evaluate", False)) or bool(force_phase2) - should_run_phase2 = ( - (phase2_always_evaluate or (not _acceptance_pass(final_policy, validation_cfg))) - and (not phase1_only) - and bool(phase2_cfg.get("enabled", True)) - ) - if not should_run_phase2: - return PromotionResult( - final_policy=final_policy, - final_sets=final_sets, - final_decision=final_decision, - final_namespace=final_namespace, - phase2_summary=phase2_summary, - ) - - phase2_decision, phase2_policy, phase2_sets, phase2_summary = _run_phase2_search( - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - env=env, - aggregated=aggregated, - phase1_candidates_frame=phase1.frame, - alpha_candidates_95=alpha_candidates_95, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - phase2_cfg=phase2_cfg, - sidecar_cfg=sidecar_cfg, - validation_cfg=validation_cfg, - ) - phase2_applied, phase2_apply_reason = _phase2_should_replace_phase1( - phase1_policy=phase1.final_policy, - phase2_policy=phase2_policy, - validation_cfg=validation_cfg, - ) - if phase2_summary is not None: - phase2_summary = { - **phase2_summary, - "run_reason": _phase2_run_reason( - force_phase2=force_phase2, - phase2_always_evaluate=phase2_always_evaluate, - ), - "always_evaluate": bool(phase2_always_evaluate), - "applied_to_final": bool(phase2_applied), - "apply_reason": phase2_apply_reason, - "phase1_namespace": phase1.final_namespace, - "phase1_decision": phase1.final_decision, - "phase2_decision": phase2_decision, - } - if phase2_applied: - final_decision = phase2_decision - final_policy = phase2_policy - final_sets = phase2_sets - if phase2_summary and phase2_summary.get("final_namespace"): - final_namespace = str(phase2_summary["final_namespace"]) - elif _acceptance_pass(phase1.final_policy, validation_cfg): - final_decision = phase1.final_decision - final_policy = dict(phase1.final_policy) - final_sets = dict(phase1.final_sets) - final_namespace = phase1.final_namespace - - return PromotionResult( - final_policy=final_policy, - final_sets=final_sets, - final_decision=final_decision, - final_namespace=final_namespace, - phase2_summary=phase2_summary, - ) - - -def main(argv: list[str] | None = None) -> int: - parser = _build_parser() - args = parser.parse_args(argv) - - run_tag = str(args.run_tag).strip() - upstream_run_tag = str(args.upstream_canonical_run_tag).strip() - resume_from_run_tag = str(args.resume_from_run_tag).strip() if args.resume_from_run_tag else "" - mode = "derived_resume" if resume_from_run_tag else "fresh_search" - - profile = _profile_cfg(args.pipeline_profile) - cfg = _phase1_cfg(profile) - phase2_cfg = _phase2_cfg(profile) - sidecar_cfg = _sidecar_cfg(profile) - validation_cfg = _validation_cfg(profile) - output_paths = _reopen_artifact_paths(run_tag) - - env = os.environ.copy() - env["PIPELINE_RUN_TAG"] = run_tag - env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag - - alpha_candidates_90 = cfg.get("alpha_candidates_90", [0.09, 0.095, 0.10, 0.105, 0.11, 0.12]) - alpha_candidates_95 = cfg.get("alpha_candidates_95", [0.045, 0.05, 0.055, 0.06]) - partition_candidates = cfg.get( - "partition_candidates", - ["grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], - ) - partition_probability_sources = cfg.get("partition_probability_sources", ["calibrated", "raw"]) - n_score_bins_candidates = cfg.get("n_score_bins_candidates", [5, 10, 15, 20]) - min_group_sizes = cfg.get("min_group_sizes", [100, 150, 250, 500, 1000]) - fallback_modes = cfg.get("fallback_modes", ["grade_then_global", "global_only"]) - score_scale_families = cfg.get( - "score_scale_families", - ["none", "bernoulli_sqrt", "bernoulli_sqrt_clipped_0.02", "bernoulli_sqrt_clipped_0.05"], - ) - calibration_fractions = cfg.get("calibration_fractions", [0.25, 0.50, 0.75, 1.00]) - tuning_holdout_ratios = cfg.get("tuning_holdout_ratios", [0.20, 0.30]) - inner_random_states = cfg.get("inner_random_states", [42, 314, 2026]) - top_k_inner = int(validation_cfg.get("top_k_inner", 3)) - configured_workers = int(cfg.get("parallel_workers", 1) or 1) - phase1_workers = int(args.phase1_workers or configured_workers or 1) - phase1_workers = max(1, phase1_workers) - resume_completed_inner = not bool(args.no_resume_completed_inner) - - if resume_from_run_tag: - phase1_result = _phase1_from_resume( - resume_from_run_tag=resume_from_run_tag, - top_k_inner=top_k_inner, - output_paths=output_paths, - ) - else: - phase1_result = _run_phase1_inner_search( - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - env=env, - output_paths=output_paths, - phase1_workers=phase1_workers, - top_k_inner=top_k_inner, - alpha_candidates_90=alpha_candidates_90, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - min_group_sizes=min_group_sizes, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - resume_completed_inner=resume_completed_inner, - ) - shortlist = phase1_result.shortlist - aggregated = phase1_result.aggregated - - phase1_confirmation = _run_phase1_oot_confirmation( - run_tag=run_tag, - env=env, - shortlist=shortlist, - output_paths=output_paths, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - sidecar_cfg=sidecar_cfg, - validation_cfg=validation_cfg, - ) - final_result = _maybe_apply_phase2( - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - env=env, - aggregated=aggregated, - phase1=phase1_confirmation, - phase1_only=bool(args.phase1_only), - force_phase2=bool(args.force_phase2), - alpha_candidates_95=alpha_candidates_95, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - calibration_fractions=calibration_fractions, - phase2_cfg=phase2_cfg, - sidecar_cfg=sidecar_cfg, - validation_cfg=validation_cfg, - ) - - _write_consolidated_status( - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - pipeline_profile=str(args.pipeline_profile), - mode=mode, - inner_search_path=phase1_result.inner_search_path, - aggregate_path=phase1_result.aggregate_path, - shortlist_path=str(output_paths["phase1_shortlist"]), - phase1_final_path=str(output_paths["phase1_final_candidates"]), - inner_search_runs=phase1_result.inner_runs, - inner_search_winner=phase1_result.inner_search_winner, - best_phase1_namespace=phase1_confirmation.best_namespace, - final_policy=final_result.final_policy, - final_sets=final_result.final_sets, - final_decision=final_result.final_decision, - final_namespace=final_result.final_namespace, - phase2_summary=final_result.phase2_summary, - resume_meta=phase1_result.resume_meta, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/search/run_conformal_search.py b/scripts/search/run_conformal_search.py deleted file mode 100644 index f131742..0000000 --- a/scripts/search/run_conformal_search.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Retired conformal search entrypoint. - -The former generic ``scripts.run_long_pipeline`` orchestrator was removed when -the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a -readable stop sign for old commands instead of failing with an import error. -""" - -from __future__ import annotations - -import sys - - -def main(argv: list[str] | None = None) -> int: - _ = argv - sys.stderr.write( - "scripts/search/run_conformal_search.py is retired. Use the frozen " - "IJDS artifacts plus paper evidence stages, or start a new isolated " - "experiment under scripts/search/run_conformal_reopen_search.py with " - "an explicit run tag and drift plan.\n" - ) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/search/run_pool93_ijds_local_refinement.py b/scripts/search/run_pool93_ijds_local_refinement.py deleted file mode 100644 index eee8edb..0000000 --- a/scripts/search/run_pool93_ijds_local_refinement.py +++ /dev/null @@ -1,1745 +0,0 @@ -"""Local exact refinement of pool93 portfolio claims for IJDS. - -This stage starts from the exact pool93 surface and searches a dense local -neighborhood around three paper-facing policies: - -* max-return policy (rank 96), -* low-bound policy above the declared return floor (rank 219), -* low-miscoverage policy with stronger return (rank 223). - -Unlike the broad frontier search, this runner ranks policies using metrics -computed from the exact full-universe allocation itself. -""" - -from __future__ import annotations - -import argparse -import json -import multiprocessing as mp -import sys -import time -from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait -from dataclasses import dataclass -from datetime import UTC, datetime -from itertools import product -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 -from scripts.search.run_portfolio_bound_aware_search import ( # noqa: E402 - SCHEMA_VERSION, - SEMANTIC_POLICY_FIELDS, - _policy_semantic_key, -) -from scripts.validate_alpha_gamma_bound import ( # noqa: E402 - DEFAULT_LGD, - DEFAULT_MAX_CONCENTRATION, - DEFAULT_T_EVAL, - DEFAULT_TIME_LIMIT, - _compute_effective_pd_vector, - _compute_intervals_at_alpha, - _load_aligned_dataset, -) -from src.optimization.certificate_semantics import ( # noqa: E402 - IJDS_DECLARED_ALPHA_GRID, - add_policy_aware_bound_columns, - compute_funded_certificate_metrics, -) -from src.optimization.portfolio_model import ( # noqa: E402 - optimize_portfolio_allocation, - solution_allocation_vector, -) -from src.utils.pipeline_runtime import ( # noqa: E402 - atomic_write_json, - atomic_write_parquet, - write_runtime_checkpoint, - write_runtime_status, -) - -STAGE_NAME = "pool93_ijds_local_refinement" -DECLARED_RETURN_FLOOR = 170464.54 -DEFAULT_ALPHA_GRID = list(IJDS_DECLARED_ALPHA_GRID) -VALID_PROFILES = { - "stage1", - "expanded", - "claim_expanded", - "claim_micro", - "claim_micro_ext", - "claim_bound_closure", - "claim_bound_floor_closure", - "claim_bound_terminal", -} -ANCHOR_REASONS = ( - (96, "source_exact_max_return"), - (219, "source_low_gamma_cp_return_floor"), - (223, "source_low_weighted_miscoverage_high_return"), -) -CLAIM_ROW_FIELDS = ( - "claim_rank", - "local_candidate_id", - "local_family", - "anchor_rank", - "source_reason", - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - "alpha01_realized_total_return", - "return_floor_surplus", - "alpha01_gamma_cp", - "alpha01_gamma_internalized", - "alpha01_gamma_residual", - "alpha01_weighted_miscoverage_V", - "alpha01_endpoint_budget", - "alpha01_endpoint_budget_upper", - "alpha01_markov_loss_threshold", - "alpha01_markov_loss_cap", - "alpha01_weighted_pd_true", - "alpha01_empirical_coverage_funded", - "alpha_exact_pass_count", - "alpha_exact_check_count", - "alpha_mean_gamma_cp", - "alpha_mean_weighted_miscoverage_V", - "return_score", - "bound_score", - "v_score", - "ijds_balanced_score", - "n_funded_mean", - "allocator_backends", -) -DEFAULT_SOURCE_BOUND_EVAL = ( - ROOT / "data/processed/experiments/champion_reopen/" - "champion-reopen-2026-06-19__hpo-wave1__pool93__portfolio-stage1-fast1-claim-26-06/" - "portfolio/portfolio_bound_aware_bound_eval_highspy.parquet" -) -DEFAULT_SOURCE_SELECTION = ( - ROOT / "models/experiments/champion_reopen/" - "champion-reopen-2026-06-19__hpo-wave1__pool93__portfolio-stage1-fast1-claim-26-06/" - "portfolio/portfolio_bound_aware_selection_highspy.json" -) - - -@dataclass(frozen=True) -class Pool93Paths: - output_dir: Path - model_dir: Path - checkpoint_dir: Path - status_path: Path - candidates_path: Path - bound_eval_path: Path - leaderboard_path: Path - claim_summary_path: Path - manifest_path: Path - - -_WORKER_ALIGNED: pd.DataFrame | None = None - - -def _init_exact_worker(aligned: pd.DataFrame) -> None: - global _WORKER_ALIGNED - _WORKER_ALIGNED = aligned - - -def _exact_policy_alpha_task( - candidate: dict[str, Any], - alpha: float, - budget: float, - t_eval: float, - threads: int, -) -> dict[str, Any]: - if _WORKER_ALIGNED is None: - raise RuntimeError("Exact-refinement worker was not initialized.") - policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} - result = _exact_policy_alpha( - _WORKER_ALIGNED, - policy=policy, - alpha=float(alpha), - budget=float(budget), - t_eval=float(t_eval), - threads=max(1, int(threads)), - ) - return {**candidate, **result} - - -def _coerce_float_grid(raw: str | None, fallback: list[float]) -> list[float]: - if not raw: - return list(fallback) - values = [float(part.strip()) for part in str(raw).split(",") if part.strip()] - return values or list(fallback) - - -def _coerce_int_grid(raw: str | None, fallback: list[int]) -> list[int]: - if not raw: - return list(fallback) - values = [int(part.strip()) for part in str(raw).split(",") if part.strip()] - return values or list(fallback) - - -def _round_grid( - values: list[float], *, lo: float | None = None, hi: float | None = None -) -> list[float]: - clean: set[float] = set() - for value in values: - v = float(value) - if lo is not None: - v = max(float(lo), v) - if hi is not None: - v = min(float(hi), v) - clean.add(round(v, 6)) - return sorted(clean) - - -def _source_anchor_rows(source_bound_eval: Path, anchor_ranks: list[int]) -> pd.DataFrame: - df = pd.read_parquet(source_bound_eval) - if "alpha" in df.columns: - df = df[np.isclose(pd.to_numeric(df["alpha"], errors="coerce"), 0.01)].copy() - anchors = df[df["candidate_rank"].astype(int).isin(anchor_ranks)].copy() - if anchors.empty: - raise ValueError(f"No anchor ranks found in {source_bound_eval}: {anchor_ranks}") - anchors = anchors.drop_duplicates("candidate_rank", keep="first") - missing = sorted(set(anchor_ranks) - set(anchors["candidate_rank"].astype(int))) - if missing: - raise ValueError(f"Missing anchor ranks in {source_bound_eval}: {missing}") - return anchors.reset_index(drop=True) - - -def _policy_base( - *, - risk_tolerance: float, - policy_mode: str, - gamma: float, - uncertainty_aversion: float, - delta_cap_quantile: float = 1.0, - tail_focus_quantile: float = 1.0, - min_budget_utilization: float = 0.0, - pd_cap_slack_penalty: float = 0.0, - solver_backend: str = "highspy", -) -> dict[str, Any]: - return { - "risk_tolerance": round(float(risk_tolerance), 6), - "policy_mode": str(policy_mode), - "gamma": round(float(gamma), 6), - "delta_cap_quantile": round(float(delta_cap_quantile), 6), - "tail_focus_quantile": round(float(tail_focus_quantile), 6), - "uncertainty_aversion": round(float(uncertainty_aversion), 6), - "min_budget_utilization": round(float(min_budget_utilization), 6), - "pd_cap_slack_penalty": round(float(pd_cap_slack_penalty), 6), - "solver_backend": str(solver_backend), - } - - -def _add_candidate( - rows: list[dict[str, Any]], - seen: set[str], - *, - family: str, - anchor_rank: int, - source_reason: str, - policy: dict[str, Any], -) -> None: - key = _policy_semantic_key(policy) - if key in seen: - return - payload = dict(policy) - payload["semantic_policy_key"] = key - payload["local_family"] = family - payload["anchor_rank"] = int(anchor_rank) - payload["source_reason"] = str(source_reason) - rows.append(payload) - seen.add(key) - - -def _candidate_frame(rows: list[dict[str, Any]]) -> pd.DataFrame: - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", np.arange(1, len(candidates) + 1, dtype=int)) - return candidates - - -def _anchor_policy( - anchor_by_rank: dict[int, dict[str, Any]], - rank: int, - *, - solver_backend: str, -) -> dict[str, Any]: - row = anchor_by_rank[rank] - return _policy_base( - risk_tolerance=float(row["risk_tolerance"]), - policy_mode=str(row["policy_mode"]), - gamma=float(row["gamma"]), - uncertainty_aversion=float(row["uncertainty_aversion"]), - delta_cap_quantile=float(row["delta_cap_quantile"]), - tail_focus_quantile=float(row["tail_focus_quantile"]), - min_budget_utilization=float(row["min_budget_utilization"]), - pd_cap_slack_penalty=float(row["pd_cap_slack_penalty"]), - solver_backend=solver_backend, - ) - - -def _add_anchor_candidates( - rows: list[dict[str, Any]], - seen: set[str], - *, - anchor_by_rank: dict[int, dict[str, Any]], - solver_backend: str, -) -> None: - for rank, reason in ANCHOR_REASONS: - if rank in anchor_by_rank: - _add_candidate( - rows, - seen, - family="anchor_policy", - anchor_rank=rank, - source_reason=reason, - policy=_anchor_policy(anchor_by_rank, rank, solver_backend=solver_backend), - ) - - -def _capped_delta_values(mode: str, capped_values: tuple[float, ...]) -> tuple[float, ...]: - return (1.0,) if mode == "blended_uncertainty" else capped_values - - -def _add_blended_grid( - rows: list[dict[str, Any]], - seen: set[str], - *, - family: str, - anchor_rank: int, - source_reason: str, - risks: list[float], - gammas: list[float], - aversions: list[float], - solver_backend: str, - capped_delta_values: tuple[float, ...] = (0.95, 1.0), -) -> None: - for risk, gamma, aversion, mode in product( - risks, - gammas, - aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - for delta_cap in _capped_delta_values(str(mode), capped_delta_values): - _add_candidate( - rows, - seen, - family=family, - anchor_rank=anchor_rank, - source_reason=source_reason, - policy=_policy_base( - risk_tolerance=risk, - policy_mode=str(mode), - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - -def _add_tail_grid( - rows: list[dict[str, Any]], - seen: set[str], - *, - family: str, - anchor_rank: int, - source_reason: str, - risks: list[float], - gammas: list[float], - aversions: list[float], - tail_focus_values: list[float], - solver_backend: str, -) -> None: - for risk, gamma, aversion, tail_focus in product( - risks, - gammas, - aversions, - tail_focus_values, - ): - _add_candidate( - rows, - seen, - family=family, - anchor_rank=anchor_rank, - source_reason=source_reason, - policy=_policy_base( - risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, - solver_backend=solver_backend, - ), - ) - - -def _append_claim_micro_candidates( - rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str -) -> None: - _add_blended_grid( - rows, - seen, - family="claim_micro_body_low_v", - anchor_rank=219, - source_reason="candidate1665_1667_body_default_micro", - risks=_round_grid([0.1715 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), - gammas=_round_grid([0.545 + 0.005 * idx for idx in range(7)], lo=0.0, hi=1.0), - aversions=[0.0, 0.0125, 0.025, 0.0375, 0.05, 0.0625, 0.075, 0.10], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_blended_grid( - rows, - seen, - family="claim_micro_bound_tight", - anchor_rank=219, - source_reason="candidate1206_tight_cap_micro", - risks=_round_grid([0.1700 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), - gammas=_round_grid([0.575 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), - aversions=[0.15, 0.1625, 0.175, 0.1875, 0.20, 0.225], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_blended_grid( - rows, - seen, - family="claim_micro_high_return_cap036", - anchor_rank=219, - source_reason="candidate1922_return_cap036_micro", - risks=_round_grid([0.1725 + 0.00025 * idx for idx in range(7)], lo=0.14, hi=0.24), - gammas=_round_grid([0.500 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), - aversions=[0.05, 0.0625, 0.075, 0.0875, 0.10], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_tail_grid( - rows, - seen, - family="claim_micro_economic_endpoint", - anchor_rank=96, - source_reason="candidate2777_2857_economic_endpoint_micro", - risks=_round_grid([0.1565 + 0.00025 * idx for idx in range(9)], lo=0.12, hi=0.22), - gammas=_round_grid([0.44 + 0.005 * idx for idx in range(13)], lo=0.0, hi=1.0), - aversions=[0.1125, 0.125, 0.1375, 0.15], - tail_focus_values=[0.95, 1.0], - solver_backend=solver_backend, - ) - - -def _append_claim_micro_ext_candidates( - rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str -) -> None: - _add_blended_grid( - rows, - seen, - family="claim_micro_ext_body_cap345", - anchor_rank=219, - source_reason="candidate37_205_body_cap345_extension", - risks=_round_grid([0.17125 + 0.000125 * idx for idx in range(11)], lo=0.14, hi=0.24), - gammas=_round_grid([0.5475, 0.55, 0.5525, 0.555, 0.5575], lo=0.0, hi=1.0), - aversions=[0.025, 0.0375, 0.05, 0.0625], - capped_delta_values=(0.975, 1.0), - solver_backend=solver_backend, - ) - _add_blended_grid( - rows, - seen, - family="claim_micro_ext_bound_tight", - anchor_rank=219, - source_reason="candidate949_bound_tight_extension", - risks=_round_grid([0.1690 + 0.00025 * idx for idx in range(8)], lo=0.14, hi=0.24), - gammas=_round_grid([0.600 + 0.005 * idx for idx in range(11)], lo=0.0, hi=1.0), - aversions=[0.2125, 0.225, 0.2375, 0.25, 0.2625, 0.275], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_blended_grid( - rows, - seen, - family="claim_micro_ext_cap036_return", - anchor_rank=219, - source_reason="candidate1975_cap036_return_extension", - risks=_round_grid([0.17375 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), - gammas=_round_grid([0.505 + 0.0025 * idx for idx in range(9)], lo=0.0, hi=1.0), - aversions=[0.0625, 0.075, 0.0875, 0.10], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_tail_grid( - rows, - seen, - family="claim_micro_ext_economic_endpoint", - anchor_rank=96, - source_reason="candidate2122_economic_endpoint_extension", - risks=_round_grid([0.15625 + 0.000125 * idx for idx in range(9)], lo=0.12, hi=0.22), - gammas=_round_grid([0.400 + 0.005 * idx for idx in range(10)], lo=0.0, hi=1.0), - aversions=[0.125, 0.1375, 0.15], - tail_focus_values=[0.90, 0.925, 0.95, 1.0], - solver_backend=solver_backend, - ) - - -def _append_bound_closure_candidates( - rows: list[dict[str, Any]], - seen: set[str], - *, - profile: str, - solver_backend: str, -) -> None: - if profile == "claim_bound_closure": - _add_blended_grid( - rows, - seen, - family="claim_bound_closure_low_cap", - anchor_rank=219, - source_reason="micro_ext_min_markov_cap_endpoint_closure", - risks=_round_grid([0.1685 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), - gammas=_round_grid([0.65 + 0.01 * idx for idx in range(11)], lo=0.0, hi=1.0), - aversions=[0.25, 0.275, 0.30, 0.325, 0.35], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - return - if profile == "claim_bound_floor_closure": - _add_blended_grid( - rows, - seen, - family="claim_bound_floor_closure_low_cap", - anchor_rank=219, - source_reason="bound_closure_cap029_floor_threshold", - risks=_round_grid([0.16775 + 0.000125 * idx for idx in range(13)], lo=0.14, hi=0.24), - gammas=_round_grid([0.75 + 0.01 * idx for idx in range(10)], lo=0.0, hi=1.0), - aversions=[0.325, 0.35, 0.375, 0.40, 0.425, 0.45], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - - -def _append_bound_terminal_candidates( - rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str -) -> None: - _add_blended_grid( - rows, - seen, - family="claim_bound_terminal_ultra_low_cap", - anchor_rank=219, - source_reason="terminal_cap_threshold_search", - risks=_round_grid([0.16675 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), - gammas=_round_grid([0.84 + 0.005 * idx for idx in range(31)], lo=0.0, hi=1.0), - aversions=[0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60, 0.65, 0.70], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_blended_grid( - rows, - seen, - family="claim_bound_terminal_return_recovery", - anchor_rank=219, - source_reason="terminal_best_return_under_low_cap", - risks=_round_grid([0.1680 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), - gammas=_round_grid([0.80 + 0.005 * idx for idx in range(25)], lo=0.0, hi=1.0), - aversions=[0.35, 0.375, 0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - - -def _append_max_return_neighborhood( - rows: list[dict[str, Any]], - seen: set[str], - *, - anchor_by_rank: dict[int, dict[str, Any]], - profile: str, - solver_backend: str, -) -> None: - if 96 not in anchor_by_rank: - return - base = anchor_by_rank[96] - risk_offsets = [-0.004, -0.002, -0.001, 0.0, 0.001, 0.002, 0.004] - gamma_offsets = [-0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - if profile == "expanded": - risk_offsets = [ - -0.006, - -0.004, - -0.003, - -0.002, - -0.001, - 0.0, - 0.001, - 0.002, - 0.003, - 0.004, - 0.006, - ] - gamma_offsets = [-0.075, -0.05, -0.035, -0.025, -0.01, 0.0, 0.01, 0.025, 0.035, 0.05, 0.075] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - risks = _round_grid([float(base["risk_tolerance"]) + x for x in risk_offsets], lo=0.12, hi=0.22) - gammas = _round_grid([float(base["gamma"]) + x for x in gamma_offsets], lo=0.0, hi=1.0) - - for risk, gamma, aversion in product(risks, gammas, aversions): - _add_candidate( - rows, - seen, - family="max_return_segment_relative_local", - anchor_rank=96, - source_reason="rank96_local_dense", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="segment_relative_tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - solver_backend=solver_backend, - ), - ) - - _add_tail_grid( - rows, - seen, - family="max_return_tail_local", - anchor_rank=96, - source_reason="rank96_tail_sensitivity", - risks=risks[1:-1] if len(risks) > 2 else risks, - gammas=gammas[1:-1] if len(gammas) > 2 else gammas, - aversions=[0.075, 0.10, 0.125] if profile == "stage1" else aversions, - tail_focus_values=[0.95, 1.0] if profile == "stage1" else [0.90, 0.95, 1.0], - solver_backend=solver_backend, - ) - - -def _append_bound_neighborhood( - rows: list[dict[str, Any]], - seen: set[str], - *, - anchor_by_rank: dict[int, dict[str, Any]], - profile: str, - solver_backend: str, -) -> None: - risk_centers = [] - gamma_centers = [] - for rank in (219, 223): - if rank in anchor_by_rank: - row = anchor_by_rank[rank] - risk_centers.append(float(row["risk_tolerance"])) - gamma_centers.append(float(row["gamma"])) - if not risk_centers: - return - risk_offsets = [-0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005] - gamma_offsets = [-0.05, -0.025, 0.0, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - capped_delta_values: tuple[float, ...] = (1.0,) - if profile == "expanded": - risk_offsets = [-0.01, -0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005, 0.0075, 0.01] - gamma_offsets = [-0.075, -0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05, 0.075] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - capped_delta_values = (0.90, 1.0) - risks = _round_grid( - [center + offset for center in risk_centers for offset in risk_offsets], lo=0.14, hi=0.24 - ) - gammas = _round_grid( - [center + offset for center in gamma_centers for offset in gamma_offsets], lo=0.0, hi=1.0 - ) - for risk, gamma, aversion, mode in product( - risks, - gammas, - aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - for delta_cap in _capped_delta_values(str(mode), capped_delta_values): - _add_candidate( - rows, - seen, - family="bound_efficient_local", - anchor_rank=219 if abs(gamma - 0.45) <= abs(gamma - 0.40) else 223, - source_reason="rank219_rank223_bound_frontier", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=str(mode), - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - -def _append_claim_expanded_candidates( - rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str -) -> None: - _add_blended_grid( - rows, - seen, - family="bound_claim_refined_local", - anchor_rank=219, - source_reason="candidate462_466_return_bound_ridge", - risks=_round_grid( - [0.1705 + 0.0005 * idx for idx in range(10)] + [0.1750], lo=0.14, hi=0.24 - ), - gammas=_round_grid([0.49, 0.50, 0.51, 0.52, 0.535, 0.55, 0.575], lo=0.0, hi=1.0), - aversions=[0.05, 0.075, 0.10, 0.1125, 0.125, 0.1375, 0.15, 0.175], - capped_delta_values=(0.95, 1.0), - solver_backend=solver_backend, - ) - _add_tail_grid( - rows, - seen, - family="max_return_claim_refined_local", - anchor_rank=96, - source_reason="candidate264_economic_frontier_endpoint", - risks=_round_grid([0.1560 + 0.0005 * idx for idx in range(9)], lo=0.12, hi=0.22), - gammas=_round_grid([0.44, 0.45, 0.46, 0.47, 0.475, 0.485, 0.495], lo=0.0, hi=1.0), - aversions=[0.10, 0.1125, 0.125, 0.1375, 0.15], - tail_focus_values=[0.95, 1.0], - solver_backend=solver_backend, - ) - - -def _generate_candidate_grid( - anchors: pd.DataFrame, - *, - profile: str, - solver_backend: str, -) -> pd.DataFrame: - profile = str(profile).strip().lower() - if profile not in VALID_PROFILES: - raise ValueError(f"profile must be one of {sorted(VALID_PROFILES)}") - - anchor_by_rank = {int(row["candidate_rank"]): row for row in anchors.to_dict(orient="records")} - rows: list[dict[str, Any]] = [] - seen: set[str] = set() - _add_anchor_candidates( - rows, - seen, - anchor_by_rank=anchor_by_rank, - solver_backend=solver_backend, - ) - - if profile == "claim_micro": - _append_claim_micro_candidates(rows, seen, solver_backend=solver_backend) - return _candidate_frame(rows) - if profile == "claim_micro_ext": - _append_claim_micro_ext_candidates(rows, seen, solver_backend=solver_backend) - return _candidate_frame(rows) - if profile in {"claim_bound_closure", "claim_bound_floor_closure"}: - _append_bound_closure_candidates( - rows, - seen, - profile=profile, - solver_backend=solver_backend, - ) - return _candidate_frame(rows) - if profile == "claim_bound_terminal": - _append_bound_terminal_candidates(rows, seen, solver_backend=solver_backend) - return _candidate_frame(rows) - - _append_max_return_neighborhood( - rows, - seen, - anchor_by_rank=anchor_by_rank, - profile=profile, - solver_backend=solver_backend, - ) - _append_bound_neighborhood( - rows, - seen, - anchor_by_rank=anchor_by_rank, - profile=profile, - solver_backend=solver_backend, - ) - if profile == "claim_expanded": - _append_claim_expanded_candidates(rows, seen, solver_backend=solver_backend) - return _candidate_frame(rows) - - -def _exact_policy_alpha( - aligned: pd.DataFrame, - *, - policy: dict[str, Any], - alpha: float, - budget: float, - t_eval: float, - threads: int, -) -> dict[str, Any]: - pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) - y_true = ( - pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0).to_numpy(dtype=float) - if "y_true" in aligned.columns - else pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=float) - ) - default_flag = ( - pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) - if "default_flag" in aligned.columns - else y_true.astype(int) - ) - effective_pd = _compute_effective_pd_vector(aligned, pd_point, pd_high, policy) - int_rates = ( - _parse_percent_series(aligned["int_rate"]) - if "int_rate" in aligned.columns - else np.full(len(aligned), 0.12) - ) - loan_amounts = ( - pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in aligned.columns - else np.ones(len(aligned), dtype=float) - ) - lgd = np.full(len(aligned), DEFAULT_LGD, dtype=float) - solution = optimize_portfolio_allocation( - loans=aligned, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - total_budget=float(budget), - max_concentration=DEFAULT_MAX_CONCENTRATION, - max_portfolio_pd=float(policy["risk_tolerance"]), - robust=True, - uncertainty_aversion=float(policy["uncertainty_aversion"]), - min_budget_utilization=float(policy["min_budget_utilization"]), - pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), - pd_constraint_override=effective_pd, - time_limit=DEFAULT_TIME_LIMIT, - threads=max(1, int(threads)), - solver_backend=str(policy["solver_backend"]), - ) - alloc = solution_allocation_vector(solution, len(aligned)) - total_allocated = float(np.sum(alloc * loan_amounts)) - weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) - certificate = compute_funded_certificate_metrics( - weights, - outcomes=y_true, - pd_point=pd_point, - pd_high=pd_high, - pd_effective=effective_pd, - alpha=alpha, - risk_tolerance=float(policy["risk_tolerance"]), - pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), - ) - realized_total_return = float( - np.sum( - np.where( - (alloc > 0.01) & (default_flag.astype(int) == 1), - alloc * loan_amounts * (-DEFAULT_LGD), - np.where(alloc > 0.01, alloc * loan_amounts * int_rates, 0.0), - ) - ) - ) - expected_return_gross = float(np.sum(alloc * loan_amounts * int_rates)) - expected_loss_point = float(np.sum(alloc * loan_amounts * pd_point * lgd)) - expected_return_net_point = expected_return_gross - expected_loss_point - pd_cap_slack = float(solution.get("pd_cap_slack", 0.0)) - risk_excess = round(certificate.realized_risk_tolerance_excess, 6) - empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) - markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) - - return { - "alpha": float(alpha), - "confidence": float(1.0 - alpha), - "gamma_cp": round(certificate.gamma_cp, 6), - "gamma_internalized": round(certificate.gamma_internalized, 6), - "gamma_residual": round(certificate.gamma_residual, 6), - "n_funded": int(solution.get("n_funded", int(np.sum(alloc > 0.01)))), - "total_allocated": round(total_allocated, 2), - "objective_value": round(float(solution.get("objective_value", 0.0)), 6), - "expected_return_gross": round(expected_return_gross, 6), - "expected_loss_point": round(expected_loss_point, 6), - "expected_return_net_point": round(expected_return_net_point, 6), - "realized_total_return": round(realized_total_return, 6), - "weighted_pd_true": round(certificate.weighted_outcome, 6), - "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), - "weighted_pd_high": round(certificate.endpoint_budget, 6), - "weighted_pd_point": round(certificate.weighted_pd_point, 6), - "worst_case_pd": round(certificate.endpoint_budget, 6), - "point_pd": round(certificate.weighted_pd_point, 6), - "endpoint_budget": round(certificate.endpoint_budget, 9), - "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), - "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), - "markov_loss_cap": round(certificate.markov_loss_cap, 9), - "tau": float(policy["risk_tolerance"]), - "realized_risk_tolerance_excess": risk_excess, - "violation": risk_excess, - "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), - "weighted_coverage_funded": round(certificate.weighted_coverage, 6), - "sqrt_alpha": round(certificate.sqrt_alpha, 6), - "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), - "empirical_risk_excess_leq_alpha": empirical_risk_screen, - "bound_a_expected_violation_leq_alpha": empirical_risk_screen, - "bound_b_prob_violation_gt_t": round(float(min(1.0, alpha / max(t_eval, 1e-8))), 4), - "bound_b_t_eval": float(t_eval), - "bound_b_is_vacuous": bool(min(1.0, alpha / max(t_eval, 1e-8)) >= 1.0), - "markov_miscoverage_screen_pass": markov_screen, - "bound_c_V_leq_sqrt_alpha": markov_screen, - "certificate_screen_pass": empirical_risk_screen and markov_screen, - "all_bounds_hold": empirical_risk_screen and markov_screen, - "allocator_mode": "exact", - "solver_status": str(solution.get("solver_status", "unknown")), - "allocator_solver_backend": str(solution.get("solver_backend", policy["solver_backend"])), - "allocator_native_solver_error": str(solution.get("native_solver_error", "")), - "pd_cap_slack": round(pd_cap_slack, 6), - } - - -def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) -> pd.DataFrame: - if bound_eval.empty: - return candidates.copy() - bound_eval = add_policy_aware_bound_columns(bound_eval) - grouped = bound_eval.groupby("local_candidate_id", dropna=False) - agg = grouped.agg( - alpha_exact_pass_count=("all_bounds_hold", "sum"), - alpha_exact_check_count=("all_bounds_hold", "size"), - alpha_exact_pass_rate=("all_bounds_hold", "mean"), - alpha_max_realized_risk_tolerance_excess=( - "realized_risk_tolerance_excess", - "max", - ), - alpha_max_violation=("realized_risk_tolerance_excess", "max"), - alpha_mean_gamma_cp=("gamma_cp", "mean"), - alpha_mean_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha_mean_weighted_pd_true=("weighted_pd_true", "mean"), - alpha_mean_empirical_coverage_funded=("empirical_coverage_funded", "mean"), - exact_return_mean=("realized_total_return", "mean"), - exact_return_max=("realized_total_return", "max"), - exact_return_min=("realized_total_return", "min"), - exact_expected_return_net_point_mean=("expected_return_net_point", "mean"), - n_funded_mean=("n_funded", "mean"), - total_allocated_mean=("total_allocated", "mean"), - allocator_backends=( - "allocator_solver_backend", - lambda s: ",".join(sorted(set(map(str, s)))), - ), - ).reset_index() - alpha01 = ( - bound_eval[np.isclose(bound_eval["alpha"], 0.01)] - .groupby("local_candidate_id", dropna=False) - .agg( - alpha01_exact_pass=("all_bounds_hold", "all"), - alpha01_realized_total_return=("realized_total_return", "mean"), - alpha01_gamma_cp=("gamma_cp", "mean"), - alpha01_gamma_internalized=("gamma_internalized", "mean"), - alpha01_gamma_residual=("gamma_residual", "mean"), - alpha01_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha01_realized_risk_tolerance_excess=( - "realized_risk_tolerance_excess", - "max", - ), - alpha01_violation=("realized_risk_tolerance_excess", "max"), - alpha01_weighted_pd_true=("weighted_pd_true", "mean"), - alpha01_weighted_pd_constraint_used=("weighted_pd_constraint_used", "mean"), - alpha01_weighted_pd_high=("weighted_pd_high", "mean"), - alpha01_weighted_pd_point=("weighted_pd_point", "mean"), - alpha01_endpoint_budget=("endpoint_budget", "mean"), - alpha01_endpoint_budget_upper=("endpoint_budget_upper", "mean"), - alpha01_markov_loss_threshold=("markov_loss_threshold", "mean"), - alpha01_markov_loss_cap=("markov_loss_cap", "mean"), - alpha01_empirical_coverage_funded=("empirical_coverage_funded", "mean"), - alpha01_n_funded=("n_funded", "mean"), - ) - .reset_index() - ) - work = candidates.merge(agg, on="local_candidate_id", how="left") - work = work.merge(alpha01, on="local_candidate_id", how="left") - for col in ["alpha01_exact_pass"]: - if col in work: - work[col] = work[col].fillna(False).infer_objects(copy=False).astype(bool) - work["all_alpha_pass"] = work["alpha_exact_pass_count"].fillna(0) >= work[ - "alpha_exact_check_count" - ].fillna(1) - work["return_floor_surplus"] = ( - work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR - ) - work = work.sort_values( - by=[ - "alpha01_exact_pass", - "all_alpha_pass", - "alpha_exact_pass_count", - "alpha01_realized_total_return", - "alpha01_weighted_miscoverage_V", - "alpha01_gamma_cp", - ], - ascending=[False, False, False, False, True, True], - kind="mergesort", - ).reset_index(drop=True) - work.insert(0, "claim_rank", np.arange(1, len(work) + 1, dtype=int)) - return work - - -def _ensure_claim_summary_columns(leaderboard: pd.DataFrame) -> pd.DataFrame: - work = leaderboard.copy() - if "alpha01_endpoint_budget" not in work.columns and "alpha01_weighted_pd_high" in work.columns: - work["alpha01_endpoint_budget"] = pd.to_numeric( - work["alpha01_weighted_pd_high"], errors="coerce" - ) - if "alpha01_endpoint_budget_upper" not in work.columns: - if { - "alpha01_weighted_pd_high", - "alpha01_weighted_pd_constraint_used", - }.issubset(work.columns): - residual = pd.to_numeric( - work["alpha01_weighted_pd_high"], errors="coerce" - ) - pd.to_numeric(work["alpha01_weighted_pd_constraint_used"], errors="coerce") - work["alpha01_gamma_residual"] = residual.clip(lower=0.0) - work["alpha01_endpoint_budget_upper"] = pd.to_numeric( - work["risk_tolerance"], errors="coerce" - ) + residual.clip(lower=0.0) - else: - alpha01_gamma = pd.to_numeric(work["alpha01_gamma_cp"], errors="coerce") - risk = pd.to_numeric(work["risk_tolerance"], errors="coerce") - gamma = pd.to_numeric(work["gamma"], errors="coerce") - work["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma - if "alpha01_markov_loss_threshold" not in work.columns and "alpha01_endpoint_budget" in work: - work["alpha01_markov_loss_threshold"] = pd.to_numeric( - work["alpha01_endpoint_budget"], errors="coerce" - ) + float(np.sqrt(0.01)) - if "alpha01_markov_loss_cap" not in work.columns: - work["alpha01_markov_loss_cap"] = pd.to_numeric( - work["alpha01_endpoint_budget_upper"], errors="coerce" - ) + float(np.sqrt(0.01)) - if "return_floor_surplus" not in work.columns: - if "champion_return_surplus" in work.columns: - work["return_floor_surplus"] = work["champion_return_surplus"] - else: - work["return_floor_surplus"] = ( - work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR - ) - return work - - -def _all_alpha_eligible(frame: pd.DataFrame) -> pd.DataFrame: - return frame[ - frame["alpha01_exact_pass"].fillna(False).astype(bool) - & frame["all_alpha_pass"].fillna(False).astype(bool) - ].copy() - - -def _row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: - if frame.empty: - return None - row = frame.iloc[0] - return { - field: row[field].item() if hasattr(row[field], "item") else row[field] - for field in CLAIM_ROW_FIELDS - if field in row.index - } - - -def _add_normalized_score( - frame: pd.DataFrame, - *, - source: str, - target: str, - higher_better: bool, -) -> None: - vals = pd.to_numeric(frame[source], errors="coerce") - lo, hi = float(vals.min()), float(vals.max()) - if hi <= lo: - frame[target] = 1.0 - elif higher_better: - frame[target] = (vals - lo) / (hi - lo) - else: - frame[target] = (hi - vals) / (hi - lo) - - -def _balanced_claim_candidates(above_return_floor: pd.DataFrame) -> pd.DataFrame: - balanced = above_return_floor.copy() - if balanced.empty: - return balanced - for source, target, higher_better in [ - ("alpha01_realized_total_return", "return_score", True), - ("alpha01_markov_loss_cap", "bound_score", False), - ("alpha01_weighted_miscoverage_V", "v_score", False), - ]: - _add_normalized_score( - balanced, - source=source, - target=target, - higher_better=higher_better, - ) - balanced["ijds_balanced_score"] = ( - 0.40 * balanced["return_score"] - + 0.40 * balanced["bound_score"] - + 0.20 * balanced["v_score"] - ) - return balanced - - -def _family_claim_summary(leaderboard: pd.DataFrame) -> dict[str, Any]: - by_family: dict[str, Any] = {} - if leaderboard.empty: - return by_family - for family, frame in leaderboard.groupby("local_family", dropna=False): - fam_eligible = _all_alpha_eligible(frame) - fam_above_floor = fam_eligible[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ] - by_family[str(family)] = { - "n_policies": int(len(frame)), - "n_all_alpha_passers": int(len(fam_eligible)), - "all_alpha_pass_rate": float(len(fam_eligible) / max(len(frame), 1)), - "best_return": float(fam_eligible["alpha01_realized_total_return"].max()) - if not fam_eligible.empty - else None, - "min_gamma_cp_above_return_floor": float(fam_above_floor["alpha01_gamma_cp"].min()) - if not fam_above_floor.empty - else None, - "min_v_above_return_floor": float( - fam_above_floor["alpha01_weighted_miscoverage_V"].min() - ) - if not fam_above_floor.empty - else None, - } - return by_family - - -def _alpha_claim_summary(bound_eval: pd.DataFrame) -> dict[str, Any]: - by_alpha: dict[str, Any] = {} - if bound_eval.empty: - return by_alpha - risk_excess_column = ( - "realized_risk_tolerance_excess" - if "realized_risk_tolerance_excess" in bound_eval.columns - else "violation" - ) - for alpha, frame in bound_eval.groupby("alpha", dropna=False): - alpha_value = float(str(alpha)) - by_alpha[str(alpha_value)] = { - "n_checks": int(len(frame)), - "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), - "max_realized_risk_tolerance_excess": float(frame[risk_excess_column].max()), - "max_violation": float(frame[risk_excess_column].max()), - "mean_gamma_cp": float(frame["gamma_cp"].mean()), - "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), - } - return by_alpha - - -def _claim_alpha_values(bound_eval: pd.DataFrame, alpha_grid: list[float] | None) -> list[float]: - if alpha_grid is not None: - values = [float(value) for value in alpha_grid] - elif "alpha" in bound_eval: - values = sorted(float(value) for value in bound_eval["alpha"].dropna().unique()) - else: - values = list(DEFAULT_ALPHA_GRID) - return sorted(dict.fromkeys(values)) - - -def _finite_grid_policy(alpha_values: list[float]) -> dict[str, Any]: - return { - "alpha_grid": alpha_values, - "alpha_grid_size": int(len(alpha_values)), - "alpha_grid_semantics": ( - "finite_predeclared_grid; all-alpha pass counts only the listed levels " - "and is not a universal alpha or conditional-coverage guarantee" - ), - "region_semantics": ( - "finite_policy_grid; candidate denominators describe evaluated policies, " - "not a continuous robust region" - ), - } - - -def _claim_selection_protocol() -> dict[str, Any]: - return { - "body_default": "balanced_return_bound_claim", - "frontier_endpoints": [ - "max_return_claim", - "best_gamma_cp_return_floor_claim", - "best_weighted_miscoverage_return_floor_claim", - ], - "required_filters": [ - "alpha01_exact_pass", - "all_alpha_pass over the finite alpha_grid", - "return_floor_surplus >= 0 for declared-return-floor claims", - ], - "balanced_score": ( - "0.40 * normalized return surplus + 0.40 * normalized inverse " - "alpha01 Markov loss cap + 0.20 * normalized inverse " - "weighted_miscoverage_V among all-alpha above-return-floor candidates" - ), - "promotion_caveat": ( - "Realized-return maxima are frontier endpoints; paper promotion should " - "prefer a declared return-bound lens unless the manuscript explicitly " - "frames the point as an economic endpoint." - ), - } - - -def _claim_summary( - leaderboard: pd.DataFrame, - bound_eval: pd.DataFrame, - *, - alpha_grid: list[float] | None = None, -) -> dict[str, Any]: - leaderboard = _ensure_claim_summary_columns(leaderboard) - eligible = _all_alpha_eligible(leaderboard) - above_return_floor = eligible[ - eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ].copy() - - max_return = _row_payload( - eligible.sort_values("alpha01_realized_total_return", ascending=False) - ) - best_gamma = _row_payload( - above_return_floor.sort_values( - ["alpha01_gamma_cp", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) - best_v = _row_payload( - above_return_floor.sort_values( - ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) - balanced = _balanced_claim_candidates(above_return_floor) - balanced_claim = _row_payload( - balanced.sort_values("ijds_balanced_score", ascending=False) - if not balanced.empty - else balanced - ) - alpha_values = _claim_alpha_values(bound_eval, alpha_grid) - - return { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "declared_return_floor": DECLARED_RETURN_FLOOR, - "finite_grid_policy": _finite_grid_policy(alpha_values), - "claim_selection_protocol": _claim_selection_protocol(), - "n_policies": int(len(leaderboard)), - "n_all_alpha_passers": int(len(eligible)), - "n_all_alpha_passers_above_return_floor": int(len(above_return_floor)), - "max_return_claim": max_return, - "best_gamma_cp_return_floor_claim": best_gamma, - "best_weighted_miscoverage_return_floor_claim": best_v, - "balanced_return_bound_claim": balanced_claim, - "by_family": _family_claim_summary(leaderboard), - "by_alpha": _alpha_claim_summary(bound_eval), - "interpretation": { - "max_return_claim": "Use when the paper emphasizes certified economic return.", - "best_gamma_cp_return_floor_claim": "Use when the paper emphasizes a tighter conformal robustness budget while preserving the declared return floor.", - "best_weighted_miscoverage_return_floor_claim": "Use when the paper emphasizes lower realized weighted miscoverage while preserving the declared return floor.", - "balanced_return_bound_claim": "Use as the default IJDS narrative if return and bound should be presented as a frontier rather than a single leaderboard point.", - }, - } - - -def _write_status( - *, - run_tag: str, - status_path: Path, - start_monotonic: float, - completed: int, - total: int, - phase: str, - state: str, - initial_completed: int = 0, - extra: dict[str, Any] | None = None, -) -> None: - elapsed = time.monotonic() - start_monotonic - remaining = max(0, int(total) - int(completed)) - completed_this_run = max(0, int(completed) - int(initial_completed)) - eta = ( - (elapsed / completed_this_run) * remaining - if completed_this_run > 0 and remaining > 0 - else (0.0 if remaining == 0 else None) - ) - payload: dict[str, Any] = { - "total_checks": int(total), - "completed_checks": int(completed), - "completed_checks_at_start": int(initial_completed), - "completed_checks_this_run": int(completed_this_run), - "pct_complete": float(completed / max(total, 1)), - "elapsed_sec": float(elapsed), - "eta_sec": eta, - } - if extra: - payload.update(extra) - write_runtime_status( - STAGE_NAME, - phase=phase, - state=state, - run_tag=run_tag, - status_path=str(status_path), - extra=payload, - ) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument("--run-tag", default="pool93_ijds_local_refine_stage1") - parser.add_argument( - "--profile", - choices=[ - "stage1", - "expanded", - "claim_expanded", - "claim_micro", - "claim_micro_ext", - "claim_bound_closure", - "claim_bound_floor_closure", - "claim_bound_terminal", - ], - default="stage1", - ) - parser.add_argument("--source-bound-eval", default=str(DEFAULT_SOURCE_BOUND_EVAL)) - parser.add_argument("--source-selection", default=str(DEFAULT_SOURCE_SELECTION)) - parser.add_argument("--conformal-intervals-path", default="") - parser.add_argument("--output-dir", default="") - parser.add_argument("--model-dir", default="") - parser.add_argument("--anchor-ranks", default="96,219,223") - parser.add_argument("--alpha-grid", default="") - parser.add_argument("--budget", type=float, default=1_000_000.0) - parser.add_argument("--t-eval", type=float, default=DEFAULT_T_EVAL) - parser.add_argument("--exact-threads", type=int, default=8) - parser.add_argument("--solver-backend", default="highspy") - parser.add_argument("--max-candidates", type=int, default=0) - parser.add_argument("--random-state", type=int, default=42) - parser.add_argument("--checkpoint-every", type=int, default=25) - parser.add_argument( - "--parallel-workers", - type=int, - default=1, - help=( - "Number of independent candidate-alpha solves to run in parallel. " - "Use --exact-threads 1 with multiple workers to avoid solver oversubscription." - ), - ) - parser.add_argument( - "--candidate-limit", - type=int, - default=0, - help="Debug/smoke option: keep only the first N generated policies when positive.", - ) - return parser - - -def _resolve_paths(args: argparse.Namespace, *, run_tag: str) -> Pool93Paths: - output_dir = ( - Path(args.output_dir) - if str(args.output_dir).strip() - else ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" - ) - model_dir = ( - Path(args.model_dir) - if str(args.model_dir).strip() - else ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" - ) - return Pool93Paths( - output_dir=output_dir, - model_dir=model_dir, - checkpoint_dir=model_dir / "runtime_checkpoints", - status_path=model_dir / "runtime_status.json", - candidates_path=output_dir / "pool93_ijds_local_refinement_candidates.parquet", - bound_eval_path=output_dir / "pool93_ijds_local_refinement_bound_eval.parquet", - leaderboard_path=output_dir / "pool93_ijds_local_refinement_leaderboard.parquet", - claim_summary_path=model_dir / "pool93_ijds_local_refinement_claim_summary.json", - manifest_path=model_dir / "pool93_ijds_local_refinement_manifest.json", - ) - - -def _ensure_pool93_dirs(paths: Pool93Paths) -> None: - paths.output_dir.mkdir(parents=True, exist_ok=True) - paths.model_dir.mkdir(parents=True, exist_ok=True) - paths.checkpoint_dir.mkdir(parents=True, exist_ok=True) - - -def _conformal_intervals_from_selection( - *, - explicit_path: str, - source_selection: Path, -) -> str: - if str(explicit_path).strip(): - return str(explicit_path).strip() - source_selection_payload = json.loads(source_selection.read_text(encoding="utf-8")) - return str(ROOT / source_selection_payload["conformal_intervals_path"]) - - -def _load_or_generate_candidates( - *, - args: argparse.Namespace, - paths: Pool93Paths, - source_bound_eval: Path, - anchor_ranks: list[int], -) -> pd.DataFrame: - if paths.candidates_path.exists(): - candidates = pd.read_parquet(paths.candidates_path) - logger.info( - "Reusing candidate manifest: {} rows from {}", - len(candidates), - paths.candidates_path, - ) - return candidates - anchors = _source_anchor_rows(source_bound_eval, anchor_ranks) - candidates = _generate_candidate_grid( - anchors, - profile=str(args.profile), - solver_backend=str(args.solver_backend), - ) - if int(args.candidate_limit) > 0: - candidates = candidates.head(int(args.candidate_limit)).copy().reset_index(drop=True) - candidates["local_candidate_id"] = np.arange(1, len(candidates) + 1, dtype=int) - atomic_write_parquet(candidates, paths.candidates_path, index=False) - logger.info( - "Wrote candidate manifest: {} policies to {}", - len(candidates), - paths.candidates_path, - ) - return candidates - - -def _manifest_payload( - *, - args: argparse.Namespace, - paths: Pool93Paths, - run_tag: str, - source_bound_eval: Path, - source_selection: Path, - conformal_intervals_path: str, - anchor_ranks: list[int], - alpha_grid: list[float], -) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": run_tag, - "profile": str(args.profile), - "source_bound_eval": str(source_bound_eval), - "source_selection": str(source_selection), - "conformal_intervals_path": conformal_intervals_path, - "anchor_ranks": anchor_ranks, - "alpha_grid": alpha_grid, - "budget": float(args.budget), - "t_eval": float(args.t_eval), - "exact_threads": int(args.exact_threads), - "solver_backend": str(args.solver_backend), - "max_candidates": int(args.max_candidates), - "random_state": int(args.random_state), - "checkpoint_every": int(args.checkpoint_every), - "parallel_workers": int(args.parallel_workers), - "candidates_path": str(paths.candidates_path), - "bound_eval_path": str(paths.bound_eval_path), - "leaderboard_path": str(paths.leaderboard_path), - "claim_summary_path": str(paths.claim_summary_path), - } - - -def _load_partial_bound_eval(path: Path) -> tuple[pd.DataFrame, set[tuple[int, float]]]: - partial = pd.DataFrame() - if path.exists(): - partial = pd.read_parquet(path) - if not partial.empty: - partial = partial.drop_duplicates( - ["local_candidate_id", "alpha"], - keep="last", - ).reset_index(drop=True) - logger.info("Resuming local refinement from {} rows", len(partial)) - completed_keys = { - (int(row["local_candidate_id"]), float(row["alpha"])) - for row in partial.to_dict(orient="records") - } - return partial, completed_keys - - -def _pending_refinement_tasks( - *, - candidates: pd.DataFrame, - alpha_grid: list[float], - completed_keys: set[tuple[int, float]], -) -> list[tuple[dict[str, Any], float]]: - return [ - (candidate, float(alpha)) - for candidate in candidates.to_dict(orient="records") - for alpha in alpha_grid - if (int(candidate["local_candidate_id"]), float(alpha)) not in completed_keys - ] - - -def _persist_refinement_progress( - *, - paths: Pool93Paths, - candidates: pd.DataFrame, - rows: list[dict[str, Any]], - alpha_grid: list[float], -) -> None: - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) - atomic_write_json( - paths.claim_summary_path, - _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid), - ) - - -def _run_serial_refinement( - *, - pending_tasks: list[tuple[dict[str, Any], float]], - aligned: pd.DataFrame, - budget: float, - t_eval: float, - exact_threads: int, - record_result: Any, -) -> None: - for candidate, alpha in pending_tasks: - policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} - result = _exact_policy_alpha( - aligned, - policy=policy, - alpha=float(alpha), - budget=float(budget), - t_eval=float(t_eval), - threads=int(exact_threads), - ) - record_result(candidate, alpha, result) - - -def _run_parallel_refinement( - *, - pending_tasks: list[tuple[dict[str, Any], float]], - aligned: pd.DataFrame, - parallel_workers: int, - budget: float, - t_eval: float, - exact_threads: int, - persist_progress: Any, - record_result: Any, -) -> None: - logger.info( - "Running exact refinement with {} parallel workers and {} solver thread(s) per worker", - parallel_workers, - int(exact_threads), - ) - mp_context = mp.get_context("fork") if sys.platform != "win32" else None - max_in_flight = max(parallel_workers, parallel_workers * 2) - next_task_idx = 0 - futures: dict[Any, tuple[dict[str, Any], float]] = {} - with ProcessPoolExecutor( - max_workers=parallel_workers, - mp_context=mp_context, - initializer=_init_exact_worker, - initargs=(aligned,), - ) as executor: - while next_task_idx < len(pending_tasks) or futures: - while next_task_idx < len(pending_tasks) and len(futures) < max_in_flight: - candidate, alpha = pending_tasks[next_task_idx] - future = executor.submit( - _exact_policy_alpha_task, - candidate, - alpha, - float(budget), - float(t_eval), - int(exact_threads), - ) - futures[future] = (candidate, alpha) - next_task_idx += 1 - done, _ = wait(futures, return_when=FIRST_COMPLETED) - for future in done: - candidate, alpha = futures.pop(future) - try: - result = future.result() - except Exception: - persist_progress() - raise - result_only = { - key: value - for key, value in result.items() - if key not in candidate or key in {"alpha", "confidence"} - } - record_result(candidate, alpha, result_only) - - -def _write_final_outputs( - *, - paths: Pool93Paths, - candidates: pd.DataFrame, - rows: list[dict[str, Any]], - alpha_grid: list[float], -) -> tuple[pd.DataFrame, dict[str, Any]]: - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) - claim_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) - atomic_write_json(paths.claim_summary_path, claim_summary) - return leaderboard, claim_summary - - -def main(argv: list[str] | None = None) -> int: - parser = _build_parser() - args = parser.parse_args(argv) - - run_tag = str(args.run_tag).strip().replace("/", "_") - paths = _resolve_paths(args, run_tag=run_tag) - _ensure_pool93_dirs(paths) - - source_bound_eval = Path(args.source_bound_eval) - source_selection = Path(args.source_selection) - conformal_intervals_path = _conformal_intervals_from_selection( - explicit_path=args.conformal_intervals_path, - source_selection=source_selection, - ) - alpha_grid = _coerce_float_grid(args.alpha_grid, DEFAULT_ALPHA_GRID) - anchor_ranks = _coerce_int_grid(args.anchor_ranks, [96, 219, 223]) - - candidates = _load_or_generate_candidates( - args=args, - paths=paths, - source_bound_eval=source_bound_eval, - anchor_ranks=anchor_ranks, - ) - atomic_write_json( - paths.manifest_path, - _manifest_payload( - args=args, - paths=paths, - run_tag=run_tag, - source_bound_eval=source_bound_eval, - source_selection=source_selection, - conformal_intervals_path=conformal_intervals_path, - anchor_ranks=anchor_ranks, - alpha_grid=alpha_grid, - ), - ) - - partial, completed_keys = _load_partial_bound_eval(paths.bound_eval_path) - rows: list[dict[str, Any]] = partial.to_dict(orient="records") if not partial.empty else [] - - total_checks = int(len(candidates) * len(alpha_grid)) - start = time.monotonic() - initial_completed = int(len(completed_keys)) - _write_status( - run_tag=run_tag, - status_path=paths.status_path, - start_monotonic=start, - completed=len(completed_keys), - total=total_checks, - phase="exact_refinement_running", - state="running", - initial_completed=initial_completed, - extra={"n_policies": int(len(candidates)), "profile": str(args.profile)}, - ) - - aligned = _load_aligned_dataset( - conformal_intervals_path=conformal_intervals_path, - max_candidates=int(args.max_candidates), - random_state=int(args.random_state), - ) - logger.info("Loaded aligned full universe: {} rows", len(aligned)) - completed = len(completed_keys) - - def persist_progress() -> None: - _persist_refinement_progress( - paths=paths, - candidates=candidates, - rows=rows, - alpha_grid=alpha_grid, - ) - - def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any]) -> None: - nonlocal completed - row = { - **candidate, - **result, - } - rows.append(row) - completed += 1 - completed_keys.add((int(candidate["local_candidate_id"]), float(alpha))) - _write_status( - run_tag=run_tag, - status_path=paths.status_path, - start_monotonic=start, - completed=completed, - total=total_checks, - phase="exact_refinement_running", - state="running", - initial_completed=initial_completed, - extra={ - "n_policies": int(len(candidates)), - "profile": str(args.profile), - "parallel_workers": int(args.parallel_workers), - "local_candidate_id": int(candidate["local_candidate_id"]), - "current_alpha": float(alpha), - "local_family": str(candidate["local_family"]), - "anchor_rank": int(candidate["anchor_rank"]), - }, - ) - if completed % max(1, int(args.checkpoint_every)) == 0: - persist_progress() - - pending_tasks = _pending_refinement_tasks( - candidates=candidates, - alpha_grid=alpha_grid, - completed_keys=completed_keys, - ) - - parallel_workers = max(1, int(args.parallel_workers)) - if parallel_workers <= 1: - _run_serial_refinement( - pending_tasks=pending_tasks, - aligned=aligned, - budget=float(args.budget), - t_eval=float(args.t_eval), - exact_threads=int(args.exact_threads), - record_result=record_result, - ) - else: - _run_parallel_refinement( - pending_tasks=pending_tasks, - aligned=aligned, - parallel_workers=parallel_workers, - budget=float(args.budget), - t_eval=float(args.t_eval), - exact_threads=int(args.exact_threads), - persist_progress=persist_progress, - record_result=record_result, - ) - - leaderboard, claim_summary = _write_final_outputs( - paths=paths, - candidates=candidates, - rows=rows, - alpha_grid=alpha_grid, - ) - - _write_status( - run_tag=run_tag, - status_path=paths.status_path, - start_monotonic=start, - completed=total_checks, - total=total_checks, - phase="selection_complete", - state="completed", - initial_completed=initial_completed, - extra={ - "n_policies": int(len(candidates)), - "n_all_alpha_passers": int(claim_summary["n_all_alpha_passers"]), - "n_all_alpha_passers_above_return_floor": int( - claim_summary["n_all_alpha_passers_above_return_floor"] - ), - "claim_summary_path": str(paths.claim_summary_path), - "leaderboard_path": str(paths.leaderboard_path), - }, - ) - write_runtime_checkpoint( - STAGE_NAME, - "selection_complete", - { - "run_tag": run_tag, - "completed_at_utc": datetime.now(tz=UTC).isoformat(), - "claim_summary_path": str(paths.claim_summary_path), - "leaderboard_path": str(paths.leaderboard_path), - }, - checkpoint_dir=paths.checkpoint_dir, - ) - logger.info("Local IJDS refinement complete: {}", paths.claim_summary_path) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/run_portfolio_bound_aware_search.py b/scripts/search/run_portfolio_bound_aware_search.py deleted file mode 100644 index cb4be29..0000000 --- a/scripts/search/run_portfolio_bound_aware_search.py +++ /dev/null @@ -1,2098 +0,0 @@ -"""Focused bound-aware portfolio search with exact alpha checks and runtime monitoring.""" - -from __future__ import annotations - -import argparse -import importlib -import json -import os -import subprocess -import sys -import time -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import yaml -from loguru import logger - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import ( # noqa: E402 - _align_loans_and_intervals, - _load_candidates, - _load_intervals, - _parse_float_grid, - _parse_percent_series, - _solve_single, -) -from scripts.run_gpu_replay import _GpuSampler # noqa: E402 -from scripts.validate_alpha_gamma_bound import ( # noqa: E402 - DEFAULT_THREADS, - _load_aligned_dataset, - _validate_single_alpha, -) -from src.optimization.certificate_semantics import ( # noqa: E402 - IJDS_DECLARED_ALPHA_GRID_CSV, -) -from src.utils.pipeline_runtime import ( # noqa: E402 - atomic_write_json, - atomic_write_parquet, - write_runtime_checkpoint, - write_runtime_status, -) - -SCHEMA_VERSION = "2026-04-05.2" -STAGE_NAME = "portfolio_bound_aware" -DEFAULT_INCUMBENT_POLICY_PATH = ROOT / "models" / "champion_portfolio_policy.json" -DEFAULT_EXACT_HELPER_SCRIPT = ROOT / "scripts" / "search" / "run_portfolio_bound_exact_eval.py" -SEMANTIC_POLICY_FIELDS = [ - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - "min_budget_utilization", - "pd_cap_slack_penalty", - "solver_backend", -] - - -@dataclass(frozen=True) -class BoundAwarePaths: - output_dir: Path - model_dir: Path - status_path: Path - checkpoint_dir: Path - resource_path: Path - gpu_csv_path: Path - frontier_raw_path: Path - frontier_path: Path - shortlist_path: Path - shortlist_exact_path: Path - bound_eval_path: Path - region_summary_path: Path - selection_path: Path - exact_context_path: Path - - -@dataclass(frozen=True) -class BoundAwareGridSpec: - risk_values: list[float] - aversion_values: list[float] - gamma_values: list[float] - delta_cap_quantiles: list[float] - tail_focus_quantiles: list[float] - alpha_grid: list[float] - random_states: list[int] - exact_random_states: list[int] - exact_max_candidates: int - budget_profiles: list[dict[str, Any]] - policy_modes: list[str] - incumbent_risk_neighbors: list[float] - incumbent_gamma_neighbors: list[float] - incumbent_policy_modes: list[str] - - @property - def policy_grid_count(self) -> int: - return _policy_grid_size( - gamma_values=self.gamma_values, - delta_cap_quantiles=self.delta_cap_quantiles, - tail_focus_quantiles=self.tail_focus_quantiles, - aversion_values=self.aversion_values, - budget_profiles=self.budget_profiles, - policy_modes=self.policy_modes, - ) - - @property - def frontier_total_units(self) -> int: - return int(len(self.random_states) * len(self.risk_values) * (1 + self.policy_grid_count)) - - def bound_total_checks(self, shortlist_size: int) -> int: - return int(shortlist_size * len(self.alpha_grid) * len(self.exact_random_states)) - - -@dataclass(frozen=True) -class BoundAwareExecutionSpec: - run_label: str - paths: BoundAwarePaths - grid: BoundAwareGridSpec - incumbent_policy: dict[str, Any] - exact_python_executable: str - exact_helper_script: Path - backend_validation: dict[str, Any] | None - cuopt_parameters: dict[str, Any] - - -@dataclass -class BoundAwareRunContext: - args: argparse.Namespace - spec: BoundAwareExecutionSpec - tracker: _ProgressTracker - search_space: dict[str, Any] - resource_payload: dict[str, Any] - - -@dataclass(frozen=True) -class BoundAwareFrontierState: - frontier_raw: pd.DataFrame - frontier: pd.DataFrame - shortlist: pd.DataFrame - bound_total_checks: int - shortlist_extra: dict[str, Any] - selection_context: dict[str, Any] - - -def _env_int(name: str, fallback: int) -> int: - raw = os.environ.get(name) - if raw is None: - return int(fallback) - try: - return int(raw) - except ValueError: - return int(fallback) - - -def _coerce_csv(raw: str | None) -> list[float]: - if not raw: - return [] - return _parse_float_grid(raw) - - -def _coerce_int_csv(raw: str | None, *, fallback: int) -> list[int]: - if not raw: - return [int(fallback)] - values = [int(part.strip()) for part in str(raw).split(",") if part.strip()] - return values or [int(fallback)] - - -def _coerce_str_csv(raw: object) -> list[str]: - return [part.strip() for part in str(raw).split(",") if part.strip()] - - -def _float_token(value: Any) -> float: - return round(float(value), 10) - - -def _policy_semantic_key(policy_like: Mapping[str, Any]) -> str: - payload: dict[str, Any] = {} - for field in SEMANTIC_POLICY_FIELDS: - value = policy_like[field] - payload[field] = ( - str(value) if field in {"policy_mode", "solver_backend"} else _float_token(value) - ) - return json.dumps(payload, sort_keys=True, separators=(",", ":")) - - -def _policy_from_row( - row: pd.Series, - *, - solver_backend_override: str | None = None, -) -> dict[str, Any]: - return { - "risk_tolerance": float(row["risk_tolerance"]), - "policy_mode": str(row["policy_mode"]), - "gamma": float(row["gamma"]), - "delta_cap_quantile": float(row["delta_cap_quantile"]), - "tail_focus_quantile": float(row["tail_focus_quantile"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - "min_budget_utilization": float(row["min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["pd_cap_slack_penalty"]), - "solver_backend": str( - solver_backend_override - if solver_backend_override is not None - else row["solver_backend"] - ), - } - - -def _normalized_policy_mode_filter(policy_modes: list[str] | None) -> set[str]: - return {str(mode).strip() for mode in (policy_modes or []) if str(mode).strip()} - - -def _mode_quantile_pairs( - *, - mode: str, - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], -) -> list[tuple[float, float]]: - if mode == "blended_uncertainty": - return [(1.0, 1.0)] - if mode == "capped_blended_uncertainty": - return [(float(delta_cap_quantile), 1.0) for delta_cap_quantile in delta_cap_quantiles] - return [(1.0, float(tail_focus_quantile)) for tail_focus_quantile in tail_focus_quantiles] - - -def _targeted_policy_grid( - *, - gamma_values: list[float], - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], - policy_modes: list[str] | None = None, -) -> list[tuple[str, float, float, float]]: - allowed = _normalized_policy_mode_filter(policy_modes) - mode_order = [ - "blended_uncertainty", - "capped_blended_uncertainty", - "tail_blended_uncertainty", - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - ] - active_modes = [mode for mode in mode_order if not allowed or mode in allowed] - grid = [ - (mode, float(gamma), delta_cap_quantile, tail_focus_quantile) - for gamma in gamma_values - for mode in active_modes - for delta_cap_quantile, tail_focus_quantile in _mode_quantile_pairs( - mode=mode, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - ) - ] - return list(dict.fromkeys(grid)) - - -def _eta_seconds(elapsed_sec: float, completed: int, total: int) -> float | None: - if completed <= 0 or total <= 0 or completed >= total: - return 0.0 if total > 0 and completed >= total else None - return (elapsed_sec / max(completed, 1)) * max(total - completed, 0) - - -def _resource_snapshot() -> dict[str, Any]: - payload: dict[str, Any] = { - "captured_at_utc": datetime.now(tz=UTC).isoformat(), - "cpu_count": int(os.cpu_count() or 0), - } - try: - meminfo = Path("/proc/meminfo").read_text(encoding="utf-8") - parsed: dict[str, float] = {} - for line in meminfo.splitlines(): - if ":" not in line: - continue - key, rest = line.split(":", 1) - token = rest.strip().split()[0] - parsed[key] = float(token) - payload["memory_total_kib"] = int(parsed.get("MemTotal", 0)) - payload["memory_available_kib"] = int(parsed.get("MemAvailable", 0)) - payload["swap_free_kib"] = int(parsed.get("SwapFree", 0)) - except Exception as exc: # pragma: no cover - best effort only - payload["memory_probe_error"] = str(exc) - try: - proc = subprocess.run( - [ - "nvidia-smi", - "--query-gpu=name,driver_version,memory.total,memory.used,memory.free,utilization.gpu,utilization.memory", - "--format=csv,noheader,nounits", - ], - capture_output=True, - text=True, - check=False, - ) - if proc.returncode == 0 and proc.stdout.strip(): - first = [part.strip() for part in proc.stdout.splitlines()[0].split(",")] - if len(first) >= 7: - payload["gpu"] = { - "name": first[0], - "driver_version": first[1], - "memory_total_mb": float(first[2]), - "memory_used_mb": float(first[3]), - "memory_free_mb": float(first[4]), - "gpu_util_pct": float(first[5]), - "memory_util_pct": float(first[6]), - } - except Exception as exc: # pragma: no cover - best effort only - payload["gpu_probe_error"] = str(exc) - return payload - - -def _validate_cuopt_runtime() -> dict[str, Any]: - modules = ["cuopt", "cudf", "cupy", "pyomo", "loguru"] - payload: dict[str, Any] = { - "validated_at_utc": datetime.now(tz=UTC).isoformat(), - "python": sys.executable, - "expected_release": "26.02_or_newer_26.x", - } - for module_name in modules: - module = importlib.import_module(module_name) - payload[module_name] = getattr(module, "__version__", "ok") - cuopt_version = str(payload.get("cuopt", "")) - parts = cuopt_version.replace("-", ".").split(".") - try: - major = int(parts[0]) - minor = int(parts[1]) - except (IndexError, ValueError) as exc: - raise RuntimeError(f"Unable to parse cuOpt version {cuopt_version!r}.") from exc - if not (major == 26 and minor >= 2): - raise RuntimeError( - f"cuOpt 26.02+ within release 26.x is required for GPU bound-aware search; " - f"found {cuopt_version!r}." - ) - return payload - - -def _blank(raw: Any) -> bool: - return str(raw).strip() == "" - - -def _optional_int(raw: Any, *, name: str) -> int | None: - if _blank(raw): - return None - try: - return int(str(raw).strip()) - except ValueError as exc: - raise ValueError(f"{name} must be an integer, got {raw!r}.") from exc - - -def _optional_bool(raw: Any, *, name: str) -> bool | None: - if _blank(raw): - return None - token = str(raw).strip().lower() - if token in {"1", "true", "yes", "y", "on"}: - return True - if token in {"0", "false", "no", "n", "off"}: - return False - raise ValueError(f"{name} must be true/false or 1/0, got {raw!r}.") - - -def _cuopt_method(raw: Any) -> str | None: - if _blank(raw): - return None - token = str(raw).strip().lower().replace("-", "_").replace(" ", "_") - aliases = { - "concurrent": "Concurrent", - "pdlp": "PDLP", - "dual_simplex": "Dual Simplex", - "dualsimplex": "Dual Simplex", - "barrier": "Barrier", - } - if token not in aliases: - raise ValueError("--cuopt-method must be one of concurrent, pdlp, dual_simplex, barrier.") - return aliases[token] - - -def _cuopt_pdlp_mode(raw: Any) -> str | None: - if _blank(raw): - return None - token = str(raw).strip().lower().replace("-", "_").replace(" ", "_") - aliases = { - "stable3": "Stable3", - "methodical1": "Methodical1", - "fast1": "Fast1", - } - if token not in aliases: - raise ValueError("--cuopt-pdlp-solver-mode must be stable3, methodical1, or fast1.") - return aliases[token] - - -def _cuopt_pdlp_precision(raw: Any) -> int | None: - if _blank(raw): - return None - token = str(raw).strip().lower() - aliases = {"default": -1, "single": 0, "double": 1, "mixed": 2} - if token in aliases: - return aliases[token] - try: - value = int(token) - except ValueError as exc: - raise ValueError("--cuopt-pdlp-precision must be default/single/double/mixed.") from exc - if value not in {-1, 0, 1, 2}: - raise ValueError("--cuopt-pdlp-precision integer value must be one of -1,0,1,2.") - return value - - -def _extra_cuopt_parameters(raw_items: list[str] | None) -> dict[str, Any]: - params: dict[str, Any] = {} - for raw in raw_items or []: - if "=" not in str(raw): - raise ValueError(f"--cuopt-extra-parameter expects name=value, got {raw!r}.") - name, value = str(raw).split("=", 1) - key = name.strip().replace("-", "_").lower() - key = key.removeprefix("cuopt_") - params[key] = value.strip() - return params - - -def _cuopt_parameter_overrides(args: argparse.Namespace, *, model_dir: Path) -> dict[str, Any]: - params = _extra_cuopt_parameters(getattr(args, "cuopt_extra_parameter", None)) - simple_values: dict[str, Any] = { - "method": _cuopt_method(args.cuopt_method), - "pdlp_solver_mode": _cuopt_pdlp_mode(args.cuopt_pdlp_solver_mode), - "pdlp_precision": _cuopt_pdlp_precision(args.cuopt_pdlp_precision), - "crossover": _optional_bool(args.cuopt_crossover, name="--cuopt-crossover"), - "first_primal_feasible": _optional_bool( - args.cuopt_first_primal_feasible, - name="--cuopt-first-primal-feasible", - ), - "save_best_primal_solution": _optional_bool( - args.cuopt_save_best_primal_solution, - name="--cuopt-save-best-primal-solution", - ), - "infeasibility_detection": _optional_bool( - args.cuopt_infeasibility_detection, - name="--cuopt-infeasibility-detection", - ), - "strict_infeasibility": _optional_bool( - args.cuopt_strict_infeasibility, - name="--cuopt-strict-infeasibility", - ), - "per_constraint_residual": _optional_bool( - args.cuopt_per_constraint_residual, - name="--cuopt-per-constraint-residual", - ), - "dual_postsolve": _optional_bool( - args.cuopt_dual_postsolve, - name="--cuopt-dual-postsolve", - ), - "cudss_deterministic": _optional_bool( - args.cuopt_cudss_deterministic, - name="--cuopt-cudss-deterministic", - ), - "eliminate_dense_columns": _optional_bool( - args.cuopt_eliminate_dense_columns, - name="--cuopt-eliminate-dense-columns", - ), - "log_to_console": _optional_bool(args.cuopt_log_to_console, name="--cuopt-log-to-console"), - "iteration_limit": _optional_int( - args.cuopt_iteration_limit, - name="--cuopt-iteration-limit", - ), - "num_cpu_threads": _optional_int( - args.cuopt_num_cpu_threads, - name="--cuopt-num-cpu-threads", - ), - "dualize": _optional_int(args.cuopt_dualize, name="--cuopt-dualize"), - "folding": _optional_int(args.cuopt_folding, name="--cuopt-folding"), - "augmented": _optional_int(args.cuopt_augmented, name="--cuopt-augmented"), - "ordering": _optional_int(args.cuopt_ordering, name="--cuopt-ordering"), - "num_gpus": _optional_int(args.cuopt_num_gpus, name="--cuopt-num-gpus"), - } - for key, value in simple_values.items(): - if value is not None: - params[key] = value - log_dir = str(args.cuopt_log_dir).strip() - if log_dir: - params["log_dir"] = str(model_dir / "cuopt_logs") if log_dir == "auto" else log_dir - return params - - -class _ProgressTracker: - def __init__( - self, - *, - status_path: Path, - checkpoint_dir: Path, - run_tag: str, - frontier_total_units: int, - ) -> None: - self.status_path = status_path - self.checkpoint_dir = checkpoint_dir - self.run_tag = run_tag - self.frontier_total_units = int(frontier_total_units) - self.bound_total_checks = 0 - self.frontier_completed_units = 0 - self.bound_completed_checks = 0 - self.started_at = time.monotonic() - self._checkpoint_seq = 0 - - def _elapsed(self) -> float: - return float(time.monotonic() - self.started_at) - - def _payload( - self, *, phase: str, state: str, extra: dict[str, Any] | None = None - ) -> dict[str, Any]: - elapsed_sec = self._elapsed() - global_total = int(self.frontier_total_units + self.bound_total_checks) - global_completed = int(self.frontier_completed_units + self.bound_completed_checks) - payload: dict[str, Any] = { - "frontier_total_units": int(self.frontier_total_units), - "frontier_completed_units": int(self.frontier_completed_units), - "frontier_pct_complete": ( - float(self.frontier_completed_units / max(self.frontier_total_units, 1)) - if self.frontier_total_units > 0 - else 1.0 - ), - "bound_total_checks": int(self.bound_total_checks), - "bound_completed_checks": int(self.bound_completed_checks), - "bound_pct_complete": ( - float(self.bound_completed_checks / max(self.bound_total_checks, 1)) - if self.bound_total_checks > 0 - else 0.0 - ), - "global_total_units": int(global_total), - "global_completed_units": int(global_completed), - "global_pct_complete": ( - float(global_completed / max(global_total, 1)) if global_total > 0 else 1.0 - ), - "elapsed_sec": float(elapsed_sec), - "eta_sec": _eta_seconds(elapsed_sec, global_completed, global_total), - } - if extra: - payload.update(extra) - write_runtime_status( - STAGE_NAME, - phase=phase, - state=state, - run_tag=self.run_tag, - status_path=self.status_path, - extra=payload, - ) - return payload - - def checkpoint(self, name: str, payload: dict[str, Any]) -> None: - self._checkpoint_seq += 1 - write_runtime_checkpoint( - STAGE_NAME, - f"{self._checkpoint_seq:03d}_{name}", - payload, - checkpoint_dir=self.checkpoint_dir, - ) - - def start(self, *, extra: dict[str, Any] | None = None) -> None: - self._payload(phase="loading_inputs", state="running", extra=extra) - - def frontier_progress( - self, *, completed_units: int, extra: dict[str, Any] | None = None - ) -> None: - self.frontier_completed_units = int(completed_units) - self._payload(phase="frontier_running", state="running", extra=extra) - - def frontier_complete(self, *, extra: dict[str, Any] | None = None) -> None: - self.frontier_completed_units = int(self.frontier_total_units) - payload = self._payload(phase="frontier_complete", state="running", extra=extra) - self.checkpoint("frontier_complete", payload) - - def set_bound_total( - self, bound_total_checks: int, *, extra: dict[str, Any] | None = None - ) -> None: - self.bound_total_checks = int(bound_total_checks) - payload = self._payload(phase="shortlist_building", state="running", extra=extra) - self.checkpoint("shortlist_built", payload) - - def bound_progress(self, *, completed_checks: int, extra: dict[str, Any] | None = None) -> None: - self.bound_completed_checks = int(completed_checks) - self._payload(phase="exact_bound_running", state="running", extra=extra) - - def complete(self, *, extra: dict[str, Any] | None = None) -> None: - self.bound_completed_checks = int(self.bound_total_checks) - payload = self._payload(phase="selection_complete", state="completed", extra=extra) - self.checkpoint("selection_complete", payload) - - def frontier_only_complete(self, *, extra: dict[str, Any] | None = None) -> None: - payload = self._payload(phase="frontier_only_complete", state="completed", extra=extra) - self.checkpoint("frontier_only_complete", payload) - - def fail(self, *, phase: str, extra: dict[str, Any] | None = None) -> None: - payload = self._payload(phase=phase, state="failed", extra=extra) - self.checkpoint("failed", payload) - - -def _load_incumbent_policy(path: str | Path | None) -> dict[str, Any]: - target = Path(path) if path is not None else DEFAULT_INCUMBENT_POLICY_PATH - payload = json.loads(target.read_text(encoding="utf-8")) - selected = payload.get("selected_policy", payload) - return { - "risk_tolerance": float(selected.get("risk_tolerance", 0.16)), - "policy_mode": str(selected.get("policy_mode", "blended_uncertainty")), - "gamma": float(selected.get("gamma", 0.5)), - "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), - "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), - "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), - "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), - "solver_backend": str(selected.get("solver_backend", "highs")), - "source_path": str(target), - } - - -def _policy_grid_size( - *, - gamma_values: list[float], - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], - aversion_values: list[float], - budget_profiles: list[dict[str, Any]], - policy_modes: list[str] | None = None, -) -> int: - policy_grid = _targeted_policy_grid( - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - policy_modes=policy_modes, - ) - return int(len(policy_grid) * len(aversion_values) * len(budget_profiles)) - - -def _build_frontier_for_seed( - *, - config_path: str, - conformal_intervals_path: str, - risk_values: list[float], - aversion_values: list[float], - gamma_values: list[float], - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], - budget_profiles: list[dict[str, float]], - max_candidates: int, - random_state: int, - solver_backend: str, - cuopt_presolve: int | None, - cuopt_parameters: dict[str, Any] | None, - policy_modes: list[str] | None, - progress_hook: Callable[[int, dict[str, Any]], None], -) -> pd.DataFrame: - with open(config_path, encoding="utf-8") as handle: - config = yaml.safe_load(handle) - - candidates = _load_candidates().reset_index(drop=True) - intervals = _load_intervals(conformal_intervals_path=conformal_intervals_path).reset_index( - drop=True - ) - loans, ints = _align_loans_and_intervals( - candidates=candidates, - intervals=intervals, - max_candidates=max_candidates, - random_state=random_state, - ) - n = len(loans) - col_point = "y_pred" if "y_pred" in ints.columns else "pd_point" - col_low = "pd_low_90" if "pd_low_90" in ints.columns else "pd_low" - col_high = "pd_high_90" if "pd_high_90" in ints.columns else "pd_high" - pd_point = ints[col_point].to_numpy(dtype=float) - pd_low = ints[col_low].to_numpy(dtype=float) - pd_high = ints[col_high].to_numpy(dtype=float) - lgd = pd.Series([0.45] * n, dtype=float).to_numpy(dtype=float) - int_rates = ( - _parse_percent_series(loans["int_rate"]) - if "int_rate" in loans.columns - else pd.Series([0.12] * n, dtype=float).to_numpy(dtype=float) - ) - default_flag = ( - pd.to_numeric(loans["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) - if "default_flag" in loans.columns - else pd.Series([0] * n, dtype=int).to_numpy(dtype=int) - ) - total_budget = float(config["portfolio"]["total_budget"]) - max_concentration = float(config["portfolio"]["max_concentration"]) - time_limit = int(config["optimization"]["time_limit"]) - threads = int(config["optimization"]["threads"]) - policy_grid = _targeted_policy_grid( - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - policy_modes=policy_modes, - ) - - rows: list[dict[str, Any]] = [] - logger.info( - "Running focused portfolio frontier on n={:,}, random_state={}, risk_values={}, aversion_values={}, policies={}", - n, - random_state, - risk_values, - aversion_values, - len(policy_grid), - ) - completed = 0 - for risk_tol in risk_values: - baseline, _ = _solve_single( - loans=loans, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - default_flag=default_flag, - total_budget=total_budget, - max_concentration=max_concentration, - risk_tolerance=float(risk_tol), - robust=False, - uncertainty_aversion=0.0, - min_budget_utilization=0.0, - pd_cap_slack_penalty=0.0, - time_limit=time_limit, - threads=threads, - solver_backend=solver_backend, - random_seed=int(random_state), - cuopt_presolve=cuopt_presolve, - cuopt_parameters=cuopt_parameters, - ) - completed += 1 - progress_hook( - completed, - { - "phase_random_state": int(random_state), - "latest_policy_mode": "point_estimate", - "latest_risk_tolerance": float(risk_tol), - "latest_gamma": 0.0, - }, - ) - baseline_ret = float(baseline["expected_return_net_point"]) - baseline_realized = float(baseline["realized_total_return"]) - for policy_mode, gamma, delta_cap_quantile, tail_focus_quantile in policy_grid: - for aversion in aversion_values: - for budget_profile in budget_profiles: - robust_run, _ = _solve_single( - loans=loans, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - default_flag=default_flag, - total_budget=total_budget, - max_concentration=max_concentration, - risk_tolerance=float(risk_tol), - robust=True, - uncertainty_aversion=float(aversion), - min_budget_utilization=float(budget_profile["min_budget_utilization"]), - pd_cap_slack_penalty=float(budget_profile["pd_cap_slack_penalty"]), - time_limit=time_limit, - threads=threads, - solver_backend=solver_backend, - policy_mode=policy_mode, - gamma=float(gamma), - delta_cap_quantile=float(delta_cap_quantile), - tail_focus_quantile=float(tail_focus_quantile), - random_seed=int(random_state), - cuopt_presolve=cuopt_presolve, - cuopt_parameters=cuopt_parameters, - ) - completed += 1 - progress_hook( - completed, - { - "phase_random_state": int(random_state), - "latest_policy_mode": str(policy_mode), - "latest_risk_tolerance": float(risk_tol), - "latest_gamma": float(gamma), - }, - ) - por = baseline_ret - float(robust_run["expected_return_net_point"]) - por_pct = por / (abs(baseline_ret) + 1e-6) * 100.0 - realized_total_return = float(robust_run["realized_total_return"]) - ab_diff_total_return = float(realized_total_return - baseline_realized) - ab_pass = bool(ab_diff_total_return >= -(abs(baseline_realized) * 0.05)) - rows.append( - { - "sample_random_state": int(random_state), - "risk_tolerance": float(risk_tol), - "policy_mode": str(policy_mode), - "gamma": float(gamma), - "delta_cap_quantile": float(delta_cap_quantile), - "tail_focus_quantile": float(tail_focus_quantile), - "uncertainty_aversion": float(aversion), - "min_budget_utilization": float( - budget_profile["min_budget_utilization"] - ), - "pd_cap_slack_penalty": float(budget_profile["pd_cap_slack_penalty"]), - "budget_profile_name": str(budget_profile["name"]), - "price_of_robustness": float(por), - "price_of_robustness_pct": float(por_pct), - "ab_diff_total_return": float(ab_diff_total_return), - "ab_pass": ab_pass, - **robust_run, - } - ) - return pd.DataFrame(rows).reset_index(drop=True) - - -def _aggregate_frontier(frontier_raw: pd.DataFrame) -> pd.DataFrame: - grouped = frontier_raw.groupby(SEMANTIC_POLICY_FIELDS, dropna=False) - frontier = grouped.agg( - seed_count=("sample_random_state", "nunique"), - sample_random_states=( - "sample_random_state", - lambda s: ",".join(map(str, sorted({int(v) for v in s}))), - ), - ab_pass_all=("ab_pass", "all"), - ab_pass_rate=("ab_pass", "mean"), - realized_total_return=("realized_total_return", "mean"), - realized_total_return_max=("realized_total_return", "max"), - price_of_robustness=("price_of_robustness", "mean"), - price_of_robustness_pct=("price_of_robustness_pct", "mean"), - ab_diff_total_return=("ab_diff_total_return", "mean"), - objective_value=("objective_value", "mean"), - n_funded=("n_funded", "mean"), - total_allocated=("total_allocated", "mean"), - expected_return_net_point=("expected_return_net_point", "mean"), - worst_case_pd=("worst_case_pd", "mean"), - point_pd=("point_pd", "mean"), - pd_cap_slack=("pd_cap_slack", "max"), - ).reset_index() - frontier["semantic_policy_key"] = frontier.apply(_policy_semantic_key, axis=1) - return frontier.reset_index(drop=True) - - -def _aggregate_alpha_grid_results(bound_eval: pd.DataFrame) -> pd.DataFrame: - """Summarize exact bound checks across every evaluated alpha level.""" - if bound_eval.empty: - return pd.DataFrame( - { - **{field: pd.Series(dtype="object") for field in SEMANTIC_POLICY_FIELDS}, - "alpha_exact_pass_count": pd.Series(dtype="int64"), - } - ) - grouped = bound_eval.groupby(SEMANTIC_POLICY_FIELDS, dropna=False) - out = grouped.agg( - alpha_exact_pass_count=("all_bounds_hold", "sum"), - alpha_exact_check_count=("all_bounds_hold", "size"), - alpha_exact_pass_rate=("all_bounds_hold", "mean"), - alpha_max_violation=("violation", "max"), - alpha_mean_gamma_cp=("gamma_cp", "mean"), - alpha_mean_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha_mean_weighted_pd_true=("weighted_pd_true", "mean"), - alpha_mean_empirical_coverage_funded=("empirical_coverage_funded", "mean"), - ).reset_index() - pass_min_alpha = ( - bound_eval.loc[bound_eval["all_bounds_hold"].fillna(False)] - .groupby(SEMANTIC_POLICY_FIELDS, dropna=False)["alpha"] - .min() - .rename("alpha_min_passed") - .reset_index() - ) - out = out.merge(pass_min_alpha, on=SEMANTIC_POLICY_FIELDS, how="left") - out["alpha_min_passed"] = out["alpha_min_passed"].where(out["alpha_min_passed"].notna(), np.nan) - return out - - -def _apply_rank( - df: pd.DataFrame, *, by: list[str], ascending: list[bool], rank_col: str -) -> pd.DataFrame: - ranked = df.sort_values(by=by, ascending=ascending, kind="mergesort").reset_index(drop=True) - ranked[rank_col] = range(1, len(ranked) + 1) - return ranked.loc[:, ["semantic_policy_key", rank_col]] - - -def _rank_frontier(frontier: pd.DataFrame) -> pd.DataFrame: - work = frontier.copy() - if "semantic_policy_key" not in work.columns: - work["semantic_policy_key"] = work.apply(_policy_semantic_key, axis=1) - return_rank = _apply_rank( - work, - by=["ab_pass_all", "realized_total_return", "price_of_robustness"], - ascending=[False, False, True], - rank_col="return_first_rank", - ) - proxy_rank = _apply_rank( - work, - by=[ - "ab_pass_all", - "risk_tolerance", - "worst_case_pd", - "point_pd", - "pd_cap_slack", - "realized_total_return", - "price_of_robustness", - ], - ascending=[False, True, True, True, True, False, True], - rank_col="bound_proxy_rank", - ) - work = work.merge(return_rank, on="semantic_policy_key", how="left") - work = work.merge(proxy_rank, on="semantic_policy_key", how="left") - return work - - -def _select_top_unique( - frame: pd.DataFrame, - *, - limit: int, - seen: set[str], - bucket_name: str, - selected_rows: list[dict[str, Any]], -) -> None: - if limit <= 0 or frame.empty: - return - added = 0 - for _, row in frame.iterrows(): - key = row["semantic_policy_key"] - if key in seen: - continue - payload = row.to_dict() - payload["shortlist_bucket"] = bucket_name - selected_rows.append(payload) - seen.add(key) - added += 1 - if added >= int(limit): - break - - -def _build_forced_policy_keys( - *, - incumbent_policy: dict[str, Any], - incumbent_risk_neighbors: list[float], - incumbent_gamma_neighbors: list[float], - incumbent_policy_modes: list[str], - budget_profiles: list[dict[str, float]], - solver_backend: str, -) -> set[str]: - keys: set[str] = set() - for risk in incumbent_risk_neighbors: - for gamma in incumbent_gamma_neighbors: - for policy_mode in incumbent_policy_modes: - delta_cap_quantile = ( - float(incumbent_policy["delta_cap_quantile"]) - if policy_mode == "capped_blended_uncertainty" - else 1.0 - ) - tail_focus_quantile = ( - float(incumbent_policy["tail_focus_quantile"]) - if policy_mode == "tail_blended_uncertainty" - else 1.0 - ) - for budget_profile in budget_profiles: - keys.add( - _policy_semantic_key( - { - "risk_tolerance": risk, - "policy_mode": policy_mode, - "gamma": gamma, - "delta_cap_quantile": delta_cap_quantile, - "tail_focus_quantile": tail_focus_quantile, - "uncertainty_aversion": float( - incumbent_policy["uncertainty_aversion"] - ), - "min_budget_utilization": float( - budget_profile["min_budget_utilization"] - ), - "pd_cap_slack_penalty": float( - budget_profile["pd_cap_slack_penalty"] - ), - "solver_backend": str(solver_backend), - } - ) - ) - return keys - - -def _build_stratified_shortlist( - *, - frontier: pd.DataFrame, - shortlist_top_k: int, - bucket_return_k: int, - bucket_proxy_k: int, - bucket_family_k: int, - bucket_region_k: int, - incumbent_policy: dict[str, Any], - incumbent_risk_neighbors: list[float], - incumbent_gamma_neighbors: list[float], - incumbent_policy_modes: list[str], - budget_profiles: list[dict[str, float]], - solver_backend: str, -) -> pd.DataFrame: - ranked = _rank_frontier(frontier) - forced_keys = _build_forced_policy_keys( - incumbent_policy=incumbent_policy, - incumbent_risk_neighbors=incumbent_risk_neighbors, - incumbent_gamma_neighbors=incumbent_gamma_neighbors, - incumbent_policy_modes=incumbent_policy_modes, - budget_profiles=budget_profiles, - solver_backend=solver_backend, - ) - region_mask = ( - ranked["risk_tolerance"].isin([float(v) for v in incumbent_risk_neighbors]) - & ranked["gamma"].isin([float(v) for v in incumbent_gamma_neighbors]) - & ranked["policy_mode"].isin([str(v) for v in incumbent_policy_modes]) - ) - - selected_rows: list[dict[str, Any]] = [] - seen: set[str] = set() - - forced = ranked[ranked["semantic_policy_key"].isin(forced_keys)].sort_values( - by=["bound_proxy_rank", "return_first_rank"], - ascending=[True, True], - kind="mergesort", - ) - _select_top_unique( - forced, - limit=len(forced), - seen=seen, - bucket_name="forced_incumbent_neighbors", - selected_rows=selected_rows, - ) - - region = ranked[region_mask].sort_values( - by=["bound_proxy_rank", "return_first_rank"], - ascending=[True, True], - kind="mergesort", - ) - _select_top_unique( - region, - limit=int(bucket_region_k), - seen=seen, - bucket_name="incumbent_region", - selected_rows=selected_rows, - ) - - conservative = ranked.sort_values( - by=["bound_proxy_rank", "return_first_rank"], - ascending=[True, True], - kind="mergesort", - ) - _select_top_unique( - conservative, - limit=int(bucket_proxy_k), - seen=seen, - bucket_name="conservative_proxy", - selected_rows=selected_rows, - ) - - for family in sorted(ranked["policy_mode"].astype(str).unique()): - family_frame = ranked[ranked["policy_mode"] == family].sort_values( - by=["bound_proxy_rank", "return_first_rank"], - ascending=[True, True], - kind="mergesort", - ) - _select_top_unique( - family_frame, - limit=int(bucket_family_k), - seen=seen, - bucket_name=f"family::{family}", - selected_rows=selected_rows, - ) - - return_global = ranked.sort_values( - by=["return_first_rank", "bound_proxy_rank"], - ascending=[True, True], - kind="mergesort", - ) - _select_top_unique( - return_global, - limit=int(bucket_return_k), - seen=seen, - bucket_name="return_global", - selected_rows=selected_rows, - ) - - shortlist = pd.DataFrame(selected_rows) - if shortlist.empty: - raise ValueError("Stratified shortlist is empty; cannot continue.") - shortlist = shortlist.iloc[: int(shortlist_top_k)].copy().reset_index(drop=True) - shortlist["candidate_rank"] = range(1, len(shortlist) + 1) - return shortlist - - -def _aggregate_exact_results( - *, - shortlist: pd.DataFrame, - bound_eval: pd.DataFrame, -) -> pd.DataFrame: - alpha01 = ( - bound_eval[bound_eval["alpha"] == 0.01] - .groupby(SEMANTIC_POLICY_FIELDS, dropna=False) - .agg( - alpha01_exact_pass=("all_bounds_hold", "all"), - alpha01_pass_rate=("all_bounds_hold", "mean"), - alpha01_gamma_cp=("gamma_cp", "mean"), - alpha01_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha01_violation=("violation", "max"), - alpha01_weighted_pd_true=("weighted_pd_true", "mean"), - alpha01_weighted_pd_constraint_used=("weighted_pd_constraint_used", "mean"), - alpha01_empirical_coverage_funded=("empirical_coverage_funded", "mean"), - ) - .reset_index() - ) - alpha03 = ( - bound_eval[bound_eval["alpha"] == 0.03] - .groupby(SEMANTIC_POLICY_FIELDS, dropna=False) - .agg( - alpha03_exact_pass=("all_bounds_hold", "all"), - alpha03_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - ) - .reset_index() - ) - alpha10 = ( - bound_eval[bound_eval["alpha"] == 0.10] - .groupby(SEMANTIC_POLICY_FIELDS, dropna=False) - .agg( - alpha10_exact_pass=("all_bounds_hold", "all"), - alpha10_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - ) - .reset_index() - ) - - work = shortlist.copy() - exact_metric_prefixes = ("alpha01_", "alpha03_", "alpha10_") - stale_exact_cols = [ - col - for col in work.columns - if any(col.startswith(prefix) for prefix in exact_metric_prefixes) - ] - if stale_exact_cols: - work = work.drop(columns=stale_exact_cols) - work = work.merge(alpha01, on=SEMANTIC_POLICY_FIELDS, how="left") - work = work.merge(alpha03, on=SEMANTIC_POLICY_FIELDS, how="left") - work = work.merge(alpha10, on=SEMANTIC_POLICY_FIELDS, how="left") - alpha_grid_summary = _aggregate_alpha_grid_results(bound_eval) - work = work.merge(alpha_grid_summary, on=SEMANTIC_POLICY_FIELDS, how="left") - bool_cols = ["alpha01_exact_pass", "alpha03_exact_pass", "alpha10_exact_pass"] - for col in bool_cols: - work[col] = work[col].where(work[col].notna(), False).astype(bool) - numeric_defaults = { - "alpha_exact_pass_count": 0, - "alpha_exact_check_count": 0, - "alpha_exact_pass_rate": 0.0, - "alpha_max_violation": float("inf"), - "alpha_mean_gamma_cp": float("inf"), - "alpha_mean_weighted_miscoverage_V": float("inf"), - "alpha_mean_weighted_pd_true": float("inf"), - "alpha_mean_empirical_coverage_funded": 0.0, - } - for col, default in numeric_defaults.items(): - if col not in work.columns: - work[col] = default - work[col] = work[col].fillna(default) - work = work.sort_values( - by=[ - "alpha01_exact_pass", - "alpha03_exact_pass", - "alpha_exact_pass_count", - "alpha_exact_pass_rate", - "ab_pass_all", - "realized_total_return", - "alpha01_weighted_miscoverage_V", - "alpha01_gamma_cp", - "price_of_robustness", - ], - ascending=[False, False, False, False, False, False, True, True, True], - kind="mergesort", - ) - return work.reset_index(drop=True) - - -def _region_summary(shortlist_eval: pd.DataFrame, bound_eval: pd.DataFrame) -> dict[str, Any]: - """Describe robust policy regions for paper-facing claim selection.""" - if shortlist_eval.empty: - return {} - - def _bucket_payload(frame: pd.DataFrame) -> dict[str, Any]: - if frame.empty: - return { - "n_policies": 0, - "n_alpha01_passers": 0, - "alpha01_pass_rate": 0.0, - "n_all_alpha_passers": 0, - "all_alpha_pass_rate": 0.0, - } - alpha01_pass = frame["alpha01_exact_pass"].fillna(False).astype(bool) - all_alpha_pass = frame["alpha_exact_pass_count"].fillna(0) >= frame[ - "alpha_exact_check_count" - ].fillna(1) - promotable = frame.loc[alpha01_pass].copy() - return { - "n_policies": int(len(frame)), - "n_alpha01_passers": int(alpha01_pass.sum()), - "alpha01_pass_rate": float(alpha01_pass.mean()), - "n_all_alpha_passers": int(all_alpha_pass.sum()), - "all_alpha_pass_rate": float(all_alpha_pass.mean()), - "best_return_alpha01": ( - float(promotable["realized_total_return"].max()) if not promotable.empty else None - ), - "min_gamma_cp_alpha01": ( - float(promotable["alpha01_gamma_cp"].min()) if not promotable.empty else None - ), - "min_weighted_miscoverage_V_alpha01": ( - float(promotable["alpha01_weighted_miscoverage_V"].min()) - if not promotable.empty - else None - ), - "min_price_of_robustness_alpha01": ( - float(promotable["price_of_robustness"].min()) if not promotable.empty else None - ), - } - - bucket_rows: dict[str, Any] = {} - if "shortlist_bucket" in shortlist_eval.columns: - for bucket, frame in shortlist_eval.groupby("shortlist_bucket", dropna=False): - bucket_rows[str(bucket)] = _bucket_payload(frame) - - exact_alpha_summary = {} - if not bound_eval.empty: - for alpha, frame in bound_eval.groupby("alpha", dropna=False): - alpha_value = float(str(alpha)) - exact_alpha_summary[str(alpha_value)] = { - "n_checks": int(len(frame)), - "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), - "max_violation": float(frame["violation"].max()), - "mean_gamma_cp": float(frame["gamma_cp"].mean()), - "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), - } - - return { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "overall_exact_shortlist": _bucket_payload(shortlist_eval), - "by_shortlist_bucket": bucket_rows, - "by_alpha": exact_alpha_summary, - "claim_lenses": { - "same_region_higher_return": "maximize realized_total_return among alpha01 passers", - "same_region_better_bound": "minimize alpha01_gamma_cp and alpha01_weighted_miscoverage_V among alpha01 passers", - "wider_region_zero_violations": "maximize n_alpha01_passers and alpha01_pass_rate", - "clean_region_for_paper": "prefer forced_incumbent_neighbors and incumbent_region buckets when they pass", - "return_bound_frontier": "retain exact shortlist, not only the selected policy", - }, - } - - -def _selection_reason(row: pd.Series) -> str: - if bool(row["alpha01_exact_pass"]): - return "selected_best_alpha01_exact_pass" - if bool(row["alpha03_exact_pass"]): - return "selected_fallback_alpha03_exact_pass" - return "selected_best_available_without_alpha01_pass" - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser() - parser.add_argument("--config", default="configs/crpto_optimization.yaml") - parser.add_argument("--conformal-intervals-path", required=True) - parser.add_argument("--run-label", default="rank1_bound_aware") - parser.add_argument("--output-dir", default="") - parser.add_argument("--model-dir", default="") - parser.add_argument("--risk-grid", default="0.14,0.15,0.16,0.17") - parser.add_argument("--aversion-grid", default="0.0,0.25") - parser.add_argument("--gamma-grid", default="0.2,0.35,0.5") - parser.add_argument("--delta-cap-grid", default="0.5,0.75,1.0") - parser.add_argument("--tail-focus-grid", default="0.9,0.95,1.0") - parser.add_argument("--budget-profiles", default="free,floored") - parser.add_argument("--shortlist-top-k", type=int, default=160) - parser.add_argument("--bucket-return-k", type=int, default=40) - parser.add_argument("--bucket-proxy-k", type=int, default=40) - parser.add_argument("--bucket-family-k", type=int, default=20) - parser.add_argument("--bucket-region-k", type=int, default=20) - parser.add_argument("--alpha-grid", default=IJDS_DECLARED_ALPHA_GRID_CSV) - parser.add_argument("--max-candidates", type=int, default=5000) - parser.add_argument( - "--exact-max-candidates", - type=int, - default=None, - help=( - "Candidate universe for exact rerank. Use 0 for the full universe. " - "Defaults to --max-candidates." - ), - ) - parser.add_argument("--random-state", type=int, default=42) - parser.add_argument("--random-states", default="") - parser.add_argument( - "--exact-random-states", - default="", - help=( - "Comma-separated seeds for exact rerank. Defaults to --random-states, " - "allowing a cheap proxy frontier and a stronger exact certificate." - ), - ) - parser.add_argument( - "--exact-checkpoint-every", - type=int, - default=100, - help="Write partial exact bound cache every N checks.", - ) - parser.add_argument( - "--exact-threads", - type=int, - default=_env_int("EXACT_THREADS", DEFAULT_THREADS), - help="Thread budget for exact HiGHS reranking when it runs in this process.", - ) - parser.add_argument("--solver-backend", choices=["highs", "cuopt"], default="highs") - parser.add_argument("--exact-solver-backend", choices=["highs", "cuopt"], default="highs") - parser.add_argument("--exact-python-executable", default="") - parser.add_argument("--exact-helper-script", default=str(DEFAULT_EXACT_HELPER_SCRIPT)) - parser.add_argument( - "--frontier-only", - action="store_true", - help="Write frontier, shortlist, and exact context, then defer exact bound reranking.", - ) - parser.add_argument("--cuopt-presolve", type=int, default=1) - parser.add_argument("--cuopt-method", default="") - parser.add_argument("--cuopt-pdlp-solver-mode", default="") - parser.add_argument("--cuopt-pdlp-precision", default="") - parser.add_argument("--cuopt-crossover", default="") - parser.add_argument("--cuopt-first-primal-feasible", default="") - parser.add_argument("--cuopt-save-best-primal-solution", default="") - parser.add_argument("--cuopt-infeasibility-detection", default="") - parser.add_argument("--cuopt-strict-infeasibility", default="") - parser.add_argument("--cuopt-per-constraint-residual", default="") - parser.add_argument("--cuopt-dual-postsolve", default="") - parser.add_argument("--cuopt-dualize", default="") - parser.add_argument("--cuopt-folding", default="") - parser.add_argument("--cuopt-augmented", default="") - parser.add_argument("--cuopt-ordering", default="") - parser.add_argument("--cuopt-cudss-deterministic", default="") - parser.add_argument("--cuopt-eliminate-dense-columns", default="") - parser.add_argument("--cuopt-iteration-limit", default="") - parser.add_argument("--cuopt-num-cpu-threads", default="") - parser.add_argument("--cuopt-num-gpus", default="") - parser.add_argument("--cuopt-log-to-console", default="false") - parser.add_argument("--cuopt-log-dir", default="") - parser.add_argument("--cuopt-extra-parameter", action="append", default=[]) - parser.add_argument("--policy-modes", default="") - parser.add_argument( - "--incumbent-policy-path", - default=str(DEFAULT_INCUMBENT_POLICY_PATH), - ) - parser.add_argument("--incumbent-risk-neighbors", default="0.155,0.16,0.165,0.17") - parser.add_argument("--incumbent-gamma-neighbors", default="0.45,0.5,0.55") - parser.add_argument( - "--incumbent-policy-modes", - default="blended_uncertainty,capped_blended_uncertainty", - ) - parser.add_argument("--budget", type=float, default=1_000_000.0) - parser.add_argument("--t-eval", type=float, default=0.05) - return parser - - -def _sanitize_run_label(raw: object) -> str: - return str(raw).strip().replace("/", "_") - - -def _resolve_run_paths(args: argparse.Namespace, *, run_label: str) -> BoundAwarePaths: - output_dir = ( - Path(str(args.output_dir)).expanduser() - if str(args.output_dir).strip() - else ROOT / "data" / "processed" / "portfolio_bound_aware" / run_label - ) - model_dir = ( - Path(str(args.model_dir)).expanduser() - if str(args.model_dir).strip() - else ROOT / "models" / "portfolio_bound_aware" / run_label - ) - return BoundAwarePaths( - output_dir=output_dir, - model_dir=model_dir, - status_path=model_dir / f"{STAGE_NAME}_runtime_status.json", - checkpoint_dir=model_dir / f"{STAGE_NAME}_runtime_checkpoints", - resource_path=model_dir / "resource_snapshot.json", - gpu_csv_path=model_dir / "gpu_samples.csv", - frontier_raw_path=output_dir / "portfolio_bound_aware_frontier_raw.parquet", - frontier_path=output_dir / "portfolio_bound_aware_frontier.parquet", - shortlist_path=output_dir / "portfolio_bound_aware_shortlist.parquet", - shortlist_exact_path=output_dir / "portfolio_bound_aware_shortlist_exact.parquet", - bound_eval_path=output_dir / "portfolio_bound_aware_bound_eval.parquet", - region_summary_path=model_dir / "portfolio_bound_aware_region_summary.json", - selection_path=model_dir / "portfolio_bound_aware_selection.json", - exact_context_path=model_dir / "portfolio_bound_aware_exact_context.json", - ) - - -def _ensure_run_dirs(paths: BoundAwarePaths) -> None: - paths.output_dir.mkdir(parents=True, exist_ok=True) - paths.model_dir.mkdir(parents=True, exist_ok=True) - - -def _budget_profiles(raw: object) -> list[dict[str, Any]]: - budget_profiles: list[dict[str, Any]] = [] - tokens = [part.strip().lower() for part in str(raw).split(",") if part.strip()] - for token in tokens: - if token == "free": - budget_profiles.append( - {"name": "free_budget", "min_budget_utilization": 0.0, "pd_cap_slack_penalty": 0.0} - ) - elif token == "floored": - budget_profiles.append( - { - "name": "floored_budget", - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - } - ) - else: - raise ValueError(f"Unsupported budget profile: {token}") - if not budget_profiles: - raise ValueError("At least one budget profile is required") - return budget_profiles - - -def _search_space_payload( - *, - args: argparse.Namespace, - risk_values: list[float], - aversion_values: list[float], - gamma_values: list[float], - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], - budget_profiles: list[dict[str, Any]], - alpha_grid: list[float], - random_states: list[int], - exact_random_states: list[int], - exact_max_candidates: int, - policy_modes: list[str], - cuopt_parameters: dict[str, Any], - incumbent_risk_neighbors: list[float], - incumbent_gamma_neighbors: list[float], - incumbent_policy_modes: list[str], -) -> dict[str, Any]: - return { - "risk_grid": risk_values, - "aversion_grid": aversion_values, - "gamma_grid": gamma_values, - "delta_cap_grid": delta_cap_quantiles, - "tail_focus_grid": tail_focus_quantiles, - "budget_profiles": budget_profiles, - "alpha_grid": alpha_grid, - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "policy_modes": policy_modes, - "cuopt_parameters": cuopt_parameters, - "bucket_return_k": int(args.bucket_return_k), - "bucket_proxy_k": int(args.bucket_proxy_k), - "bucket_family_k": int(args.bucket_family_k), - "bucket_region_k": int(args.bucket_region_k), - "incumbent_policy_path": str(args.incumbent_policy_path), - "incumbent_risk_neighbors": incumbent_risk_neighbors, - "incumbent_gamma_neighbors": incumbent_gamma_neighbors, - "incumbent_policy_modes": incumbent_policy_modes, - } - - -def _selection_policy_payload() -> dict[str, Any]: - return { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "alpha_exact_pass_count(desc)", - "alpha_exact_pass_rate(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)", - "price_of_robustness(asc)", - ], - } - - -def _selection_context_payload( - *, - args: argparse.Namespace, - paths: BoundAwarePaths, - run_label: str, - search_space: dict[str, Any], - exact_max_candidates: int, - random_states: list[int], - exact_random_states: list[int], - alpha_grid: list[float], -) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": search_space, - "selection_policy": _selection_policy_payload(), - "frontier_raw_path": str(paths.frontier_raw_path), - "frontier_path": str(paths.frontier_path), - "shortlist_path": str(paths.shortlist_path), - "shortlist_exact_path": str(paths.shortlist_exact_path), - "bound_eval_path": str(paths.bound_eval_path), - "region_summary_path": str(paths.region_summary_path), - "selection_path": str(paths.selection_path), - "runtime_status_path": str(paths.status_path), - "runtime_checkpoint_dir": str(paths.checkpoint_dir), - "resource_snapshot_path": str(paths.resource_path), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "budget": float(args.budget), - "t_eval": float(args.t_eval), - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "alpha_grid": alpha_grid, - } - - -def _run_in_process_exact_bound_eval( - *, - args: argparse.Namespace, - shortlist: pd.DataFrame, - alpha_grid: list[float], - exact_random_states: list[int], - exact_max_candidates: int, - tracker: _ProgressTracker, -) -> pd.DataFrame: - aligned_by_seed = { - int(seed): _load_aligned_dataset( - conformal_intervals_path=args.conformal_intervals_path, - max_candidates=int(exact_max_candidates), - random_state=int(seed), - ) - for seed in exact_random_states - } - bound_rows: list[dict[str, Any]] = [] - completed_checks = 0 - for _, row in shortlist.iterrows(): - policy = _policy_from_row( - row, - solver_backend_override=str(args.exact_solver_backend), - ) - candidate_payload = row.to_dict() - for eval_seed in exact_random_states: - aligned = aligned_by_seed[int(eval_seed)] - for alpha in alpha_grid: - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="exact", - budget=float(args.budget), - t_eval=float(args.t_eval), - threads=int(args.exact_threads), - ) - bound_rows.append( - { - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - **candidate_payload, - **result, - } - ) - completed_checks += 1 - tracker.bound_progress( - completed_checks=completed_checks, - extra={ - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "current_alpha": float(alpha), - "exact_threads": int(args.exact_threads), - }, - ) - return pd.DataFrame(bound_rows) - - -def _build_selection_payload( - *, - args: argparse.Namespace, - run_label: str, - selection_context: dict[str, Any], - selected: pd.Series, - selected_policy: dict[str, Any], - region_payload: dict[str, Any], -) -> dict[str, Any]: - return { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": selection_context["search_space"], - "selection_policy": selection_context["selection_policy"], - "selected_policy": selected_policy, - "selected_metrics": selected.to_dict(), - "selection_reason": _selection_reason(selected), - "frontier_raw_path": selection_context["frontier_raw_path"], - "frontier_path": selection_context["frontier_path"], - "shortlist_path": selection_context["shortlist_path"], - "shortlist_exact_path": selection_context["shortlist_exact_path"], - "bound_eval_path": selection_context["bound_eval_path"], - "region_summary_path": selection_context["region_summary_path"], - "robust_region_summary": region_payload, - "runtime_status_path": selection_context["runtime_status_path"], - "runtime_checkpoint_dir": selection_context["runtime_checkpoint_dir"], - "resource_snapshot_path": selection_context["resource_snapshot_path"], - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "exact_threads": int(args.exact_threads), - } - - -def _write_selection_outputs( - *, - paths: BoundAwarePaths, - shortlist_eval: pd.DataFrame, - bound_eval: pd.DataFrame, - region_payload: dict[str, Any], - payload: dict[str, Any], -) -> None: - atomic_write_parquet(shortlist_eval, paths.shortlist_exact_path, index=False) - atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) - atomic_write_json(paths.region_summary_path, region_payload) - atomic_write_json(paths.selection_path, payload) - - -def _build_frontier_outputs( - *, - args: argparse.Namespace, - risk_values: list[float], - aversion_values: list[float], - gamma_values: list[float], - delta_cap_quantiles: list[float], - tail_focus_quantiles: list[float], - budget_profiles: list[dict[str, Any]], - random_states: list[int], - policy_grid_count: int, - policy_modes: list[str], - cuopt_parameters: dict[str, Any], - tracker: _ProgressTracker, -) -> tuple[pd.DataFrame, pd.DataFrame]: - frontier_frames: list[pd.DataFrame] = [] - frontier_completed = 0 - for seed in random_states: - seed_offset = frontier_completed - - def _progress_hook( - local_completed: int, - extra: dict[str, Any], - _offset: int = seed_offset, - ) -> None: - tracker.frontier_progress(completed_units=_offset + local_completed, extra=extra) - - frontier_seed = _build_frontier_for_seed( - config_path=args.config, - conformal_intervals_path=args.conformal_intervals_path, - risk_values=risk_values, - aversion_values=aversion_values, - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - budget_profiles=budget_profiles, - max_candidates=int(args.max_candidates), - random_state=int(seed), - solver_backend=str(args.solver_backend), - cuopt_presolve=int(args.cuopt_presolve) - if str(args.solver_backend) == "cuopt" - else None, - cuopt_parameters=cuopt_parameters, - policy_modes=policy_modes, - progress_hook=_progress_hook, - ) - frontier_frames.append(frontier_seed) - frontier_completed += int(len(risk_values) * (1 + policy_grid_count)) - - frontier_raw = ( - pd.concat(frontier_frames, ignore_index=True) if frontier_frames else pd.DataFrame() - ) - if frontier_raw.empty: - raise ValueError("Frontier search produced zero candidate rows.") - return frontier_raw, _aggregate_frontier(frontier_raw) - - -def _write_frontier_artifacts( - *, - paths: BoundAwarePaths, - frontier_raw: pd.DataFrame, - frontier: pd.DataFrame, - shortlist: pd.DataFrame, -) -> None: - atomic_write_parquet(frontier_raw, paths.frontier_raw_path, index=False) - atomic_write_parquet(frontier, paths.frontier_path, index=False) - atomic_write_parquet(shortlist, paths.shortlist_path, index=False) - - -def _complete_frontier_only( - *, - paths: BoundAwarePaths, - tracker: _ProgressTracker, - resource_payload: dict[str, Any], - shortlist_extra: dict[str, Any], - bound_total_checks: int, -) -> None: - resource_payload["end"] = _resource_snapshot() - atomic_write_json(paths.resource_path, resource_payload) - tracker.frontier_only_complete( - extra={ - **shortlist_extra, - "frontier_only": True, - "deferred_bound_total_checks": bound_total_checks, - "exact_context_path": str(paths.exact_context_path), - "frontier_path": str(paths.frontier_path), - "shortlist_path": str(paths.shortlist_path), - } - ) - - -def _delegate_exact_stage_if_requested( - *, - exact_python_executable: str, - exact_helper_script: Path, - paths: BoundAwarePaths, - resource_payload: dict[str, Any], -) -> bool: - if not exact_python_executable: - return False - current_python = Path(sys.executable) - requested_python = Path(exact_python_executable) - if requested_python == current_python: - return False - cmd = [ - str(requested_python), - str(exact_helper_script), - "--context-path", - str(paths.exact_context_path), - ] - logger.info("Delegating exact bound stage to external Python: {}", " ".join(cmd)) - subprocess.run(cmd, cwd=str(ROOT), check=True) - resource_payload["end"] = _resource_snapshot() - atomic_write_json(paths.resource_path, resource_payload) - return True - - -def _build_grid_spec(args: argparse.Namespace) -> BoundAwareGridSpec: - random_states = _coerce_int_csv(args.random_states, fallback=int(args.random_state)) - exact_random_states = ( - _coerce_int_csv(args.exact_random_states, fallback=int(args.random_state)) - if str(args.exact_random_states).strip() - else list(random_states) - ) - exact_max_candidates = ( - int(args.exact_max_candidates) - if args.exact_max_candidates is not None - else int(args.max_candidates) - ) - return BoundAwareGridSpec( - risk_values=_coerce_csv(args.risk_grid), - aversion_values=_coerce_csv(args.aversion_grid), - gamma_values=_coerce_csv(args.gamma_grid), - delta_cap_quantiles=_coerce_csv(args.delta_cap_grid), - tail_focus_quantiles=_coerce_csv(args.tail_focus_grid), - alpha_grid=_coerce_csv(args.alpha_grid), - random_states=random_states, - exact_random_states=exact_random_states, - exact_max_candidates=exact_max_candidates, - budget_profiles=_budget_profiles(args.budget_profiles), - policy_modes=_coerce_str_csv(args.policy_modes), - incumbent_risk_neighbors=_coerce_csv(args.incumbent_risk_neighbors), - incumbent_gamma_neighbors=_coerce_csv(args.incumbent_gamma_neighbors), - incumbent_policy_modes=_coerce_str_csv(args.incumbent_policy_modes), - ) - - -def _prepare_execution_spec(args: argparse.Namespace) -> BoundAwareExecutionSpec: - run_label = _sanitize_run_label(args.run_label) - paths = _resolve_run_paths(args, run_label=run_label) - _ensure_run_dirs(paths) - grid = _build_grid_spec(args) - uses_cuopt = "cuopt" in { - str(args.solver_backend).strip().lower(), - str(args.exact_solver_backend).strip().lower(), - } - backend_validation = _validate_cuopt_runtime() if uses_cuopt else None - cuopt_parameters = ( - _cuopt_parameter_overrides(args, model_dir=paths.model_dir) - if str(args.solver_backend).strip().lower() == "cuopt" - else {} - ) - return BoundAwareExecutionSpec( - run_label=run_label, - paths=paths, - grid=grid, - incumbent_policy=_load_incumbent_policy(args.incumbent_policy_path), - exact_python_executable=str(args.exact_python_executable).strip(), - exact_helper_script=Path(str(args.exact_helper_script)).resolve(), - backend_validation=backend_validation, - cuopt_parameters=cuopt_parameters, - ) - - -def _initialize_run_context(args: argparse.Namespace) -> BoundAwareRunContext: - spec = _prepare_execution_spec(args) - grid = spec.grid - tracker = _ProgressTracker( - status_path=spec.paths.status_path, - checkpoint_dir=spec.paths.checkpoint_dir, - run_tag=spec.run_label, - frontier_total_units=grid.frontier_total_units, - ) - search_space = _search_space_payload( - args=args, - risk_values=grid.risk_values, - aversion_values=grid.aversion_values, - gamma_values=grid.gamma_values, - delta_cap_quantiles=grid.delta_cap_quantiles, - tail_focus_quantiles=grid.tail_focus_quantiles, - budget_profiles=grid.budget_profiles, - alpha_grid=grid.alpha_grid, - random_states=grid.random_states, - exact_random_states=grid.exact_random_states, - exact_max_candidates=grid.exact_max_candidates, - policy_modes=grid.policy_modes, - cuopt_parameters=spec.cuopt_parameters, - incumbent_risk_neighbors=grid.incumbent_risk_neighbors, - incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, - incumbent_policy_modes=grid.incumbent_policy_modes, - ) - resource_payload = { - "schema_version": SCHEMA_VERSION, - "run_label": spec.run_label, - "solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "cuopt_parameters": spec.cuopt_parameters, - "start": _resource_snapshot(), - "backend_validation": spec.backend_validation, - } - atomic_write_json(spec.paths.resource_path, resource_payload) - tracker.start( - extra={ - "random_states": grid.random_states, - "search_space": search_space, - } - ) - return BoundAwareRunContext( - args=args, - spec=spec, - tracker=tracker, - search_space=search_space, - resource_payload=resource_payload, - ) - - -def _start_gpu_sampler(context: BoundAwareRunContext) -> _GpuSampler | None: - if str(context.args.solver_backend).strip().lower() == "cuopt": - gpu_sampler = _GpuSampler(context.spec.paths.gpu_csv_path) - gpu_sampler.start() - return gpu_sampler - return None - - -def _build_frontier_state(context: BoundAwareRunContext) -> BoundAwareFrontierState: - args = context.args - spec = context.spec - grid = spec.grid - tracker = context.tracker - frontier_raw, frontier = _build_frontier_outputs( - args=args, - risk_values=grid.risk_values, - aversion_values=grid.aversion_values, - gamma_values=grid.gamma_values, - delta_cap_quantiles=grid.delta_cap_quantiles, - tail_focus_quantiles=grid.tail_focus_quantiles, - budget_profiles=grid.budget_profiles, - random_states=grid.random_states, - policy_grid_count=grid.policy_grid_count, - policy_modes=grid.policy_modes, - cuopt_parameters=spec.cuopt_parameters, - tracker=tracker, - ) - tracker.frontier_complete( - extra={ - "frontier_policy_count": len(frontier), - "frontier_raw_rows": len(frontier_raw), - } - ) - shortlist = _build_stratified_shortlist( - frontier=frontier, - shortlist_top_k=int(args.shortlist_top_k), - bucket_return_k=int(args.bucket_return_k), - bucket_proxy_k=int(args.bucket_proxy_k), - bucket_family_k=int(args.bucket_family_k), - bucket_region_k=int(args.bucket_region_k), - incumbent_policy=spec.incumbent_policy, - incumbent_risk_neighbors=grid.incumbent_risk_neighbors, - incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, - incumbent_policy_modes=grid.incumbent_policy_modes, - budget_profiles=grid.budget_profiles, - solver_backend=str(args.solver_backend), - ) - bound_total_checks = grid.bound_total_checks(len(shortlist)) - shortlist_extra = { - "shortlist_size": len(shortlist), - "shortlist_buckets": shortlist["shortlist_bucket"].value_counts(dropna=False).to_dict(), - } - if not args.frontier_only: - tracker.set_bound_total(bound_total_checks, extra=shortlist_extra) - _write_frontier_artifacts( - paths=spec.paths, - frontier_raw=frontier_raw, - frontier=frontier, - shortlist=shortlist, - ) - selection_context = _selection_context_payload( - args=args, - paths=spec.paths, - run_label=spec.run_label, - search_space=context.search_space, - exact_max_candidates=grid.exact_max_candidates, - random_states=grid.random_states, - exact_random_states=grid.exact_random_states, - alpha_grid=grid.alpha_grid, - ) - atomic_write_json(spec.paths.exact_context_path, selection_context) - return BoundAwareFrontierState( - frontier_raw=frontier_raw, - frontier=frontier, - shortlist=shortlist, - bound_total_checks=bound_total_checks, - shortlist_extra=shortlist_extra, - selection_context=selection_context, - ) - - -def _finish_after_frontier( - context: BoundAwareRunContext, - state: BoundAwareFrontierState, -) -> bool: - spec = context.spec - if context.args.frontier_only: - _complete_frontier_only( - paths=spec.paths, - tracker=context.tracker, - resource_payload=context.resource_payload, - shortlist_extra=state.shortlist_extra, - bound_total_checks=state.bound_total_checks, - ) - return True - return _delegate_exact_stage_if_requested( - exact_python_executable=spec.exact_python_executable, - exact_helper_script=spec.exact_helper_script, - paths=spec.paths, - resource_payload=context.resource_payload, - ) - - -def _run_exact_selection( - context: BoundAwareRunContext, - state: BoundAwareFrontierState, -) -> tuple[pd.Series, dict[str, Any]]: - args = context.args - spec = context.spec - grid = spec.grid - bound_eval = _run_in_process_exact_bound_eval( - args=args, - shortlist=state.shortlist, - alpha_grid=grid.alpha_grid, - exact_random_states=grid.exact_random_states, - exact_max_candidates=grid.exact_max_candidates, - tracker=context.tracker, - ) - shortlist_eval = _aggregate_exact_results(shortlist=state.shortlist, bound_eval=bound_eval) - region_payload = _region_summary(shortlist_eval, bound_eval) - selected = shortlist_eval.iloc[0].copy() - selected_policy = _policy_from_row( - selected, - solver_backend_override=str(args.exact_solver_backend), - ) - payload = _build_selection_payload( - args=args, - run_label=spec.run_label, - selection_context=state.selection_context, - selected=selected, - selected_policy=selected_policy, - region_payload=region_payload, - ) - _write_selection_outputs( - paths=spec.paths, - shortlist_eval=shortlist_eval, - bound_eval=bound_eval, - region_payload=region_payload, - payload=payload, - ) - return selected, payload - - -def _complete_run( - context: BoundAwareRunContext, - *, - selected: pd.Series, - payload: dict[str, Any], - gpu_sampler: _GpuSampler | None, -) -> None: - if gpu_sampler is not None: - context.resource_payload["gpu_summary"] = gpu_sampler.stop() - context.resource_payload["end"] = _resource_snapshot() - atomic_write_json(context.spec.paths.resource_path, context.resource_payload) - context.tracker.complete( - extra={ - "selection_reason": str(payload["selection_reason"]), - "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), - "selected_realized_total_return": float(selected["realized_total_return"]), - } - ) - - -def _log_completed_run( - paths: BoundAwarePaths, - *, - selected: pd.Series, -) -> None: - logger.info( - "Focused bound-aware search complete: selected risk_tolerance={}, mode={}, gamma={}, q_cap={}, q_tail={}, ab_pass_all={}, alpha01_pass={}", - selected["risk_tolerance"], - selected["policy_mode"], - selected["gamma"], - selected["delta_cap_quantile"], - selected["tail_focus_quantile"], - selected["ab_pass_all"], - selected["alpha01_exact_pass"], - ) - for label, path in ( - ("frontier raw", paths.frontier_raw_path), - ("frontier aggregate", paths.frontier_path), - ("shortlist", paths.shortlist_path), - ("exact shortlist", paths.shortlist_exact_path), - ("bound evaluations", paths.bound_eval_path), - ("selection payload", paths.selection_path), - ): - logger.info("Saved {}: {}", label, path) - - -def _execute_bound_aware_search( - context: BoundAwareRunContext, - *, - gpu_sampler: _GpuSampler | None, -) -> int: - state = _build_frontier_state(context) - if _finish_after_frontier(context, state): - return 0 - selected, payload = _run_exact_selection(context, state) - _complete_run(context, selected=selected, payload=payload, gpu_sampler=gpu_sampler) - _log_completed_run(context.spec.paths, selected=selected) - return 0 - - -def _record_run_failure(context: BoundAwareRunContext, exc: Exception) -> None: - tracker = context.tracker - error_payload = { - "error_type": type(exc).__name__, - "error": str(exc), - "frontier_completed_units": int(tracker.frontier_completed_units), - "frontier_total_units": int(tracker.frontier_total_units), - "bound_completed_checks": int(tracker.bound_completed_checks), - "bound_total_checks": int(tracker.bound_total_checks), - } - context.resource_payload["error"] = error_payload - context.resource_payload["end"] = _resource_snapshot() - atomic_write_json(context.spec.paths.resource_path, context.resource_payload) - tracker.fail(phase="failed", extra=error_payload) - logger.exception("Focused bound-aware portfolio search failed.") - - -def _finalize_gpu_sampler( - context: BoundAwareRunContext, - gpu_sampler: _GpuSampler | None, -) -> None: - if gpu_sampler is None or "gpu_summary" in context.resource_payload: - return - try: - context.resource_payload["gpu_summary"] = gpu_sampler.stop() - context.resource_payload["end"] = _resource_snapshot() - atomic_write_json(context.spec.paths.resource_path, context.resource_payload) - except Exception: # pragma: no cover - best effort cleanup only - pass - - -def main(argv: list[str] | None = None) -> int: - args = _build_parser().parse_args(argv) - context = _initialize_run_context(args) - gpu_sampler = _start_gpu_sampler(context) - - try: - return _execute_bound_aware_search(context, gpu_sampler=gpu_sampler) - except Exception as exc: - _record_run_failure(context, exc) - raise - finally: - _finalize_gpu_sampler(context, gpu_sampler) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/search/run_portfolio_search.py b/scripts/search/run_portfolio_search.py deleted file mode 100644 index 7982599..0000000 --- a/scripts/search/run_portfolio_search.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Retired portfolio search entrypoint. - -The former generic ``scripts.run_long_pipeline`` orchestrator was removed when -the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a -readable stop sign for old commands instead of failing with an import error. -""" - -from __future__ import annotations - -import sys - - -def main(argv: list[str] | None = None) -> int: - _ = argv - sys.stderr.write( - "scripts/search/run_portfolio_search.py is retired. The submitted IJDS " - "claim uses the closed finite-grid frontier; use " - "scripts/search/run_pool93_ijds_local_refinement.py only for an explicitly " - "tagged isolated refinement, not as a default paper path.\n" - ) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/search/run_regret_auditability_sandbox.py b/scripts/search/run_regret_auditability_sandbox.py deleted file mode 100644 index 9e02ed7..0000000 --- a/scripts/search/run_regret_auditability_sandbox.py +++ /dev/null @@ -1,2654 +0,0 @@ -"""Resumable CPU-only sandbox for regret-auditability search planning. - -The sandbox is intentionally isolated from the frozen CRPTO champion artifacts. -Its default outputs live outside the repository under -``D:/crpto_experiments/regret_auditability/``. -""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import os -import pickle -import shutil -import sqlite3 -import subprocess -import sys -import time -from collections import deque -from collections.abc import Iterable, Mapping, Sequence -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait -from dataclasses import asdict, dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, cast - -import yaml - -ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT)) - -from src.features.feature_config_io import load_feature_config # noqa: E402 -from src.optimization.certificate_semantics import ( # noqa: E402 - IJDS_DECLARED_ALPHA_GRID_CSV, -) -from src.utils.pipeline_runtime import atomic_write_json # noqa: E402 - - -def _env_int(name: str, default: int) -> int: - raw = os.environ.get(name, "").strip() - if not raw: - return int(default) - try: - return int(raw) - except ValueError: - return int(default) - - -def _env_str(name: str, default: str) -> str: - raw = os.environ.get(name, "").strip() - return raw or default - - -SCHEMA_VERSION = "2026-05-12.1" -STAGE_NAME = "regret_auditability_sandbox" -DEFAULT_ARTIFACT_ROOT_BASE = Path("D:/crpto_experiments/regret_auditability") -CHAMPION_PD_CONFIG_PATH = ROOT / "configs" / "crpto_pd_model.yaml" -CHAMPION_PORTFOLIO_POLICY_PATH = ROOT / "models" / "champion_portfolio_policy.json" -DEFAULT_RESERVED_LOGICAL_CPUS = 2 -DEFAULT_MIN_AVAILABLE_RAM_GB = 10.0 -DEFAULT_HEARTBEAT_SECONDS = 60 -DEFAULT_PD_WORKERS = 4 -DEFAULT_PD_THREADS = 5 -MLFLOW_LOG_SIZE_LIMIT_BYTES = 100 * 1024 * 1024 -PD_SMOKE_TRIALS = _env_int("CRPTO_SANDBOX_PD_SMOKE_TRIALS", 12) -PD_BROAD_TRIALS = _env_int("CRPTO_SANDBOX_PD_BROAD_TRIALS", 300) -PD_REFINE_TRIALS = _env_int("CRPTO_SANDBOX_PD_REFINE_TRIALS", 300) -PD_BROAD_TOP_K_LANES = _env_int("CRPTO_SANDBOX_PD_BROAD_TOP_K", 6) -PD_REFINE_TOP_K_LANES = _env_int("CRPTO_SANDBOX_PD_REFINE_TOP_K", 3) -PORTFOLIO_RISK_GRID = _env_str( - "CRPTO_SANDBOX_PORTFOLIO_RISK_GRID", - "0.08,0.09,0.10,0.11,0.12,0.13,0.14,0.15,0.155,0.16,0.165,0.17,0.175,0.18,0.19,0.20,0.21,0.22,0.23,0.24,0.25", -) -PORTFOLIO_GAMMA_GRID = _env_str( - "CRPTO_SANDBOX_PORTFOLIO_GAMMA_GRID", - "0,0.10,0.20,0.30,0.40,0.425,0.45,0.475,0.50,0.525,0.55,0.575,0.60,0.70,0.80,0.90,1.0", -) -PORTFOLIO_AVERSION_GRID = _env_str( - "CRPTO_SANDBOX_PORTFOLIO_AVERSION_GRID", - "0,0.02,0.05,0.10,0.25,0.50", -) -PORTFOLIO_CAP_TAIL_GRID = _env_str( - "CRPTO_SANDBOX_PORTFOLIO_CAP_TAIL_GRID", - "0.60,0.75,0.90,1.0", -) -PORTFOLIO_RANDOM_STATES = _env_str("CRPTO_SANDBOX_PORTFOLIO_RANDOM_STATES", "42,52,62") -PORTFOLIO_ALPHA_GRID = _env_str( - "CRPTO_SANDBOX_PORTFOLIO_ALPHA_GRID", - IJDS_DECLARED_ALPHA_GRID_CSV, -) -PORTFOLIO_MAX_CANDIDATES = _env_int("CRPTO_SANDBOX_PORTFOLIO_MAX_CANDIDATES", 100000) -PORTFOLIO_SHORTLIST_TOP_K = _env_int("CRPTO_SANDBOX_PORTFOLIO_SHORTLIST_TOP_K", 1000) - -PD_PHASES = {"pd-smoke", "pd-broad", "pd-refine"} -PHASE_CHOICES = ( - "plan", - "deps", - "pd-smoke", - "pd-broad", - "pd-refine", - "conformal", - "portfolio", - "metrics", - "all", -) - -MONOTONIC_POLICIES: dict[str, dict[str, int]] = { - "canonical_4": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - }, - "affordability_rate_5": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - "int_rate": 1, - }, - "credit_history_7": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - "int_rate": 1, - "delinq_severity": 1, - "delinq_recency": -1, - }, - "bureau_utilization_11": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - "int_rate": 1, - "rev_utilization": 1, - "high_util_pct": 1, - "bc_util": 1, - "percent_bc_gt_75": 1, - "fico_score": -1, - "credit_age_years": -1, - }, - "bureau_behavior_15": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - "int_rate": 1, - "delinq_severity": 1, - "delinq_recency": -1, - "pub_rec": 1, - "has_bankruptcy": 1, - "num_accts_ever_120_pd": 1, - "num_tl_90g_dpd_24m": 1, - "num_tl_30dpd": 1, - "pct_tl_nvr_dlq": -1, - "fico_score": -1, - "credit_age_years": -1, - }, - "inquiry_velocity_12": { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - "int_rate": 1, - "inq_last_6mths": 1, - "inq_last_12m": 1, - "inq_fi": 1, - "mths_since_recent_inq": -1, - "acc_open_past_24mths": 1, - "num_tl_op_past_12m": 1, - "fico_score": -1, - }, -} - -FEATURE_PROFILES: dict[str, dict[str, Any]] = { - "core_stable": { - "description": "Champion-like CatBoost feature set with the stable-core gate enabled.", - "groups": ["CATBOOST_FEATURES"], - "stable_core_enabled": True, - }, - "core_wide": { - "description": "Champion-like CatBoost feature set without the stable-core gate.", - "groups": ["CATBOOST_FEATURES"], - "stable_core_enabled": False, - }, - "core_woe": { - "description": "Champion-like CatBoost features plus train-only WOE transforms.", - "groups": ["CATBOOST_FEATURES", "WOE_FEATURES"], - "stable_core_enabled": False, - }, - "bureau_high": { - "description": "CatBoost core plus high-coverage bureau utilization and balance fields.", - "groups": ["CATBOOST_FEATURES", "HIGH_COVERAGE_BUREAU_FEATURES"], - "stable_core_enabled": False, - }, - "full_challenger": { - "description": "All materialized challenger features from feature_config.yml.", - "groups": ["CHALLENGER_FEATURE_POOL_V2"], - "stable_core_enabled": False, - }, - "full_challenger_woe": { - "description": "All materialized challenger features plus WOE transforms.", - "groups": ["CHALLENGER_FEATURE_POOL_V2", "WOE_FEATURES"], - "stable_core_enabled": False, - }, -} - -AUDITABILITY_WEIGHTS: dict[str, int] = { - "coverage90_pass": 15, - "coverage95_pass": 10, - "min_group_coverage_pass": 15, - "no_critical_alerts": 10, - "alpha01_exact_pass": 15, - "violation_zero": 10, - "v_within_sqrt_alpha": 10, - "monotonic_audit_pass": 10, - "reproducible_resume_manifest": 5, -} - -PROTECTED_REPO_PATHS = ( - "EXTRACTION_MANIFEST.json", - "models/pd_canonical.cbm", - "models/pd_canonical_calibrator.pkl", - "models/final_project_promotion.json", - "models/conformal_policy_status.json", - "data/processed/conformal_intervals_mondrian.parquet", -) -PROTECTED_REPO_DIRS = ( - "data/processed/portfolio_bound_aware", - "data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734", -) -PROTECTED_REPO_GLOBS = ( - "data/processed/portfolio_bound_aware/rank1_*", - "models/portfolio_bound_aware/rank1_*", -) - - -@dataclass(frozen=True) -class PhaseCommand: - """A resumable unit the sandbox can plan or execute.""" - - name: str - phase: str - command: list[str] - outputs: list[str] - checkpoint: str - env: dict[str, str] - max_workers: int - cpu_threads: int - feature_profile: str = "" - monotonic_policy: str = "" - lane_id: str = "" - stdout_log: str = "" - stderr_log: str = "" - - -def utc_now_iso() -> str: - """Return an ISO timestamp in UTC.""" - return datetime.now(tz=UTC).isoformat() - - -def default_run_tag() -> str: - """Create a stable human-readable run tag.""" - return datetime.now(tz=UTC).strftime("regret_auditability_%Y%m%d_%H%M%S") - - -def default_artifact_root(run_tag: str) -> Path: - """Return the default external artifact root for a run tag.""" - return DEFAULT_ARTIFACT_ROOT_BASE / sanitize_tag(run_tag) - - -def sanitize_tag(raw: str) -> str: - """Normalize a user-provided tag for filesystem use.""" - safe = str(raw).strip().replace("/", "_").replace("\\", "_") - if not safe: - raise ValueError("run tag cannot be empty") - return safe - - -def resolve_artifact_root(raw: str | None, run_tag: str) -> Path: - """Resolve the user artifact root or the D-drive default.""" - if raw: - return Path(raw).expanduser().resolve() - return default_artifact_root(run_tag).resolve() - - -def _resolve_against_repo(path: str | Path, *, repo_root: Path) -> Path: - candidate = Path(path).expanduser() - if not candidate.is_absolute(): - candidate = repo_root / candidate - return candidate.resolve() - - -def _is_relative_to(child: Path, parent: Path) -> bool: - try: - child.relative_to(parent) - except ValueError: - return False - return True - - -def assert_safe_output_path(path: str | Path, *, repo_root: Path = ROOT) -> Path: - """Reject outputs that would overwrite frozen CRPTO artifacts.""" - resolved_repo = repo_root.resolve() - resolved = _resolve_against_repo(path, repo_root=resolved_repo) - protected_exact = {(resolved_repo / protected).resolve() for protected in PROTECTED_REPO_PATHS} - if resolved in protected_exact: - raise ValueError(f"Refusing to write protected CRPTO artifact: {resolved}") - - for protected_dir in PROTECTED_REPO_DIRS: - resolved_dir = (resolved_repo / protected_dir).resolve() - if resolved == resolved_dir or _is_relative_to(resolved, resolved_dir): - raise ValueError(f"Refusing to write inside protected CRPTO directory: {resolved}") - - if _is_relative_to(resolved, resolved_repo): - relative = resolved.relative_to(resolved_repo) - for pattern in PROTECTED_REPO_GLOBS: - if relative.match(pattern): - raise ValueError(f"Refusing to write protected CRPTO glob output: {resolved}") - return resolved - - -def assert_safe_output_paths(paths: Iterable[str | Path], *, repo_root: Path = ROOT) -> None: - """Apply frozen-artifact guardrails to many output paths.""" - for path in paths: - assert_safe_output_path(path, repo_root=repo_root) - - -def materialize_monotonic_policies() -> dict[str, dict[str, int]]: - """Return the monotonic policy maps used by the sandbox.""" - return {name: dict(policy) for name, policy in MONOTONIC_POLICIES.items()} - - -def materialize_feature_profiles() -> dict[str, dict[str, Any]]: - """Return the feature profile maps used by the sandbox.""" - return {name: dict(profile) for name, profile in FEATURE_PROFILES.items()} - - -def compute_decision_regret( - oracle_realized_return: float, - policy_realized_return: float, -) -> float: - """Compute decision regret under the same budget and ex-post cap.""" - return float(oracle_realized_return) - float(policy_realized_return) - - -def compute_auditability_score(metrics: Mapping[str, Any]) -> dict[str, Any]: - """Score the CRPTO auditability checks on a 0-100 scale.""" - coverage90 = float(metrics.get("coverage90", metrics.get("coverage_90", 0.0))) - coverage95 = float(metrics.get("coverage95", metrics.get("coverage_95", 0.0))) - min_group_coverage = float( - metrics.get("min_group_coverage", metrics.get("min_group_coverage_90", 0.0)) - ) - target90 = float(metrics.get("target_coverage90", metrics.get("target_coverage_90", 0.90))) - target95 = float(metrics.get("target_coverage95", metrics.get("target_coverage_95", 0.95))) - min_group_target = float(metrics.get("min_group_coverage_target", 0.88)) - alpha = float(metrics.get("alpha", metrics.get("alpha_exact", 0.01))) - weighted_v = float( - metrics.get( - "weighted_miscoverage_V", - metrics.get("alpha01_weighted_miscoverage_V", float("inf")), - ) - ) - violation = float(metrics.get("violation", metrics.get("alpha01_violation", 0.0))) - checks = { - "coverage90_pass": coverage90 >= target90, - "coverage95_pass": coverage95 >= target95, - "min_group_coverage_pass": min_group_coverage >= min_group_target, - "no_critical_alerts": int(metrics.get("critical_alerts", 0)) == 0, - "alpha01_exact_pass": bool(metrics.get("alpha01_exact_pass", False)), - "violation_zero": abs(violation) <= 1e-12, - "v_within_sqrt_alpha": weighted_v <= math.sqrt(alpha), - "monotonic_audit_pass": bool(metrics.get("monotonic_audit_pass", False)), - "reproducible_resume_manifest": bool(metrics.get("reproducible_resume_manifest", False)), - } - score = sum(weight for key, weight in AUDITABILITY_WEIGHTS.items() if checks[key]) - return { - "score": int(score), - "max_score": int(sum(AUDITABILITY_WEIGHTS.values())), - "checks": checks, - "weights": dict(AUDITABILITY_WEIGHTS), - } - - -def load_resume_manifest(path: str | Path) -> dict[str, Any]: - """Load a sandbox resume manifest if it exists.""" - target = Path(path) - if not target.exists(): - return {} - payload = json.loads(target.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"Resume manifest must be a JSON object: {target}") - return payload - - -def _format_monotone_constraints(policy: Mapping[str, int]) -> str: - return ",".join(f"{feature}:{int(direction)}" for feature, direction in policy.items()) - - -def _command_log_files(artifact_root: Path, phase: str, name: str) -> tuple[Path, Path]: - safe_name = sanitize_tag(name) - log_root = artifact_root / "logs" / phase - return log_root / f"{safe_name}.out.log", log_root / f"{safe_name}.err.log" - - -def _command_env(base_env: Mapping[str, str], *, phase_threads: int) -> dict[str, str]: - env = dict(base_env) - threads = str(max(1, int(phase_threads))) - env.update( - { - "OMP_NUM_THREADS": threads, - "MKL_NUM_THREADS": threads, - "OPENBLAS_NUM_THREADS": threads, - "NUMEXPR_NUM_THREADS": threads, - "VECLIB_MAXIMUM_THREADS": threads, - "CATBOOST_THREAD_COUNT": threads, - } - ) - return env - - -def _ordered_unique(values: Iterable[str]) -> list[str]: - seen: set[str] = set() - ordered: list[str] = [] - for raw in values: - value = str(raw) - if value in seen: - continue - seen.add(value) - ordered.append(value) - return ordered - - -def _load_feature_config() -> dict[str, Any]: - return load_feature_config( - yaml_path=ROOT / "data" / "processed" / "feature_config.yml", - parquet_path=ROOT / "data" / "processed" / "feature_config.parquet", - pickle_path=ROOT / "data" / "processed" / "feature_config.pkl", - prefer="auto", - ) - - -def _features_from_profile(base_config: Mapping[str, Any], profile_name: str) -> list[str]: - profile = FEATURE_PROFILES[profile_name] - features: list[str] = [] - for group_name in profile.get("groups", []): - values = base_config.get(str(group_name), []) - if isinstance(values, Sequence) and not isinstance(values, str): - features.extend(str(value) for value in values) - return _ordered_unique(features) - - -def _feature_profile_config(base_config: Mapping[str, Any], profile_name: str) -> dict[str, Any]: - profile = FEATURE_PROFILES[profile_name] - materialized = dict(base_config) - catboost_features = _features_from_profile(base_config, profile_name) - materialized["CATBOOST_FEATURES"] = catboost_features - - categorical = base_config.get("CATEGORICAL_FEATURES", []) - if isinstance(categorical, Sequence) and not isinstance(categorical, str): - materialized["CATEGORICAL_FEATURES"] = [ - str(feature) for feature in categorical if str(feature) in set(catboost_features) - ] - - logreg_features = list(base_config.get("LOGREG_FEATURES", []) or []) - woe_features = list(base_config.get("WOE_FEATURES", []) or []) - materialized["LOGREG_FEATURES"] = _ordered_unique( - str(feature) for feature in [*logreg_features, *woe_features] - ) - materialized["SANDBOX_FEATURE_PROFILE"] = { - "name": profile_name, - "description": str(profile.get("description", "")), - "groups": [str(group) for group in profile.get("groups", [])], - "stable_core_enabled": bool(profile.get("stable_core_enabled", False)), - "catboost_feature_count": len(catboost_features), - } - return materialized - - -def _write_feature_profile_snapshot(*, artifact_root: Path, profile_name: str) -> Path: - base_config = _load_feature_config() - profile_config = _feature_profile_config(base_config, profile_name) - feature_root = artifact_root / "configs" / "feature_profiles" - feature_root.mkdir(parents=True, exist_ok=True) - pkl_path = feature_root / f"{profile_name}.pkl" - yaml_path = feature_root / f"{profile_name}.yaml" - with pkl_path.open("wb") as fh: - pickle.dump(profile_config, fh) - _write_yaml( - yaml_path, - { - "profile": profile_config["SANDBOX_FEATURE_PROFILE"], - "catboost_features": profile_config["CATBOOST_FEATURES"], - "categorical_features": profile_config.get("CATEGORICAL_FEATURES", []), - "logreg_features": profile_config.get("LOGREG_FEATURES", []), - }, - ) - assert_safe_output_path(pkl_path) - assert_safe_output_path(yaml_path) - return pkl_path - - -def _lane_id(feature_profile: str, monotonic_policy: str) -> str: - return f"{feature_profile}__{monotonic_policy}" - - -def _monotonic_policy_for_feature_profile( - *, - policy_name: str, - feature_profile_name: str, - feature_profile_path: Path, -) -> dict[str, int]: - with feature_profile_path.open("rb") as fh: - feature_config = pickle.load(fh) - features = set(feature_config.get("CATBOOST_FEATURES", [])) - profile = FEATURE_PROFILES[feature_profile_name] - if bool(profile.get("stable_core_enabled", False)): - stable_core_cfg = _load_yaml(ROOT / "configs" / "crpto_pd_model.yaml").get( - "stable_core", {} - ) - excluded = stable_core_cfg.get("exclude_features", ["rev_utilization", "high_util_pct"]) - if isinstance(excluded, Sequence) and not isinstance(excluded, str): - features -= {str(feature) for feature in excluded} - return { - feature: direction - for feature, direction in MONOTONIC_POLICIES[policy_name].items() - if feature in features - } - - -def _float_range(start: float, stop: float, step: float) -> str: - values: list[str] = [] - current = float(start) - while current <= stop + step / 10: - values.append(f"{current:.3f}".rstrip("0").rstrip(".")) - current += step - return ",".join(values) - - -def _resource_snapshot(artifact_root: Path) -> dict[str, Any]: - payload: dict[str, Any] = { - "captured_at_utc": utc_now_iso(), - "logical_cpu_count": int(os.cpu_count() or 0), - } - try: - import psutil - - vm = psutil.virtual_memory() - payload.update( - { - "cpu_percent": float(psutil.cpu_percent(interval=0.1)), - "ram_total_gb": float(vm.total / 1024**3), - "ram_available_gb": float(vm.available / 1024**3), - } - ) - except Exception as exc: # pragma: no cover - platform probe only - payload["resource_probe_error"] = str(exc) - try: - usage = shutil.disk_usage(artifact_root.anchor or artifact_root) - payload.update( - { - "artifact_root": str(artifact_root), - "disk_total_gb": float(usage.total / 1024**3), - "disk_free_gb": float(usage.free / 1024**3), - } - ) - except Exception as exc: # pragma: no cover - platform probe only - payload["disk_probe_error"] = str(exc) - return payload - - -def _write_heartbeat( - *, - artifact_root: Path, - phase: str, - completed_units: int, - total_units: int, - current_best_metric: float | None, - last_checkpoint_path: Path | None, - state: str, -) -> Path: - payload = _resource_snapshot(artifact_root) - completed = int(completed_units) - total = int(total_units) - eta_seconds: float | None = None - if completed > 0 and total > completed: - eta_seconds = float((total - completed) / completed) - payload.update( - { - "schema_version": SCHEMA_VERSION, - "stage_name": STAGE_NAME, - "phase": phase, - "state": state, - "completed_units": completed, - "total_units": total, - "eta_units_ratio": eta_seconds, - "current_best_metric": current_best_metric, - "last_checkpoint_path": str(last_checkpoint_path or ""), - } - ) - heartbeat_path = artifact_root / "heartbeat.json" - last_error: OSError | None = None - for attempt in range(6): - try: - return atomic_write_json(heartbeat_path, payload) - except PermissionError as exc: - last_error = exc - time.sleep(0.25 * (attempt + 1)) - if last_error is not None: - fallback_path = artifact_root / f"heartbeat.{os.getpid()}.json" - return atomic_write_json(fallback_path, payload) - return heartbeat_path - - -def _load_yaml(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - return payload if isinstance(payload, dict) else {} - - -def _write_yaml(path: Path, payload: Mapping[str, Any]) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(yaml.safe_dump(dict(payload), sort_keys=False), encoding="utf-8") - return path - - -OPTUNA_TRIAL_PARAM_KEYS = { - "bootstrap_type", - "grow_policy", - "learning_rate", - "l2_leaf_reg", - "min_data_in_leaf", - "random_strength", - "border_count", - "leaf_estimation_iterations", - "rsm", - "depth", - "max_leaves", - "subsample", - "bagging_temperature", -} - - -def _load_champion_pd_params() -> dict[str, Any]: - config = _load_yaml(CHAMPION_PD_CONFIG_PATH) - return dict(_nested_get(config, "model", "params", default={}) or {}) - - -def _sanitize_enqueue_trial_params( - params: Mapping[str, Any], - *, - include_iterations: bool, -) -> dict[str, Any]: - allowed = set(OPTUNA_TRIAL_PARAM_KEYS) - if include_iterations: - allowed.add("iterations") - trial: dict[str, Any] = {} - for key, value in dict(params or {}).items(): - key_str = str(key) - if key_str in allowed: - trial[key_str] = value - if str(trial.get("grow_policy", "")).strip() != "Lossguide": - trial.pop("max_leaves", None) - if str(trial.get("grow_policy", "")).strip() == "Lossguide": - trial.pop("depth", None) - else: - trial["grow_policy"] = "SymmetricTree" - if str(trial.get("bootstrap_type", "")).strip() == "Bayesian": - trial.pop("subsample", None) - else: - trial.pop("bagging_temperature", None) - return {key: value for key, value in trial.items() if value is not None} - - -def _same_trial_params(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: - if set(left) != set(right): - return False - for key, left_value in left.items(): - right_value = right.get(key) - if right_value is None: - return False - try: - if abs(float(left_value) - float(right_value)) > 1e-12: - return False - except (TypeError, ValueError): - if str(left_value) != str(right_value): - return False - return True - - -def _append_warm_start_candidate( - rows: list[dict[str, Any]], - *, - source: str, - params: Mapping[str, Any] | None, - include_iterations: bool, -) -> None: - if not isinstance(params, Mapping): - return - sanitized = _sanitize_enqueue_trial_params(params, include_iterations=include_iterations) - if not sanitized: - return - if any(_same_trial_params(sanitized, row["params"]) for row in rows): - return - rows.append({"source": source, "params": sanitized}) - - -def _pd_warm_start_candidates( - *, - artifact_root: Path, - phase: str, - lane_id: str, -) -> list[dict[str, Any]]: - include_iterations = phase == "pd-refine" - rows: list[dict[str, Any]] = [] - _append_warm_start_candidate( - rows, - source="frozen_champion_pd_config", - params=_load_champion_pd_params(), - include_iterations=include_iterations, - ) - previous_phase = _previous_pd_phase(phase) - if previous_phase is None: - return rows - - selection = _load_pd_selection(artifact_root, previous_phase) - selected = selection.get("selected", []) - if not isinstance(selected, list): - return rows - for row in selected: - if not isinstance(row, Mapping): - continue - if row.get("lane_id") == lane_id: - _append_warm_start_candidate( - rows, - source=f"{previous_phase}:same_lane_best", - params=row.get("best_params"), - include_iterations=include_iterations, - ) - break - for index, row in enumerate(selected[:5], start=1): - if not isinstance(row, Mapping): - continue - best_params = row.get("best_params") - _append_warm_start_candidate( - rows, - source=f"{previous_phase}:top_{index}:{row.get('lane_id', 'unknown')}", - params=cast(Mapping[str, Any], best_params) - if isinstance(best_params, Mapping) - else None, - include_iterations=include_iterations, - ) - return rows - - -def _apply_pd_feature_profile_config( - *, - config: dict[str, Any], - feature_profile_path: Path, - profile: Mapping[str, Any], -) -> None: - config["feature_source"] = dict(config.get("feature_source", {}) or {}) - config["feature_source"]["feature_config_path"] = str(feature_profile_path) - config["stable_core"] = dict(config.get("stable_core", {}) or {}) - config["stable_core"]["enabled"] = bool(profile.get("stable_core_enabled", False)) - - -def _apply_pd_model_params( - *, - config: dict[str, Any], - policy: Mapping[str, int], - cpu_threads: int, - base_params_override: Mapping[str, Any] | None, -) -> None: - config["model"] = dict(config.get("model", {}) or {}) - config["model"]["params"] = dict(config["model"].get("params", {}) or {}) - if base_params_override: - blocked_keys = { - "task_type", - "devices", - "thread_count", - "allow_writing_files", - "monotone_constraints", - } - config["model"]["params"].update( - {key: value for key, value in base_params_override.items() if key not in blocked_keys} - ) - config["model"]["params"].update( - { - "task_type": "CPU", - "devices": "", - "thread_count": int(cpu_threads), - "allow_writing_files": False, - "monotone_constraints": _format_monotone_constraints(policy), - } - ) - - -def _apply_venn_abers_calibration(config: dict[str, Any]) -> None: - config["calibration"] = dict(config.get("calibration", {}) or {}) - config["calibration"]["method"] = "venn_abers" - config["calibration"]["candidates"] = ["venn_abers"] - - -def _apply_pd_hpo_config( - *, - config: dict[str, Any], - artifact_root: Path, - phase_root: Path, - run_tag: str, - phase: str, - lane: str, - n_trials: int, -) -> list[dict[str, Any]]: - config["hpo"] = dict(config.get("hpo", {}) or {}) - config["hpo"].update( - { - "enabled": True, - "n_trials": int(n_trials), - "sampler": "tpe", - "pruner": "median", - "n_startup_trials": min(80, max(10, int(n_trials // 10))), - "multivariate_tpe": True, - "group_tpe": True, - "constant_liar": True, - "warn_independent_sampling": False, - "search_space_version": "cb_space_v3_monotone_symmetric", - "study_storage": f"sqlite:///{(phase_root / 'optuna_pd_catboost.db').as_posix()}", - "study_name": f"pd_{run_tag}_{lane}_{phase}", - "load_if_exists": True, - "storage_heartbeat_interval": 60, - "storage_grace_period": 180, - "sqlite_timeout_seconds": 240, - "retry_failed_trials": 2, - "n_jobs": 1, - "constraints_policy": { - "max_brier_delta": 0.0025, - "max_ece_delta": 0.0025, - "min_auc_delta": -0.0010, - }, - } - ) - warm_start = _pd_warm_start_candidates( - artifact_root=artifact_root, - phase=phase, - lane_id=lane, - ) - if warm_start: - config["hpo"]["enqueue_trials"] = [dict(row["params"]) for row in warm_start] - if phase == "pd-refine": - config["hpo"]["search_space_mode"] = "local_refine" - config["hpo"]["local_refine"] = { - "enqueue_base_trial": True, - "iterations": {"low": 2500, "high": 5200, "step": 100}, - "learning_rate": {"low": 0.02, "high": 0.10, "log": True}, - "l2_leaf_reg": {"low": 30.0, "high": 200.0, "log": True}, - "min_data_in_leaf": {"low": 80, "high": 220, "step": 5}, - "rsm": {"low": 0.50, "high": 0.75}, - "bootstrap_type": ["MVS", "Bernoulli"], - "grow_policy": ["SymmetricTree"], - } - return warm_start - - -def _pd_validation_policy(phase: str) -> tuple[dict[str, Any], bool]: - if phase == "pd-smoke": - return {"enabled": True, "top_k_trials": 1, "seeds": [42]}, False - if phase == "pd-broad": - return {"enabled": True, "top_k_trials": 10, "seeds": [42, 52, 62]}, True - return { - "enabled": True, - "top_k_trials": 30, - "seeds": [42, 52, 62, 72, 82], - }, True - - -def _apply_pd_validation_config(*, config: dict[str, Any], phase: str) -> None: - seed_replay, walk_forward_enabled = _pd_validation_policy(phase) - config["validation"] = dict(config.get("validation", {}) or {}) - config["validation"]["seed_replay"] = { - **seed_replay, - "prioritize_gate_pass": True, - } - config["validation"]["walk_forward"] = dict(config["validation"].get("walk_forward", {}) or {}) - config["validation"]["walk_forward"]["enabled"] = walk_forward_enabled - - -def _pd_output_paths(phase_root: Path) -> dict[str, Any]: - return { - "model_path": str(phase_root / "models" / "pd_model.cbm"), - "default_model_path": str(phase_root / "models" / "pd_default.cbm"), - "tuned_model_path": str(phase_root / "models" / "pd_tuned.cbm"), - "canonical_model_path": str(phase_root / "models" / "pd_shadow_canonical.cbm"), - "conformal_path": str(phase_root / "models" / "pd_calibrator.pkl"), - "canonical_calibrator_path": str(phase_root / "models" / "pd_shadow_calibrator.pkl"), - "contract_path": str(phase_root / "models" / "pd_model_contract.json"), - "status_path": str(phase_root / "models" / "pd_training_status.json"), - "checkpoint_dir": str(phase_root / "models" / "pd_training_checkpoints"), - "logreg_model_path": str(phase_root / "models" / "pd_logreg_baseline.pkl"), - "threshold_semantics_path": str(phase_root / "models" / "threshold_semantics.json"), - "brier_decomposition_path": str(phase_root / "data" / "brier_decomposition_test.parquet"), - "murphy_diagram_path": str(phase_root / "data" / "murphy_diagram_test.parquet"), - "test_predictions_path": str(phase_root / "data" / "test_predictions.parquet"), - "training_record_path": str(phase_root / "models" / "pd_training_record.pkl"), - "seed_replay_status_path": str(phase_root / "models" / "pd_hpo_seed_replay_status.json"), - "shap_dir": str(phase_root / "reports" / "shap"), - "write_legacy_model_copy": False, - } - - -def _apply_pd_decision_threshold_config(*, config: dict[str, Any], phase_root: Path) -> None: - config["decision_threshold"] = dict(config.get("decision_threshold", {}) or {}) - config["decision_threshold"]["enabled"] = False - config["decision_threshold"]["fairness_policy_path"] = "" - config["decision_threshold"]["output_path"] = str( - phase_root / "models" / "decision_threshold.json" - ) - config["decision_threshold"]["output_path_v2"] = str( - phase_root / "models" / "decision_threshold_v2.json" - ) - - -def _apply_pd_sandbox_metadata( - *, - config: dict[str, Any], - run_tag: str, - phase: str, - feature_profile_name: str, - policy_name: str, - lane: str, - policy: Mapping[str, int], - base_params_override: Mapping[str, Any] | None, - warm_start: list[dict[str, Any]], -) -> None: - config["sandbox_search"] = { - "run_tag": run_tag, - "phase": phase, - "feature_profile": feature_profile_name, - "monotonic_policy": policy_name, - "lane_id": lane, - "effective_monotone_constraints": dict(policy), - "base_params_from_previous_phase": bool(base_params_override), - "hpo_warm_start": { - "enabled": bool(warm_start), - "n_prior_trials": len(warm_start), - "sources": [str(row["source"]) for row in warm_start], - }, - "skip_auxiliary_models": True, - "skip_diagnostic_exports": True, - "skip_shap_export": True, - } - - -def write_pd_config_snapshot( - *, - artifact_root: Path, - run_tag: str, - feature_profile_name: str, - policy_name: str, - phase: str, - n_trials: int, - cpu_threads: int, - base_params_override: Mapping[str, Any] | None = None, -) -> Path: - """Write an external PD config snapshot for one feature/policy lane and phase.""" - config = _load_yaml(CHAMPION_PD_CONFIG_PATH) - feature_profile_path = _write_feature_profile_snapshot( - artifact_root=artifact_root, - profile_name=feature_profile_name, - ) - policy = _monotonic_policy_for_feature_profile( - policy_name=policy_name, - feature_profile_name=feature_profile_name, - feature_profile_path=feature_profile_path, - ) - lane = _lane_id(feature_profile_name, policy_name) - phase_root = artifact_root / "pd" / feature_profile_name / policy_name / phase - profile = FEATURE_PROFILES[feature_profile_name] - - _apply_pd_feature_profile_config( - config=config, - feature_profile_path=feature_profile_path, - profile=profile, - ) - _apply_pd_model_params( - config=config, - policy=policy, - cpu_threads=cpu_threads, - base_params_override=base_params_override, - ) - _apply_venn_abers_calibration(config) - warm_start = _apply_pd_hpo_config( - config=config, - artifact_root=artifact_root, - phase_root=phase_root, - run_tag=run_tag, - phase=phase, - lane=lane, - n_trials=n_trials, - ) - _apply_pd_validation_config(config=config, phase=phase) - config["output"] = _pd_output_paths(phase_root) - _apply_pd_decision_threshold_config(config=config, phase_root=phase_root) - _apply_pd_sandbox_metadata( - config=config, - run_tag=run_tag, - phase=phase, - feature_profile_name=feature_profile_name, - policy_name=policy_name, - lane=lane, - policy=policy, - base_params_override=base_params_override, - warm_start=warm_start, - ) - target = artifact_root / "configs" / f"pd_{lane}_{phase}.yaml" - assert_safe_output_paths( - value for value in config["output"].values() if isinstance(value, (str, Path)) - ) - assert_safe_output_path(config["decision_threshold"]["output_path"]) - assert_safe_output_path(config["decision_threshold"]["output_path_v2"]) - return _write_yaml(target, config) - - -def write_pd_incumbent_config_snapshot( - *, - artifact_root: Path, - run_tag: str, - phase: str, - cpu_threads: int, -) -> Path: - """Write a sandbox-local replay config for the frozen PD champion.""" - config = _load_yaml(CHAMPION_PD_CONFIG_PATH) - phase_root = artifact_root / "pd_baselines" / "champion" / phase - config["model"] = dict(config.get("model", {}) or {}) - config["model"]["params"] = dict(config["model"].get("params", {}) or {}) - config["model"]["params"].update( - { - "task_type": "CPU", - "devices": "", - "thread_count": int(cpu_threads), - "allow_writing_files": False, - } - ) - config["calibration"] = dict(config.get("calibration", {}) or {}) - config["calibration"]["method"] = "venn_abers" - config["calibration"]["candidates"] = ["venn_abers"] - config["hpo"] = dict(config.get("hpo", {}) or {}) - config["hpo"].update({"enabled": False, "n_trials": 0}) - config["validation"] = dict(config.get("validation", {}) or {}) - config["validation"]["seed_replay"] = {"enabled": False, "seeds": []} - config["validation"]["walk_forward"] = dict(config["validation"].get("walk_forward", {}) or {}) - config["validation"]["walk_forward"]["enabled"] = phase != "pd-smoke" - config["output"] = { - "model_path": str(phase_root / "models" / "pd_model.cbm"), - "default_model_path": str(phase_root / "models" / "pd_default.cbm"), - "tuned_model_path": str(phase_root / "models" / "pd_tuned.cbm"), - "canonical_model_path": str(phase_root / "models" / "pd_shadow_canonical.cbm"), - "conformal_path": str(phase_root / "models" / "pd_calibrator.pkl"), - "canonical_calibrator_path": str(phase_root / "models" / "pd_shadow_calibrator.pkl"), - "contract_path": str(phase_root / "models" / "pd_model_contract.json"), - "status_path": str(phase_root / "models" / "pd_training_status.json"), - "checkpoint_dir": str(phase_root / "models" / "pd_training_checkpoints"), - "logreg_model_path": str(phase_root / "models" / "pd_logreg_baseline.pkl"), - "threshold_semantics_path": str(phase_root / "models" / "threshold_semantics.json"), - "brier_decomposition_path": str(phase_root / "data" / "brier_decomposition_test.parquet"), - "murphy_diagram_path": str(phase_root / "data" / "murphy_diagram_test.parquet"), - "test_predictions_path": str(phase_root / "data" / "test_predictions.parquet"), - "training_record_path": str(phase_root / "models" / "pd_training_record.pkl"), - "seed_replay_status_path": str(phase_root / "models" / "pd_hpo_seed_replay_status.json"), - "shap_dir": str(phase_root / "reports" / "shap"), - "write_legacy_model_copy": False, - } - config["decision_threshold"] = dict(config.get("decision_threshold", {}) or {}) - config["decision_threshold"]["enabled"] = False - config["decision_threshold"]["fairness_policy_path"] = "" - config["decision_threshold"]["output_path"] = str( - phase_root / "models" / "decision_threshold.json" - ) - config["decision_threshold"]["output_path_v2"] = str( - phase_root / "models" / "decision_threshold_v2.json" - ) - config["sandbox_search"] = { - "run_tag": run_tag, - "phase": phase, - "candidate_role": "frozen_champion_replay", - "source_config": str(CHAMPION_PD_CONFIG_PATH), - "skip_auxiliary_models": True, - "skip_diagnostic_exports": True, - "skip_shap_export": True, - } - assert_safe_output_paths( - value for value in config["output"].values() if isinstance(value, (str, Path)) - ) - assert_safe_output_path(config["decision_threshold"]["output_path"]) - assert_safe_output_path(config["decision_threshold"]["output_path_v2"]) - target = artifact_root / "configs" / f"pd_incumbent_champion_{phase}.yaml" - return _write_yaml(target, config) - - -def _pd_trials_for_phase(phase: str) -> int: - if phase == "pd-smoke": - return PD_SMOKE_TRIALS - if phase == "pd-broad": - return PD_BROAD_TRIALS - if phase == "pd-refine": - return PD_REFINE_TRIALS - raise ValueError(f"Unsupported PD phase: {phase}") - - -def _pd_optuna_complete_trials( - *, - phase_root: Path, - run_tag: str, - lane_id: str, - phase: str, -) -> int: - """Return COMPLETE Optuna trials for the current PD study version.""" - db_path = phase_root / "optuna_pd_catboost.db" - if not db_path.exists(): - return 0 - study_name = f"pd_{run_tag}_{lane_id}_{phase}__cb_space_v3_monotone_symmetric" - try: - con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5) - row = con.execute( - "select study_id from studies where study_name = ? order by study_id desc limit 1", - (study_name,), - ).fetchone() - if row is None: - con.close() - return 0 - count = con.execute( - "select count(*) from trials where study_id = ? and state = 'COMPLETE'", - (int(row[0]),), - ).fetchone()[0] - con.close() - return int(count) - except sqlite3.Error: - return 0 - - -def _pd_remaining_trials( - *, - phase_root: Path, - run_tag: str, - lane_id: str, - phase: str, - target_trials: int, -) -> int: - complete = _pd_optuna_complete_trials( - phase_root=phase_root, - run_tag=run_tag, - lane_id=lane_id, - phase=phase, - ) - return max(0, int(target_trials) - complete) - - -def _pd_selection_path(artifact_root: Path, phase: str) -> Path: - return artifact_root / "pd" / "_selection" / f"{phase}_selection.json" - - -def _pd_selection_limit_for_phase(phase: str) -> int | None: - if phase == "pd-broad": - return PD_BROAD_TOP_K_LANES - if phase == "pd-refine": - return PD_REFINE_TOP_K_LANES - return None - - -def _previous_pd_phase(phase: str) -> str | None: - if phase == "pd-broad": - return "pd-smoke" - if phase == "pd-refine": - return "pd-broad" - return None - - -def _load_pd_selection(artifact_root: Path, phase: str) -> dict[str, Any]: - path = _pd_selection_path(artifact_root, phase) - if not path.exists(): - return {} - payload = json.loads(path.read_text(encoding="utf-8")) - return payload if isinstance(payload, dict) else {} - - -def _selected_pd_lanes(artifact_root: Path, phase: str) -> list[tuple[str, str]]: - all_lanes = [ - (feature_profile, monotonic_policy) - for feature_profile in FEATURE_PROFILES - for monotonic_policy in MONOTONIC_POLICIES - ] - previous_phase = _previous_pd_phase(phase) - if previous_phase is None: - return all_lanes - selection = _load_pd_selection(artifact_root, previous_phase) - rows = selection.get("selected", []) - if not isinstance(rows, list) or not rows: - return all_lanes - limit = _pd_selection_limit_for_phase(phase) - selected_ids = { - str(row.get("lane_id")) - for row in rows[:limit] - if isinstance(row, Mapping) and row.get("lane_id") - } - filtered = [ - (feature_profile, monotonic_policy) - for feature_profile, monotonic_policy in all_lanes - if _lane_id(feature_profile, monotonic_policy) in selected_ids - ] - return filtered or all_lanes - - -def _load_pickle_record(path: Path) -> dict[str, Any]: - with path.open("rb") as fh: - payload = pickle.load(fh) - return payload if isinstance(payload, dict) else {} - - -def _nested_get(payload: Mapping[str, Any], *keys: str, default: Any = None) -> Any: - current: Any = payload - for key in keys: - if not isinstance(current, Mapping): - return default - current = current.get(key, default) - return current - - -def _float_metric(payload: Mapping[str, Any], *paths: tuple[str, ...], default: float) -> float: - for path in paths: - value = _nested_get(payload, *path, default=None) - if value is None: - continue - try: - return float(value) - except (TypeError, ValueError): - continue - return float(default) - - -def _rank_pd_candidate_rows(rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: - ranked = sorted( - (dict(row) for row in rows), - key=lambda row: ( - -float(row.get("auc_roc", float("-inf"))), - float(row.get("brier_score", float("inf"))), - float(row.get("ece", float("inf"))), - -float(row.get("hpo_best_validation_auc", float("-inf"))), - -float(row.get("walk_forward_auc_mean", float("-inf"))), - ), - ) - for rank, row in enumerate(ranked, start=1): - row["selection_rank"] = rank - return ranked - - -def _pd_candidate_row_from_record(record_path: Path, artifact_root: Path) -> dict[str, Any] | None: - try: - relative = record_path.relative_to(artifact_root) - except ValueError: - return None - parts = relative.parts - if len(parts) < 6 or parts[0] != "pd": - return None - feature_profile, monotonic_policy, phase = parts[1], parts[2], parts[3] - if phase not in PD_PHASES: - return None - record = _load_pickle_record(record_path) - lane = _lane_id(feature_profile, monotonic_policy) - phase_root = artifact_root / "pd" / feature_profile / monotonic_policy / phase - return { - "lane_id": lane, - "feature_profile": feature_profile, - "monotonic_policy": monotonic_policy, - "phase": phase, - "record_path": str(record_path), - "model_path": str(phase_root / "models" / "pd_shadow_canonical.cbm"), - "calibrator_path": str(phase_root / "models" / "pd_shadow_calibrator.pkl"), - "contract_path": str(phase_root / "models" / "pd_model_contract.json"), - "auc_roc": _float_metric( - record, - ("final_test_metrics", "auc_roc"), - ("test_metrics", "auc_roc"), - ("metrics", "auc_roc"), - default=float("-inf"), - ), - "brier_score": _float_metric( - record, - ("final_test_metrics", "brier_score"), - ("test_metrics", "brier_score"), - ("metrics", "brier_score"), - default=float("inf"), - ), - "ece": _float_metric( - record, - ("final_test_metrics", "ece"), - ("test_metrics", "ece"), - ("metrics", "ece"), - default=float("inf"), - ), - "hpo_best_validation_auc": _float_metric( - record, - ("hpo_best_validation_auc",), - ("hpo", "best_validation_auc"), - ("best_validation_auc",), - default=float("-inf"), - ), - "walk_forward_auc_mean": _float_metric( - record, - ("walk_forward", "auc_roc_mean"), - ("walk_forward_metrics", "auc_roc_mean"), - default=float("-inf"), - ), - "best_params": record.get("optuna_best_params") - or record.get("best_params") - or _nested_get(record, "hpo", "best_params", default={}), - } - - -def _pd_incumbent_row_from_record( - record_path: Path, - artifact_root: Path, - phase: str, -) -> dict[str, Any] | None: - if not record_path.exists(): - return None - record = _load_pickle_record(record_path) - phase_root = artifact_root / "pd_baselines" / "champion" / phase - return { - "candidate_role": "frozen_champion_replay", - "lane_id": "incumbent__frozen_champion", - "feature_profile": "incumbent", - "monotonic_policy": "canonical_4", - "phase": phase, - "record_path": str(record_path), - "model_path": str(phase_root / "models" / "pd_shadow_canonical.cbm"), - "calibrator_path": str(phase_root / "models" / "pd_shadow_calibrator.pkl"), - "contract_path": str(phase_root / "models" / "pd_model_contract.json"), - "auc_roc": _float_metric( - record, - ("final_test_metrics", "auc_roc"), - ("test_metrics", "auc_roc"), - ("metrics", "auc_roc"), - default=float("-inf"), - ), - "brier_score": _float_metric( - record, - ("final_test_metrics", "brier_score"), - ("test_metrics", "brier_score"), - ("metrics", "brier_score"), - default=float("inf"), - ), - "ece": _float_metric( - record, - ("final_test_metrics", "ece"), - ("test_metrics", "ece"), - ("metrics", "ece"), - default=float("inf"), - ), - "hpo_best_validation_auc": _float_metric( - record, - ("hpo_best_validation_auc",), - ("hpo", "best_validation_auc"), - ("best_validation_auc",), - default=float("-inf"), - ), - "walk_forward_auc_mean": _float_metric( - record, - ("walk_forward", "auc_roc_mean"), - ("walk_forward_metrics", "auc_roc_mean"), - default=float("-inf"), - ), - "best_params": record.get("optuna_best_params") - or record.get("best_params") - or _nested_get(record, "hpo", "best_params", default={}), - } - - -def _select_pd_phase_winners(artifact_root: Path, phase: str) -> Path | None: - record_paths = sorted(artifact_root.glob(f"pd/*/*/{phase}/models/pd_training_record.pkl")) - rows = [ - row - for record_path in record_paths - if (row := _pd_candidate_row_from_record(record_path, artifact_root)) is not None - ] - if not rows: - return None - ranked = _rank_pd_candidate_rows(rows) - incumbent_record = ( - artifact_root / "pd_baselines" / "champion" / phase / "models" / "pd_training_record.pkl" - ) - incumbent_baseline = _pd_incumbent_row_from_record(incumbent_record, artifact_root, phase) - selection_path = _pd_selection_path(artifact_root, phase) - selected_payload = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": utc_now_iso(), - "phase": phase, - "rank_criteria": ["auc_roc desc", "brier_score asc", "ece asc"], - "incumbent_baseline": incumbent_baseline, - "selected": ranked, - } - atomic_write_json(selection_path, selected_payload) - - best = ranked[0] - best_models = artifact_root / "pd" / "best" / phase / "models" - best_models.mkdir(parents=True, exist_ok=True) - for source_key, target_name in { - "model_path": "pd_shadow_canonical.cbm", - "calibrator_path": "pd_shadow_calibrator.pkl", - "contract_path": "pd_model_contract.json", - "record_path": "pd_training_record.pkl", - }.items(): - source = Path(str(best.get(source_key, ""))) - if source.exists(): - shutil.copy2(source, best_models / target_name) - if phase == "pd-refine": - final_best = artifact_root / "pd" / "best" / "models" - final_best.mkdir(parents=True, exist_ok=True) - for source in best_models.iterdir(): - if source.is_file(): - shutil.copy2(source, final_best / source.name) - return selection_path - - -def _previous_best_params_for_lane( - *, - artifact_root: Path, - phase: str, - lane_id: str, -) -> Mapping[str, Any] | None: - previous_phase = _previous_pd_phase(phase) - if previous_phase is None: - return None - selection = _load_pd_selection(artifact_root, previous_phase) - rows = selection.get("selected", []) - if not isinstance(rows, list): - return None - for row in rows: - if not isinstance(row, Mapping) or row.get("lane_id") != lane_id: - continue - params = row.get("best_params") - return params if isinstance(params, Mapping) else None - return None - - -def _resolve_pd_candidate_for_conformal(artifact_root: Path, artifact_name: str) -> Path: - best_path = artifact_root / "pd" / "best" / "models" / artifact_name - if best_path.exists(): - return best_path - phase_best = artifact_root / "pd" / "best" / "pd-refine" / "models" / artifact_name - if phase_best.exists(): - return phase_best - refined = sorted(artifact_root.glob(f"pd/*/*/pd-refine/models/{artifact_name}")) - if refined: - return refined[0] - canonical = { - "pd_shadow_canonical.cbm": ROOT / "models" / "pd_canonical.cbm", - "pd_shadow_calibrator.pkl": ROOT / "models" / "pd_canonical_calibrator.pkl", - } - return canonical[artifact_name] - - -def _sandbox_base_env(artifact_root: Path, safe_tag: str) -> dict[str, str]: - return { - "CRPTO_RUN_TAG": safe_tag, - "PIPELINE_RUN_TAG": safe_tag, - "RUN_TAG": safe_tag, - "CRPTO_OFFICIAL_RUN_TAG": safe_tag, - "GPU_REPLAY_ARTIFACT_ROOT": str(artifact_root), - "CRPTO_SANDBOX_ARTIFACT_ROOT": str(artifact_root), - } - - -def _phase_resources( - phase: str, - *, - max_workers: int, - cpu_threads: int, -) -> tuple[int, int]: - workers = int(max_workers) if int(max_workers) > 0 else _default_workers_for_phase(phase) - threads = int(cpu_threads) if int(cpu_threads) > 0 else _default_threads_for_phase(phase) - return workers, threads - - -def _pd_incumbent_command( - *, - artifact_root: Path, - safe_tag: str, - phase: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], -) -> PhaseCommand: - config_path = write_pd_incumbent_config_snapshot( - artifact_root=artifact_root, - run_tag=safe_tag, - phase=phase, - cpu_threads=phase_threads, - ) - output_root = artifact_root / "pd_baselines" / "champion" / phase - outputs = [ - output_root / "models" / "pd_model.cbm", - output_root / "models" / "pd_training_status.json", - output_root / "models" / "pd_training_record.pkl", - ] - command_name = f"{phase}_incumbent__frozen_champion" - stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) - return PhaseCommand( - name=command_name, - phase=phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(config_path), - "--hpo_enabled", - "false", - "--hpo_n_trials", - "0", - "--walk_forward_enabled", - "true" if phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "false", - ], - outputs=[str(path) for path in outputs], - checkpoint=str(output_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile="incumbent", - monotonic_policy="canonical_4", - lane_id="incumbent__frozen_champion", - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - - -def _pd_lane_command( - *, - artifact_root: Path, - safe_tag: str, - phase: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], - feature_profile_name: str, - policy_name: str, - n_trials: int, -) -> PhaseCommand: - lane = _lane_id(feature_profile_name, policy_name) - base_params = _previous_best_params_for_lane( - artifact_root=artifact_root, - phase=phase, - lane_id=lane, - ) - config_path = write_pd_config_snapshot( - artifact_root=artifact_root, - run_tag=safe_tag, - feature_profile_name=feature_profile_name, - policy_name=policy_name, - phase=phase, - n_trials=n_trials, - cpu_threads=phase_threads, - base_params_override=base_params, - ) - output_root = artifact_root / "pd" / feature_profile_name / policy_name / phase - remaining_trials = _pd_remaining_trials( - phase_root=output_root, - run_tag=safe_tag, - lane_id=lane, - phase=phase, - target_trials=n_trials, - ) - outputs = [ - output_root / "models" / "pd_model.cbm", - output_root / "models" / "pd_training_status.json", - output_root / "models" / "pd_hpo_seed_replay_status.json", - ] - command_name = f"{phase}_{lane}" - stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) - return PhaseCommand( - name=command_name, - phase=phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(config_path), - "--hpo_enabled", - "true", - "--hpo_n_trials", - str(remaining_trials), - "--walk_forward_enabled", - "true" if phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "true", - ], - outputs=[str(path) for path in outputs], - checkpoint=str(output_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile=feature_profile_name, - monotonic_policy=policy_name, - lane_id=lane, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - - -def _build_pd_phase_commands( - *, - artifact_root: Path, - safe_tag: str, - phase: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], -) -> list[PhaseCommand]: - commands = [ - _pd_incumbent_command( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase=phase, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - ) - ] - n_trials = _pd_trials_for_phase(phase) - commands.extend( - _pd_lane_command( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase=phase, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - feature_profile_name=feature_profile_name, - policy_name=policy_name, - n_trials=n_trials, - ) - for feature_profile_name, policy_name in _selected_pd_lanes(artifact_root, phase) - ) - return commands - - -def _build_conformal_phase_command( - *, - artifact_root: Path, - safe_tag: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], -) -> PhaseCommand: - phase = "conformal" - conformal_root = artifact_root / phase / safe_tag - pd_model_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_canonical.cbm", - ) - pd_calibrator_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_calibrator.pkl", - ) - outputs = [ - conformal_root / "data" / "conformal_intervals_mondrian.parquet", - conformal_root / "models" / "conformal_results_mondrian.pkl", - conformal_root / "models" / "pd_conformal_width_attribution_status.json", - ] - command_name = "conformal_extensive_grid" - stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) - return PhaseCommand( - name=command_name, - phase=phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "generate_conformal_intervals.py"), - "--artifact_namespace", - safe_tag, - "--artifact_root", - str(artifact_root / phase), - "--model_override_path", - str(pd_model_path), - "--alpha_candidates_90", - "0.05,0.075,0.09,0.095,0.10,0.105,0.11,0.125,0.15,0.20", - "--alpha_candidates_95", - "0.025,0.04,0.045,0.05,0.055,0.06,0.075", - "--partition_candidates", - "grade,score_decile_mondrian,grade_x_scoreband_mondrian", - "--n_score_bins_candidates", - "5,10,15,20,30", - "--min_group_sizes", - "100,150,250,500,1000,2000", - "--score_scale_families", - "none,bernoulli_sqrt,bernoulli_sqrt_clipped_0.02,bernoulli_sqrt_clipped_0.05", - "--calibrator_override_path", - str(pd_calibrator_path), - ], - outputs=[str(path) for path in outputs], - checkpoint=str(conformal_root / "checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - - -def _build_portfolio_phase_command( - *, - artifact_root: Path, - safe_tag: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], -) -> PhaseCommand: - phase = "portfolio" - portfolio_root = artifact_root / phase / safe_tag - conformal_path = ( - artifact_root / "conformal" / safe_tag / "data" / "conformal_intervals_mondrian.parquet" - ) - outputs = [ - portfolio_root / "data" / "portfolio_bound_aware_frontier.parquet", - portfolio_root / "data" / "portfolio_bound_aware_bound_eval.parquet", - portfolio_root / "models" / "portfolio_bound_aware_selection.json", - ] - command_name = "portfolio_extensive_frontier" - stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) - return PhaseCommand( - name=command_name, - phase=phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "search" / "run_portfolio_bound_aware_search.py"), - "--config", - str(ROOT / "configs" / "crpto_optimization.yaml"), - "--conformal-intervals-path", - str(conformal_path), - "--run-label", - safe_tag, - "--output-dir", - str(portfolio_root / "data"), - "--model-dir", - str(portfolio_root / "models"), - "--incumbent-policy-path", - str(CHAMPION_PORTFOLIO_POLICY_PATH), - "--incumbent-risk-neighbors", - "0.155,0.16,0.165,0.17,0.175,0.18", - "--incumbent-gamma-neighbors", - "0.425,0.45,0.475,0.50,0.525,0.55,0.575", - "--incumbent-policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--risk-grid", - PORTFOLIO_RISK_GRID, - "--gamma-grid", - PORTFOLIO_GAMMA_GRID, - "--aversion-grid", - PORTFOLIO_AVERSION_GRID, - "--delta-cap-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--tail-focus-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--alpha-grid", - PORTFOLIO_ALPHA_GRID, - "--max-candidates", - str(PORTFOLIO_MAX_CANDIDATES), - "--shortlist-top-k", - str(PORTFOLIO_SHORTLIST_TOP_K), - "--random-states", - PORTFOLIO_RANDOM_STATES, - "--solver-backend", - "highs", - "--exact-solver-backend", - "highs", - ], - outputs=[str(path) for path in outputs], - checkpoint=str(portfolio_root / "models" / "portfolio_bound_aware_runtime_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - - -def _build_metrics_phase_command( - *, - artifact_root: Path, - safe_tag: str, - base_env: Mapping[str, str], -) -> PhaseCommand: - phase = "metrics" - metrics_path = artifact_root / phase / "frontier_metrics_manifest.json" - command_name = "metrics_manifest" - stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) - return PhaseCommand( - name=command_name, - phase=phase, - command=[ - sys.executable, - str(Path(__file__).resolve()), - "--run-tag", - safe_tag, - "--artifact-root", - str(artifact_root), - "--phase", - "plan", - "--resume", - ], - outputs=[str(metrics_path)], - checkpoint=str(artifact_root / phase), - env=_command_env(base_env, phase_threads=1), - max_workers=1, - cpu_threads=1, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - - -def _commands_for_phase( - *, - artifact_root: Path, - safe_tag: str, - phase: str, - phase_workers: int, - phase_threads: int, - base_env: Mapping[str, str], -) -> list[PhaseCommand]: - if phase in PD_PHASES: - return _build_pd_phase_commands( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase=phase, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - ) - if phase == "conformal": - return [ - _build_conformal_phase_command( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - ) - ] - if phase == "portfolio": - return [ - _build_portfolio_phase_command( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - ) - ] - if phase == "metrics": - return [ - _build_metrics_phase_command( - artifact_root=artifact_root, - safe_tag=safe_tag, - base_env=base_env, - ) - ] - if phase in {"plan", "deps"}: - return [] - raise ValueError(f"Unknown sandbox phase: {phase}") - - -def _validate_phase_command_paths(commands: Iterable[PhaseCommand]) -> None: - for command in commands: - assert_safe_output_paths(command.outputs) - assert_safe_output_path(command.checkpoint) - if command.stdout_log: - assert_safe_output_path(command.stdout_log) - if command.stderr_log: - assert_safe_output_path(command.stderr_log) - - -def build_phase_commands( - *, - artifact_root: Path, - run_tag: str, - phase: str, - max_workers: int, - cpu_threads: int, -) -> list[PhaseCommand]: - """Build commands for the requested sandbox phase.""" - safe_tag = sanitize_tag(run_tag) - artifact_root = artifact_root.resolve() - base_env = _sandbox_base_env(artifact_root, safe_tag) - selected_phases = ( - ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] - if phase == "all" - else [phase] - ) - commands: list[PhaseCommand] = [] - for selected_phase in selected_phases: - phase_workers, phase_threads = _phase_resources( - selected_phase, - max_workers=max_workers, - cpu_threads=cpu_threads, - ) - commands.extend( - _commands_for_phase( - artifact_root=artifact_root, - safe_tag=safe_tag, - phase=selected_phase, - phase_workers=phase_workers, - phase_threads=phase_threads, - base_env=base_env, - ) - ) - _validate_phase_command_paths(commands) - return commands - - -def _write_dependency_snapshot(path: Path) -> Path: - packages = [ - "catboost", - "mapie", - "optuna", - "optuna-integration", - "pyomo", - "highspy", - "venn-abers", - "scikit-learn", - ] - cmd = [ - sys.executable, - "-c", - ( - "import importlib.metadata as m, json; " - f"pkgs={packages!r}; " - "print(json.dumps({p: m.version(p) for p in pkgs}, sort_keys=True))" - ), - ] - proc = subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=True) - payload = json.loads(proc.stdout) - return atomic_write_json( - path, - { - "schema_version": SCHEMA_VERSION, - "captured_at_utc": utc_now_iso(), - "versions": payload, - }, - ) - - -def _write_frontier_metrics_manifest(*, artifact_root: Path, run_tag: str) -> Path: - metrics_root = artifact_root / "metrics" - safe_tag = sanitize_tag(run_tag) - paths = { - "pd_incumbent_baseline": artifact_root - / "pd_baselines" - / "champion" - / "pd-smoke" - / "models" - / "pd_training_record.pkl", - "pd_selection": artifact_root / "pd" / "_selection" / "pd-refine_selection.json", - "pd_model": artifact_root / "pd" / "best" / "models" / "pd_shadow_canonical.cbm", - "pd_calibrator": artifact_root / "pd" / "best" / "models" / "pd_shadow_calibrator.pkl", - "conformal_intervals": artifact_root - / "conformal" - / safe_tag - / "data" - / "conformal_intervals_mondrian.parquet", - "portfolio_frontier": artifact_root - / "portfolio" - / safe_tag - / "data" - / "portfolio_bound_aware_frontier.parquet", - "portfolio_exact": artifact_root - / "portfolio" - / safe_tag - / "data" - / "portfolio_bound_aware_bound_eval.parquet", - "portfolio_selection": artifact_root - / "portfolio" - / safe_tag - / "models" - / "portfolio_bound_aware_selection.json", - } - payload = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": utc_now_iso(), - "run_tag": safe_tag, - "artifact_root": str(artifact_root), - "artifacts": { - name: {"path": str(path), "exists": path.exists()} for name, path in paths.items() - }, - "auditability_weights": dict(AUDITABILITY_WEIGHTS), - "regret_formula": ( - "oracle_realized_return_same_budget_concentration_and_expost_default_cap " - "- policy_realized_return" - ), - } - selection_path = paths["pd_selection"] - if selection_path.exists(): - payload["pd_selection"] = json.loads(selection_path.read_text(encoding="utf-8")) - return atomic_write_json(metrics_root / "frontier_metrics_manifest.json", payload) - - -def _write_command_manifest( - *, - artifact_root: Path, - run_tag: str, - phase: str, - commands: Sequence[PhaseCommand], - resume_manifest: Mapping[str, Any], -) -> Path: - manifest_path = artifact_root / "sandbox_manifest.json" - payload = { - "schema_version": SCHEMA_VERSION, - "stage_name": STAGE_NAME, - "run_tag": run_tag, - "phase": phase, - "artifact_root": str(artifact_root), - "generated_at_utc": utc_now_iso(), - "resume_manifest_loaded": bool(resume_manifest), - "resource_policy": { - "reserve_logical_cpus": DEFAULT_RESERVED_LOGICAL_CPUS, - "min_available_ram_gb": DEFAULT_MIN_AVAILABLE_RAM_GB, - "heartbeat_seconds": DEFAULT_HEARTBEAT_SECONDS, - }, - "mlflow_tracking": { - "tracking_uri": _mlflow_tracking_uri(artifact_root), - "experiment_name": _mlflow_experiment_name(run_tag), - "scope": "sandbox command/lane registry; Optuna trial detail remains in per-lane storage", - "size_limit_bytes": MLFLOW_LOG_SIZE_LIMIT_BYTES, - }, - "monotonic_policies": materialize_monotonic_policies(), - "feature_profiles": materialize_feature_profiles(), - "pd_lane_selection": { - "pd_broad_top_k_from_smoke": PD_BROAD_TOP_K_LANES, - "pd_refine_top_k_from_broad": PD_REFINE_TOP_K_LANES, - }, - "incumbent_replay": { - "enabled": True, - "pd_config_path": str(CHAMPION_PD_CONFIG_PATH), - "portfolio_policy_path": str(CHAMPION_PORTFOLIO_POLICY_PATH), - "pd_baseline_root": str(artifact_root / "pd_baselines" / "champion"), - "role": "baseline only; never overwrites frozen champion artifacts", - }, - "pd_warm_start": { - "enabled": True, - "sources": [ - "frozen_champion_pd_config", - "same-lane best params from previous PD phase", - "top previous-phase PD params as cross-lane priors", - ], - }, - "resource_tuned_search_budget": { - "pd_smoke_trials_per_lane": PD_SMOKE_TRIALS, - "pd_broad_trials_per_lane": PD_BROAD_TRIALS, - "pd_refine_trials_per_lane": PD_REFINE_TRIALS, - "portfolio_risk_grid": PORTFOLIO_RISK_GRID, - "portfolio_gamma_grid": PORTFOLIO_GAMMA_GRID, - "portfolio_aversion_grid": PORTFOLIO_AVERSION_GRID, - "portfolio_cap_tail_grid": PORTFOLIO_CAP_TAIL_GRID, - "portfolio_random_states": PORTFOLIO_RANDOM_STATES, - "portfolio_max_candidates": PORTFOLIO_MAX_CANDIDATES, - "portfolio_shortlist_top_k": PORTFOLIO_SHORTLIST_TOP_K, - }, - "auditability_weights": dict(AUDITABILITY_WEIGHTS), - "protected_paths": { - "exact": list(PROTECTED_REPO_PATHS), - "directories": list(PROTECTED_REPO_DIRS), - "globs": list(PROTECTED_REPO_GLOBS), - }, - "commands": [asdict(command) for command in commands], - } - return atomic_write_json(manifest_path, payload) - - -def _completed_outputs(command: PhaseCommand) -> bool: - return all(Path(output).exists() for output in command.outputs) - - -def _resource_allows_launch(artifact_root: Path) -> bool: - snapshot = _resource_snapshot(artifact_root) - available = snapshot.get("ram_available_gb") - if available is None: - return True - return float(available) >= DEFAULT_MIN_AVAILABLE_RAM_GB - - -def _append_command_log(path: Path, row: Mapping[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - exists = path.exists() - with path.open("a", newline="", encoding="utf-8") as fh: - writer = csv.DictWriter( - fh, - fieldnames=[ - "captured_at_utc", - "phase", - "name", - "state", - "returncode", - "checkpoint", - "stdout_log", - "stderr_log", - ], - ) - if not exists: - writer.writeheader() - writer.writerow(dict(row)) - - -def _mlflow_tracking_uri(artifact_root: Path) -> str: - configured_uri = os.environ.get("MLFLOW_TRACKING_URI", "").strip() - if configured_uri: - return configured_uri - return (artifact_root / "mlruns").resolve().as_uri() - - -def _mlflow_experiment_name(run_tag: str) -> str: - configured_name = os.environ.get("CRPTO_SANDBOX_MLFLOW_EXPERIMENT", "").strip() - if configured_name: - return configured_name - return f"crpto_regret_auditability_sandbox_{sanitize_tag(run_tag)}" - - -def _log_mlflow_error(artifact_root: Path, message: str) -> None: - log_path = artifact_root / "logs" / "mlflow_tracking_errors.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - with log_path.open("a", encoding="utf-8") as fh: - fh.write(f"[{utc_now_iso()}] {message}\n") - - -def _extract_command_artifact_paths(command: PhaseCommand) -> list[Path]: - paths: list[Path] = [] - flag_names = { - "--config", - "--profile-config", - "--artifact_root", - "--artifact-root", - "--output-dir", - "--output_dir", - "--model-dir", - "--model_dir", - } - for index, item in enumerate(command.command[:-1]): - if item in flag_names: - paths.append(Path(command.command[index + 1])) - if command.stdout_log: - paths.append(Path(command.stdout_log)) - if command.stderr_log: - paths.append(Path(command.stderr_log)) - if command.checkpoint: - checkpoint = Path(command.checkpoint) - if checkpoint.exists(): - paths.append(checkpoint) - for output in command.outputs: - path = Path(output) - if path.suffix.lower() in {".json", ".yaml", ".yml", ".csv", ".parquet", ".db"}: - paths.append(path) - return paths - - -def _log_command_to_mlflow( - *, - artifact_root: Path, - command: PhaseCommand, - state: str, - returncode: int, -) -> None: - run_tag = command.env.get("PIPELINE_RUN_TAG") or command.env.get("CRPTO_RUN_TAG") or "unknown" - try: - import mlflow - - tracking_uri = _mlflow_tracking_uri(artifact_root) - mlflow.set_tracking_uri(tracking_uri) - mlflow.set_experiment(_mlflow_experiment_name(run_tag)) - with mlflow.start_run(run_name=command.name): - mlflow.set_tags( - { - "crpto.stage": STAGE_NAME, - "crpto.run_tag": run_tag, - "crpto.phase": command.phase, - "crpto.state": state, - "crpto.lane_id": command.lane_id or "", - "crpto.feature_profile": command.feature_profile or "", - "crpto.monotonic_policy": command.monotonic_policy or "", - } - ) - mlflow.log_params( - { - "max_workers": command.max_workers, - "cpu_threads": command.cpu_threads, - "command_name": command.name, - "checkpoint": command.checkpoint, - } - ) - mlflow.log_metric("returncode", float(returncode)) - command_payload = { - "schema_version": SCHEMA_VERSION, - "logged_at_utc": utc_now_iso(), - "state": state, - "returncode": returncode, - "command": command.command, - "outputs": command.outputs, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - } - command_json = artifact_root / "logs" / command.phase / f"{command.name}.mlflow.json" - atomic_write_json(command_json, command_payload) - mlflow.log_artifact(str(command_json), artifact_path=f"commands/{command.phase}") - seen: set[Path] = set() - for path in _extract_command_artifact_paths(command): - resolved = path.resolve() - if resolved in seen or not resolved.exists(): - continue - seen.add(resolved) - if resolved.is_file(): - size = resolved.stat().st_size - if size <= MLFLOW_LOG_SIZE_LIMIT_BYTES: - mlflow.log_artifact( - str(resolved), artifact_path=f"artifacts/{command.phase}" - ) - elif resolved.is_dir(): - for child in resolved.rglob("*"): - if child.is_file() and child.stat().st_size <= MLFLOW_LOG_SIZE_LIMIT_BYTES: - mlflow.log_artifact( - str(child), - artifact_path=f"artifacts/{command.phase}/{resolved.name}", - ) - except Exception as exc: - _log_mlflow_error(artifact_root, f"{command.name}: {type(exc).__name__}: {exc}") - - -def _run_one_command(command: PhaseCommand) -> tuple[PhaseCommand, int]: - env = dict(os.environ) - env.update(command.env) - stdout_path = Path(command.stdout_log) if command.stdout_log else None - stderr_path = Path(command.stderr_log) if command.stderr_log else None - if stdout_path is not None: - stdout_path.parent.mkdir(parents=True, exist_ok=True) - if stderr_path is not None: - stderr_path.parent.mkdir(parents=True, exist_ok=True) - stdout_fh = stdout_path.open("a", encoding="utf-8", errors="replace") if stdout_path else None - stderr_fh = stderr_path.open("a", encoding="utf-8", errors="replace") if stderr_path else None - returncode = 1 - try: - if stdout_fh: - stdout_fh.write(f"\n[{utc_now_iso()}] START {' '.join(command.command)}\n") - stdout_fh.flush() - if stderr_fh: - stderr_fh.write(f"\n[{utc_now_iso()}] START {' '.join(command.command)}\n") - stderr_fh.flush() - proc = subprocess.run( - command.command, - cwd=str(ROOT), - env=env, - stdout=stdout_fh, - stderr=stderr_fh, - check=False, - ) - returncode = int(proc.returncode) - except OSError as exc: - if stderr_fh: - stderr_fh.write(f"\n[{utc_now_iso()}] LAUNCH_ERROR {exc}\n") - stderr_fh.flush() - finally: - if stdout_fh: - stdout_fh.close() - if stderr_fh: - stderr_fh.close() - return command, returncode - - -def _phase_command_groups(commands: Sequence[PhaseCommand]) -> list[list[PhaseCommand]]: - phase_groups: list[list[PhaseCommand]] = [] - for command in commands: - if not phase_groups or phase_groups[-1][0].phase != command.phase: - phase_groups.append([command]) - else: - phase_groups[-1].append(command) - return phase_groups - - -def _log_skipped_completed( - *, - artifact_root: Path, - log_path: Path, - command: PhaseCommand, -) -> None: - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "skipped_completed", - "returncode": 0, - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) - _log_command_to_mlflow( - artifact_root=artifact_root, - command=command, - state="skipped_completed", - returncode=0, - ) - - -def _pending_commands_for_group( - *, - artifact_root: Path, - log_path: Path, - group: list[PhaseCommand], - resume: bool, -) -> tuple[deque[PhaseCommand], int]: - pending: deque[PhaseCommand] = deque() - skipped = 0 - for command in group: - if resume and _completed_outputs(command): - _log_skipped_completed( - artifact_root=artifact_root, - log_path=log_path, - command=command, - ) - skipped += 1 - continue - pending.append(command) - return pending, skipped - - -def _start_available_commands( - *, - artifact_root: Path, - log_path: Path, - pending: deque[PhaseCommand], - running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand], - executor: ThreadPoolExecutor, - worker_limit: int, -) -> Path | None: - last_checkpoint: Path | None = None - while pending and len(running) < worker_limit and _resource_allows_launch(artifact_root): - command = pending.popleft() - last_checkpoint = Path(command.checkpoint) - future = executor.submit(_run_one_command, command) - running[future] = command - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "started", - "returncode": "", - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) - return last_checkpoint - - -def _record_finished_command( - *, - artifact_root: Path, - log_path: Path, - finished_command: PhaseCommand, - returncode: int, - failed_commands: list[PhaseCommand], -) -> None: - state = "complete" if returncode == 0 else "failed" - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": finished_command.phase, - "name": finished_command.name, - "state": state, - "returncode": returncode, - "checkpoint": finished_command.checkpoint, - "stdout_log": finished_command.stdout_log, - "stderr_log": finished_command.stderr_log, - }, - ) - _log_command_to_mlflow( - artifact_root=artifact_root, - command=finished_command, - state=state, - returncode=returncode, - ) - if returncode != 0 and finished_command.phase in PD_PHASES: - failed_commands.append(finished_command) - elif returncode != 0: - raise RuntimeError( - f"Sandbox command failed ({finished_command.name}) with return code {returncode}" - ) - - -def _select_phase_winners_if_needed( - *, - artifact_root: Path, - group: list[PhaseCommand], - failed_commands: list[PhaseCommand], - completed: int, - total_commands: int, -) -> None: - if not group or group[0].phase not in PD_PHASES: - return - selection_path = _select_pd_phase_winners(artifact_root, group[0].phase) - if selection_path is None: - failed_names = ", ".join(command.name for command in failed_commands[:10]) - raise RuntimeError( - "No successful PD candidates available after phase " - f"{group[0].phase}. Failed lanes: {failed_names}" - ) - _write_heartbeat( - artifact_root=artifact_root, - phase=group[0].phase, - completed_units=completed, - total_units=total_commands, - current_best_metric=None, - last_checkpoint_path=selection_path, - state="selected_with_failures" if failed_commands else "selected", - ) - - -def _run_commands( - *, - artifact_root: Path, - commands: Sequence[PhaseCommand], - resume: bool, -) -> None: - log_path = artifact_root / "command_log.csv" - completed = 0 - phase_groups = _phase_command_groups(commands) - - for group in phase_groups: - pending, skipped = _pending_commands_for_group( - artifact_root=artifact_root, - log_path=log_path, - group=group, - resume=resume, - ) - completed += skipped - worker_limit = max(1, max((command.max_workers for command in group), default=1)) - running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand] = {} - failed_commands: list[PhaseCommand] = [] - last_checkpoint: Path | None = None - with ThreadPoolExecutor(max_workers=worker_limit) as executor: - while pending or running: - next_checkpoint = _start_available_commands( - artifact_root=artifact_root, - log_path=log_path, - pending=pending, - running=running, - executor=executor, - worker_limit=worker_limit, - ) - if next_checkpoint is not None: - last_checkpoint = next_checkpoint - _write_heartbeat( - artifact_root=artifact_root, - phase=group[0].phase if group else "waiting_for_ram", - completed_units=completed, - total_units=len(commands), - current_best_metric=None, - last_checkpoint_path=last_checkpoint, - state="running" if running else "waiting_for_ram", - ) - if not running: - time.sleep(min(DEFAULT_HEARTBEAT_SECONDS, 5)) - continue - done, _ = wait( - running.keys(), - timeout=DEFAULT_HEARTBEAT_SECONDS, - return_when=FIRST_COMPLETED, - ) - for future in done: - running.pop(future) - finished_command, returncode = future.result() - completed += 1 - _record_finished_command( - artifact_root=artifact_root, - log_path=log_path, - finished_command=finished_command, - returncode=returncode, - failed_commands=failed_commands, - ) - _select_phase_winners_if_needed( - artifact_root=artifact_root, - group=group, - failed_commands=failed_commands, - completed=completed, - total_commands=len(commands), - ) - _write_heartbeat( - artifact_root=artifact_root, - phase=commands[-1].phase if commands else "plan", - completed_units=len(commands), - total_units=len(commands), - current_best_metric=None, - last_checkpoint_path=Path(commands[-1].checkpoint) if commands else None, - state="complete", - ) - - -def _default_workers_for_phase(phase: str) -> int: - if phase in PD_PHASES: - return DEFAULT_PD_WORKERS - if phase == "conformal": - return 6 - if phase == "portfolio": - return 4 - return 1 - - -def _default_threads_for_phase(phase: str) -> int: - if phase in PD_PHASES: - return DEFAULT_PD_THREADS - if phase == "portfolio": - return 4 - return 1 - - -def _execution_phases(requested_phase: str) -> list[str]: - if requested_phase == "all": - return ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] - return [requested_phase] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--run-tag", default=None) - parser.add_argument("--artifact-root", default=None) - parser.add_argument("--phase", choices=PHASE_CHOICES, default="plan") - parser.add_argument("--resume", action="store_true") - parser.add_argument("--max-workers", type=int, default=0) - parser.add_argument("--cpu-threads", type=int, default=0) - parser.add_argument("--no-gpu", action="store_true", default=True) - args = parser.parse_args(argv) - - run_tag = sanitize_tag(args.run_tag or default_run_tag()) - artifact_root = resolve_artifact_root(args.artifact_root, run_tag) - assert_safe_output_path(artifact_root) - artifact_root.mkdir(parents=True, exist_ok=True) - - max_workers = int(args.max_workers) - cpu_threads = int(args.cpu_threads) - if max_workers < 0 or cpu_threads < 0: - raise ValueError("--max-workers and --cpu-threads must be non-negative") - if not bool(args.no_gpu): - raise ValueError("This sandbox is CPU-only; pass --no-gpu or omit GPU options.") - - resume_manifest_path = artifact_root / "sandbox_manifest.json" - resume_manifest = load_resume_manifest(resume_manifest_path) if args.resume else {} - before_path = artifact_root / "dependency_versions_before.json" - if not before_path.exists(): - _write_dependency_snapshot(before_path) - - commands = ( - [] - if args.phase == "all" - else build_phase_commands( - artifact_root=artifact_root, - run_tag=run_tag, - phase=args.phase, - max_workers=max_workers, - cpu_threads=cpu_threads, - ) - ) - manifest_path = _write_command_manifest( - artifact_root=artifact_root, - run_tag=run_tag, - phase=args.phase, - commands=commands, - resume_manifest=resume_manifest, - ) - _write_heartbeat( - artifact_root=artifact_root, - phase=args.phase, - completed_units=0, - total_units=len(commands), - current_best_metric=None, - last_checkpoint_path=manifest_path, - state="planned" if args.phase in {"plan", "deps"} else "ready", - ) - - if args.phase == "deps": - atomic_write_json( - artifact_root / "dependency_upgrade_command.json", - { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": utc_now_iso(), - "command": [ - "uv", - "lock", - "--upgrade-package", - "catboost", - "--upgrade-package", - "mapie", - "--upgrade-package", - "optuna", - "--upgrade-package", - "optuna-integration", - "--upgrade-package", - "pyomo", - "--upgrade-package", - "highspy", - "--upgrade-package", - "venn-abers", - "--upgrade-package", - "scikit-learn", - ], - }, - ) - return 0 - if args.phase == "plan": - _write_frontier_metrics_manifest(artifact_root=artifact_root, run_tag=run_tag) - return 0 - if args.phase != "plan": - if args.phase == "all": - for execution_phase in _execution_phases(args.phase): - phase_commands = build_phase_commands( - artifact_root=artifact_root, - run_tag=run_tag, - phase=execution_phase, - max_workers=max_workers, - cpu_threads=cpu_threads, - ) - _write_command_manifest( - artifact_root=artifact_root, - run_tag=run_tag, - phase=execution_phase, - commands=phase_commands, - resume_manifest=resume_manifest, - ) - _run_commands( - artifact_root=artifact_root, - commands=phase_commands, - resume=bool(args.resume), - ) - else: - _run_commands(artifact_root=artifact_root, commands=commands, resume=bool(args.resume)) - if args.phase == "metrics": - _write_frontier_metrics_manifest(artifact_root=artifact_root, run_tag=run_tag) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/select_economic_portfolio_policy.py b/scripts/select_economic_portfolio_policy.py deleted file mode 100644 index f292d04..0000000 --- a/scripts/select_economic_portfolio_policy.py +++ /dev/null @@ -1,768 +0,0 @@ -"""Select canonical portfolio policy using actual A/B economics on the real universe.""" - -from __future__ import annotations - -import argparse -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Any, cast - -import numpy as np -import pandas as pd -import yaml -from loguru import logger - -from scripts.optimize_portfolio_tradeoff import _allocation_similarity -from scripts.simulate_ab_test import ( - _apply_candidate_universe, - _apply_decision_scenario, - _build_common_inputs, - _candidate_metrics, - _run_strategy, -) -from src.evaluation.ab_testing import compare_strategies -from src.optimization.portfolio_model import solution_allocation_vector -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.script_helpers import artifact_path as _artifact_path, try_load_json - -SCHEMA_VERSION = "2026-03-10.1" - - -@dataclass(frozen=True) -class SelectionSettings: - top_k: int - selector_name: str - min_funded_ratio: float - min_total_allocated_ratio: float - min_breadth_score: float - breadth_weight_funded_ratio: float - breadth_weight_allocation_ratio: float - breadth_weight_allocation_similarity: float - max_por_pct: float - canonical_modes: set[str] - ab_like_top_m: int - ab_like_bootstrap_n: int - ab_like_seed: int - - -@dataclass(frozen=True) -class DecisionInputs: - common: dict[str, object] - default_flag: np.ndarray - loan_amnt: np.ndarray - int_rates: np.ndarray - pd_high: np.ndarray - total_budget: float - universe_source: str | None - scenario_meta: dict[str, Any] - - -@dataclass(frozen=True) -class SelectionResult: - selected: dict[str, Any] - selected_policy: dict[str, Any] - selector_outcome: str - fallback_applied: bool - fallback_reason: str | None - - -def _policy_key(row: pd.Series) -> tuple[object, ...]: - return ( - str(row.get("policy_mode", "")), - float(row.get("gamma", 0.0)), - float(row.get("risk_tolerance", 0.0)), - float(row.get("delta_cap_quantile", 1.0)), - float(row.get("tail_focus_quantile", 1.0)), - float(row.get("uncertainty_aversion", 0.0)), - float(row.get("min_budget_utilization", 0.0)), - float(row.get("pd_cap_slack_penalty", 0.0)), - ) - - -def _policy_from_row(row: pd.Series, source: str) -> dict[str, Any]: - return { - "source": source, - "risk_tolerance": float(row["risk_tolerance"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - "min_budget_utilization": float(row["min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["pd_cap_slack_penalty"]), - "policy_mode": str(row["policy_mode"]), - "gamma": float(row["gamma"]), - "delta_cap_quantile": float(row.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(row.get("tail_focus_quantile", 1.0)), - } - - -def _load_json(path: Path) -> dict[str, Any]: - return try_load_json(path) - - -def _load_config(path: str) -> dict[str, Any]: - with open(path, encoding="utf-8") as f: - payload = yaml.safe_load(f) - return payload if isinstance(payload, dict) else {} - - -def _selection_settings(config: dict[str, Any]) -> SelectionSettings: - selection_cfg = dict(config.get("portfolio_selection", {}) or {}) - min_funded_ratio = float(selection_cfg.get("min_funded_ratio", 0.95)) - return SelectionSettings( - top_k=int(selection_cfg.get("actual_ab_top_k", 20)), - selector_name=str(selection_cfg.get("canonical_selector", "economic_actual_ab_v1")), - min_funded_ratio=min_funded_ratio, - min_total_allocated_ratio=float(selection_cfg.get("min_total_allocated_ratio", 0.98)), - min_breadth_score=float(selection_cfg.get("min_breadth_score", min_funded_ratio)), - breadth_weight_funded_ratio=float(selection_cfg.get("breadth_weight_funded_ratio", 0.5)), - breadth_weight_allocation_ratio=float( - selection_cfg.get("breadth_weight_allocation_ratio", 0.3) - ), - breadth_weight_allocation_similarity=float( - selection_cfg.get("breadth_weight_allocation_similarity", 0.2) - ), - max_por_pct=float(selection_cfg.get("max_price_of_robustness_pct", -15.0)), - canonical_modes={ - str(x) for x in selection_cfg.get("canonical_policy_modes", ["blended_uncertainty"]) - }, - ab_like_top_m=int(selection_cfg.get("ab_like_top_m", 8)), - ab_like_bootstrap_n=int(selection_cfg.get("ab_like_bootstrap_n", 200)), - ab_like_seed=int(selection_cfg.get("ab_like_seed", 42)), - ) - - -def _load_frontier(frontier_path: str) -> pd.DataFrame: - frontier = pd.read_parquet(_artifact_path(frontier_path)) - if frontier.empty: - raise ValueError("portfolio_robustness_frontier.parquet is empty") - return frontier - - -def _prepare_decision_inputs( - *, - config: dict[str, Any], - candidate_universe_path: str, - decision_scenario: str, -) -> DecisionInputs: - test_df = pd.read_parquet("data/processed/test_fe.parquet") - intervals = pd.read_parquet("data/processed/conformal_intervals_mondrian.parquet") - test_df, intervals, universe_source = _apply_candidate_universe( - test_df, - intervals, - candidate_universe_path=candidate_universe_path, - max_candidates=0, - ) - test_df, intervals, scenario_meta = _apply_decision_scenario( - test_df, - intervals, - decision_scenario=decision_scenario, - ) - common, default_flag, loan_amnt, int_rates, pd_high = _build_common_inputs(test_df, intervals) - return DecisionInputs( - common=common, - default_flag=default_flag, - loan_amnt=loan_amnt, - int_rates=int_rates, - pd_high=pd_high, - total_budget=float(config["portfolio"]["total_budget"]), - universe_source=universe_source, - scenario_meta=cast(dict[str, Any], scenario_meta), - ) - - -def _dedupe_candidates(rows: list[pd.Series]) -> list[pd.Series]: - out: list[pd.Series] = [] - seen: set[tuple[object, ...]] = set() - for row in rows: - key = _policy_key(row) - if key in seen: - continue - seen.add(key) - out.append(row) - return out - - -def _select_candidate_rows(frontier: pd.DataFrame, top_k: int) -> list[pd.Series]: - work = frontier.copy() - if "tail_focus_quantile" not in work.columns: - work["tail_focus_quantile"] = 1.0 - work = work.loc[ - (work["policy"] != "nonrobust") - & work["eligible_for_canonical_selection"].fillna(False).astype(bool) - ].copy() - if work.empty: - return [] - - work["realized_total_return"] = pd.to_numeric(work["realized_total_return"], errors="coerce") - candidates: list[pd.Series] = [] - bucket_cols = [ - "policy_mode", - "gamma", - "risk_tolerance", - "delta_cap_quantile", - "tail_focus_quantile", - ] - for _, bucket in work.groupby(bucket_cols, dropna=False): - top = bucket.sort_values("realized_total_return", ascending=False).head(int(top_k)) - candidates.extend(top.to_dict(orient="records")) - - flag_cols = [ - "selected_for_champion", - "selected_for_balanced_robustness", - "selected_for_guardrail_robustness", - ] - for col in flag_cols: - if col in work.columns: - flagged = work.loc[work[col].fillna(False).astype(bool)] - candidates.extend(flagged.to_dict(orient="records")) - return _dedupe_candidates([pd.Series(r) for r in candidates]) - - -def _breadth_score( - *, - funded_ratio: float, - total_allocated_ratio: float, - allocation_similarity: float, - weight_funded_ratio: float, - weight_allocation_ratio: float, - weight_allocation_similarity: float, -) -> float: - total_weight = ( - float(weight_funded_ratio) - + float(weight_allocation_ratio) - + float(weight_allocation_similarity) - ) - if total_weight <= 0: - return float(np.clip(allocation_similarity, 0.0, 1.0)) - score = ( - float(weight_funded_ratio) * float(np.clip(funded_ratio, 0.0, 1.0)) - + float(weight_allocation_ratio) * float(np.clip(total_allocated_ratio, 0.0, 1.0)) - + float(weight_allocation_similarity) * float(np.clip(allocation_similarity, 0.0, 1.0)) - ) / total_weight - return float(np.clip(score, 0.0, 1.0)) - - -def _ab_like_score( - *, - returns_control: np.ndarray, - returns_candidate: np.ndarray, - seed: int, - n_boot: int, -) -> dict[str, float | bool]: - stats = compare_strategies( - returns_a=returns_control, - returns_b=returns_candidate, - method="bootstrap", - n_boot=n_boot, - alpha=0.05, - seed=seed, - ) - diff_total = float(np.sum(returns_candidate) - np.sum(returns_control)) - tolerance_total = abs(float(np.sum(returns_control))) * 0.05 - return { - "ab_like_diff_total_return": diff_total, - "ab_like_tolerance_total_return": tolerance_total, - "ab_like_passed_no_regression": bool(diff_total >= -tolerance_total), - "ab_like_mean_diff": float(stats["diff"]), - "ab_like_ci_low": float(stats["ci_low"]), - "ab_like_ci_high": float(stats["ci_high"]), - "ab_like_p_value": float(stats["p_value"]), - } - - -def _control_metrics_by_risk( - *, - common: dict[str, object], - default_flag: np.ndarray, - loan_amnt: np.ndarray, - int_rates: np.ndarray, - risk_values: list[float], - total_budget: float, - solver_backend: str, -) -> dict[float, dict[str, Any]]: - controls: dict[float, dict[str, Any]] = {} - for risk_tol in sorted({float(x) for x in risk_values}): - sol, _ = _run_strategy( - common=common, - robust=False, - total_budget=total_budget, - max_portfolio_pd=risk_tol, - solver_backend=solver_backend, - ) - returns, metrics = _candidate_metrics( - solution=sol, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=0.45, - ) - alloc = solution_allocation_vector(sol, len(loan_amnt)) - controls[risk_tol] = { - "solution": sol, - "returns": returns, - "metrics": metrics, - "allocation": alloc, - "worst_case_pd": float( - np.sum(alloc * loan_amnt * np.asarray(common["pd_high"], dtype=float)) - / (float(sol["total_allocated"]) + 1e-6) - ), - } - return controls - - -def _evaluate_candidate_row( - *, - row: pd.Series, - inputs: DecisionInputs, - controls: dict[float, dict[str, Any]], - settings: SelectionSettings, - solver_backend: str, -) -> dict[str, Any]: - policy = _policy_from_row(row, source="economic_actual_ab_v1") - risk_tol = float(policy["risk_tolerance"]) - control = controls[risk_tol] - sol_b, _ = _run_strategy( - common=inputs.common, - robust=True, - robust_policy=policy, - total_budget=inputs.total_budget, - max_portfolio_pd=risk_tol, - solver_backend=solver_backend, - ) - returns_b, metrics_b = _candidate_metrics( - solution=sol_b, - loan_amnt=inputs.loan_amnt, - int_rates=inputs.int_rates, - default_flag=inputs.default_flag, - lgd_val=0.45, - ) - control_metrics = control["metrics"] - returns_control = np.asarray(control["returns"], dtype=float) - diff_total_return = float(metrics_b["total_return"] - float(control_metrics["total_return"])) - return_delta_pct = float( - diff_total_return / (abs(float(control_metrics["total_return"])) + 1e-6) * 100.0 - ) - tolerance_total_return = abs(float(control_metrics["total_return"])) * 0.05 - funded_ratio = float(metrics_b["n_funded"] / max(float(control_metrics["n_funded"]), 1.0)) - total_allocated_ratio = float( - metrics_b["total_allocated"] / max(float(control_metrics["total_allocated"]), 1.0) - ) - alloc_b = solution_allocation_vector(sol_b, len(inputs.loan_amnt)) - allocation_similarity = _allocation_similarity(control["allocation"], alloc_b) - breadth_score = _breadth_score( - funded_ratio=funded_ratio, - total_allocated_ratio=total_allocated_ratio, - allocation_similarity=allocation_similarity, - weight_funded_ratio=settings.breadth_weight_funded_ratio, - weight_allocation_ratio=settings.breadth_weight_allocation_ratio, - weight_allocation_similarity=settings.breadth_weight_allocation_similarity, - ) - cand_worst_pd = float( - np.sum(alloc_b * inputs.loan_amnt * inputs.pd_high) - / (float(sol_b["total_allocated"]) + 1e-6) - ) - return { - "policy": policy, - "risk_tolerance": risk_tol, - "passed_no_regression": bool(diff_total_return >= -tolerance_total_return), - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "funded_ratio": funded_ratio, - "total_allocated_ratio": total_allocated_ratio, - "worst_case_pd_reduction_bps": float( - (float(control["worst_case_pd"]) - cand_worst_pd) * 1e4 - ), - "price_of_robustness_pct": float(min(return_delta_pct, 0.0)), - "return_delta_pct": return_delta_pct, - "frontier_price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0.0)), - "return_per_funded_delta": float( - metrics_b["avg_return_per_funded"] - float(control_metrics["avg_return_per_funded"]) - ), - "allocation_similarity": allocation_similarity, - "breadth_score": breadth_score, - "n_funded_candidate": int(metrics_b["n_funded"]), - "n_funded_control": int(control_metrics["n_funded"]), - "total_return_candidate": float(metrics_b["total_return"]), - "total_return_control": float(control_metrics["total_return"]), - "eligible_hard_filters": False, - "_returns_candidate": returns_b, - "_returns_control": returns_control, - } - - -def _evaluate_candidate_rows( - *, - candidate_rows: list[pd.Series], - inputs: DecisionInputs, - controls: dict[float, dict[str, Any]], - settings: SelectionSettings, - solver_backend: str, -) -> list[dict[str, Any]]: - return [ - _evaluate_candidate_row( - row=row, - inputs=inputs, - controls=controls, - settings=settings, - solver_backend=solver_backend, - ) - for row in candidate_rows - ] - - -def _base_hard_filters(item: dict[str, Any], settings: SelectionSettings) -> bool: - return bool( - item["passed_no_regression"] - and float(item["price_of_robustness_pct"]) >= settings.max_por_pct - and str(item["policy"]["policy_mode"]) in settings.canonical_modes - ) - - -def _mark_hard_filter_eligibility( - evaluated: list[dict[str, Any]], - settings: SelectionSettings, -) -> None: - for item in evaluated: - base_filters = _base_hard_filters(item, settings) - if settings.selector_name in {"economic_actual_ab_v2", "economic_actual_ab_v3"}: - item["eligible_hard_filters"] = bool( - base_filters - and float(item["total_allocated_ratio"]) >= settings.min_total_allocated_ratio - and float(item["breadth_score"]) >= settings.min_breadth_score - and float(item["funded_ratio"]) >= settings.min_funded_ratio - ) - else: - item["eligible_hard_filters"] = bool( - base_filters and float(item["funded_ratio"]) >= settings.min_funded_ratio - ) - - -def _base_rank_key(item: dict[str, Any]) -> tuple[float, float, float, float, float]: - return ( - float(item["worst_case_pd_reduction_bps"]), - float(item["diff_total_return"]), - float(item.get("breadth_score", 0.0)), - -abs(float(item["price_of_robustness_pct"])), - float(item["funded_ratio"]), - ) - - -def _v3_rank_key(item: dict[str, Any]) -> tuple[bool, float, float, float, float, float]: - return ( - bool(item.get("ab_like_passed_no_regression", False)), - float(item.get("ab_like_diff_total_return", item["diff_total_return"])), - float(item["worst_case_pd_reduction_bps"]), - float(item.get("breadth_score", 0.0)), - -abs(float(item["price_of_robustness_pct"])), - float(item["funded_ratio"]), - ) - - -def _candidate_pool_after_filters(evaluated: list[dict[str, Any]]) -> list[dict[str, Any]]: - eligible = [x for x in evaluated if bool(x["eligible_hard_filters"])] - robust_eligible = [x for x in eligible if float(x["policy"]["gamma"]) > 0.0] - return robust_eligible or eligible - - -def _apply_ab_like_screen( - candidate_pool: list[dict[str, Any]], - settings: SelectionSettings, -) -> list[dict[str, Any]]: - if settings.selector_name != "economic_actual_ab_v3" or not candidate_pool: - return candidate_pool - pre_ranked = sorted(candidate_pool, key=_base_rank_key, reverse=True)[ - : max(1, settings.ab_like_top_m) - ] - for idx, item in enumerate(pre_ranked): - item.update( - _ab_like_score( - returns_control=np.asarray(item["_returns_control"], dtype=float), - returns_candidate=np.asarray(item["_returns_candidate"], dtype=float), - seed=settings.ab_like_seed + idx, - n_boot=settings.ab_like_bootstrap_n, - ) - ) - robust_ab_like = [x for x in pre_ranked if bool(x.get("ab_like_passed_no_regression", False))] - return robust_ab_like or pre_ranked - - -def _select_ranked_candidate( - candidate_pool: list[dict[str, Any]], - settings: SelectionSettings, -) -> dict[str, Any]: - rank_key = _v3_rank_key if settings.selector_name == "economic_actual_ab_v3" else _base_rank_key - return sorted(candidate_pool, key=rank_key, reverse=True)[0] - - -def _fallback_selected_candidate( - frontier: pd.DataFrame, - controls: dict[float, dict[str, Any]], -) -> dict[str, Any]: - fallback_row = frontier.loc[frontier["selected_for_champion"].fillna(False).astype(bool)] - selected_row = fallback_row.iloc[0] if not fallback_row.empty else frontier.iloc[0] - risk_tol = float(selected_row["risk_tolerance"]) - control_metrics = controls[risk_tol]["metrics"] - return { - "policy": { - **_policy_from_row(selected_row, source="economic_actual_ab_v1_fallback"), - "gamma": 0.0, - "policy_mode": "blended_uncertainty", - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - }, - "risk_tolerance": risk_tol, - "passed_no_regression": True, - "diff_total_return": 0.0, - "tolerance_total_return": abs(float(control_metrics["total_return"])) * 0.05, - "funded_ratio": 1.0, - "total_allocated_ratio": 1.0, - "worst_case_pd_reduction_bps": 0.0, - "price_of_robustness_pct": 0.0, - "return_per_funded_delta": 0.0, - "allocation_similarity": 1.0, - "breadth_score": 1.0, - "n_funded_candidate": int(control_metrics["n_funded"]), - "n_funded_control": int(control_metrics["n_funded"]), - "total_return_candidate": float(control_metrics["total_return"]), - "total_return_control": float(control_metrics["total_return"]), - "eligible_hard_filters": False, - } - - -def _choose_selected_candidate( - *, - frontier: pd.DataFrame, - controls: dict[float, dict[str, Any]], - evaluated: list[dict[str, Any]], - settings: SelectionSettings, -) -> SelectionResult: - _mark_hard_filter_eligibility(evaluated, settings) - candidate_pool = _apply_ab_like_screen(_candidate_pool_after_filters(evaluated), settings) - fallback_reason = "no_economically_viable_robust_policy" - if not candidate_pool: - selected = _fallback_selected_candidate(frontier, controls) - return SelectionResult( - selected=selected, - selected_policy=cast(dict[str, Any], selected["policy"]), - selector_outcome="fallback_nonrobust", - fallback_applied=True, - fallback_reason=fallback_reason, - ) - - selected = _select_ranked_candidate(candidate_pool, settings) - selected_policy = cast(dict[str, Any], selected["policy"]) - if float(selected_policy["gamma"]) <= 0.0: - return SelectionResult( - selected=selected, - selected_policy=selected_policy, - selector_outcome="fallback_nonrobust", - fallback_applied=True, - fallback_reason=fallback_reason, - ) - return SelectionResult( - selected=selected, - selected_policy=selected_policy, - selector_outcome="robust_selected", - fallback_applied=False, - fallback_reason=None, - ) - - -def _without_internal_returns(item: dict[str, Any]) -> dict[str, Any]: - return {k: v for k, v in item.items() if not str(k).startswith("_returns_")} - - -def _build_champion_payload( - *, - settings: SelectionSettings, - selection: SelectionResult, - research_policy: dict[str, Any], - universe_path: str, - decision_scenario: str, - resolved_run_tag: str, -) -> dict[str, Any]: - selected = selection.selected - return { - "selection_stage": settings.selector_name, - "selection_universe_path": universe_path, - "decision_scenario": str(decision_scenario), - "selection_outcome": selection.selector_outcome, - "selected_policy": selection.selected_policy, - "economic_metrics": { - "diff_total_return": float(selected["diff_total_return"]), - "passed_no_regression": bool(selected["passed_no_regression"]), - "funded_ratio": float(selected["funded_ratio"]), - "total_allocated_ratio": float(selected.get("total_allocated_ratio", 1.0)), - "return_per_funded_delta": float(selected["return_per_funded_delta"]), - }, - "robustness_metrics": { - "worst_case_pd_reduction_bps": float(selected["worst_case_pd_reduction_bps"]), - "price_of_robustness_pct": float(selected["price_of_robustness_pct"]), - "allocation_similarity": float(selected["allocation_similarity"]), - "breadth_score": float(selected.get("breadth_score", 1.0)), - }, - "research_alternatives": { - "promotion_first": research_policy.get("selected_policy"), - "robustness_aware": research_policy.get("selected_policy_robustness_aware"), - "balanced_robustness": research_policy.get("selected_policy_balanced_robustness"), - "guardrail_robustness": research_policy.get("selected_policy_guardrail_robustness"), - }, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - - -def _build_status_payload( - *, - settings: SelectionSettings, - selection: SelectionResult, - inputs: DecisionInputs, - evaluated: list[dict[str, Any]], - controls: dict[float, dict[str, Any]], - universe_path: str, - decision_scenario: str, - resolved_run_tag: str, -) -> dict[str, Any]: - return { - "selector_name": settings.selector_name, - "universe_path": universe_path, - "decision_scenario": str(decision_scenario), - "decision_scenario_meta": inputs.scenario_meta, - "control_metrics": {str(k): dict(v["metrics"]) for k, v in controls.items()}, - "evaluated_candidates": [_without_internal_returns(item) for item in evaluated], - "selected_candidate": _without_internal_returns(selection.selected), - "selector_outcome": selection.selector_outcome, - "fallback_applied": selection.fallback_applied, - "fallback_reason": selection.fallback_reason, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - - -def _write_selection_outputs( - *, - champion_policy_path: str, - status_path: str, - champion_payload: dict[str, Any], - status_payload: dict[str, Any], -) -> None: - champion_out = _artifact_path(champion_policy_path) - status_out = _artifact_path(status_path) - champion_out.parent.mkdir(parents=True, exist_ok=True) - status_out.parent.mkdir(parents=True, exist_ok=True) - champion_out.write_text(json.dumps(champion_payload, indent=2), encoding="utf-8") - status_out.write_text(json.dumps(status_payload, indent=2), encoding="utf-8") - logger.info("Saved champion portfolio policy: {}", champion_out) - logger.info("Saved champion policy selection status: {}", status_out) - - -def main( - config_path: str = "configs/optimization.yaml", - frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", - research_policy_path: str = "models/portfolio_research_policy.json", - champion_policy_path: str = "models/champion_portfolio_policy.json", - status_path: str = "models/champion_policy_selection_status.json", - candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", - run_tag: str | None = None, - solver_backend: str = "highs", - decision_scenario: str = "baseline", -) -> None: - config = _load_config(config_path) - settings = _selection_settings(config) - frontier = _load_frontier(frontier_path) - inputs = _prepare_decision_inputs( - config=config, - candidate_universe_path=candidate_universe_path, - decision_scenario=decision_scenario, - ) - controls = _control_metrics_by_risk( - common=inputs.common, - default_flag=inputs.default_flag, - loan_amnt=inputs.loan_amnt, - int_rates=inputs.int_rates, - risk_values=frontier["risk_tolerance"].tolist(), - total_budget=inputs.total_budget, - solver_backend=solver_backend, - ) - - candidate_rows = _select_candidate_rows(frontier, top_k=settings.top_k) - if not candidate_rows: - raise ValueError("No eligible canonical candidates found in frontier") - - evaluated = _evaluate_candidate_rows( - candidate_rows=candidate_rows, - inputs=inputs, - controls=controls, - settings=settings, - solver_backend=solver_backend, - ) - selection = _choose_selected_candidate( - frontier=frontier, - controls=controls, - evaluated=evaluated, - settings=settings, - ) - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - research_policy = _load_json(_artifact_path(research_policy_path)) - universe_path = inputs.universe_source or str(_artifact_path(candidate_universe_path)) - champion_payload = _build_champion_payload( - settings=settings, - selection=selection, - research_policy=research_policy, - universe_path=universe_path, - decision_scenario=decision_scenario, - resolved_run_tag=resolved_run_tag, - ) - status_payload = _build_status_payload( - settings=settings, - selection=selection, - inputs=inputs, - evaluated=evaluated, - controls=controls, - universe_path=universe_path, - decision_scenario=decision_scenario, - resolved_run_tag=resolved_run_tag, - ) - _write_selection_outputs( - champion_policy_path=champion_policy_path, - status_path=status_path, - champion_payload=champion_payload, - status_payload=status_payload, - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--config", default="configs/optimization.yaml") - parser.add_argument( - "--frontier_path", default="data/processed/portfolio_robustness_frontier.parquet" - ) - parser.add_argument("--research_policy_path", default="models/portfolio_research_policy.json") - parser.add_argument("--champion_policy_path", default="models/champion_portfolio_policy.json") - parser.add_argument("--status_path", default="models/champion_policy_selection_status.json") - parser.add_argument( - "--candidate_universe_path", default="data/processed/champion_candidate_universe.parquet" - ) - parser.add_argument("--run-tag", default=None) - parser.add_argument("--solver_backend", choices=["highs", "cuopt"], default="highs") - parser.add_argument("--decision-scenario", default="baseline") - args = parser.parse_args() - main( - config_path=args.config, - frontier_path=args.frontier_path, - research_policy_path=args.research_policy_path, - champion_policy_path=args.champion_policy_path, - status_path=args.status_path, - candidate_universe_path=args.candidate_universe_path, - run_tag=args.run_tag, - solver_backend=args.solver_backend, - decision_scenario=args.decision_scenario, - ) diff --git a/scripts/simulate_ab_test.py b/scripts/simulate_ab_test.py deleted file mode 100644 index 7ed3a8f..0000000 --- a/scripts/simulate_ab_test.py +++ /dev/null @@ -1,975 +0,0 @@ -"""Simulate A/B test: robust vs non-robust portfolio on OOT test set. - -Retroactively applies two portfolio strategies to the OOT test set -and compares realized outcomes using actual default_flag as ground truth. - -Strategy A (control): non-robust portfolio (pd_point for PD constraint) -Strategy B (treatment): robust portfolio (pd_high for PD constraint) - -No-regression gate policy (paper-grade run 2026-03-13): -- baseline scenario (5K candidates): diff=-$2K, p=0.405 → no-regression PASS - (both strategies negative by construction on PD-weighted return universe) -- ambiguity_defer scenario (2.4K candidates): diff=-$13.5K → no-regression FAIL - → ambiguity_defer NOT recommended for operational use - → the FAIL is an artifact of the restricted 276K candidate universe, not a real strategy failure - → gate is diagnostic only for ambiguity_defer; baseline scenario is the promoted strategy - -Usage: - uv run python scripts/simulate_ab_test.py -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any, cast - -import numpy as np -import pandas as pd -from loguru import logger - -from src.evaluation.ab_testing import ab_summary, compare_strategies -from src.optimization.policy_evaluation import solve_policy_allocation -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.script_helpers import artifact_path as _artifact_path - -SCHEMA_VERSION = "2026-03-01.1" - - -def _default_robust_policy(max_portfolio_pd: float) -> dict[str, Any]: - return { - "source": "fallback_default", - "risk_tolerance": float(max_portfolio_pd), - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "hard_worst_case", - "gamma": 1.0, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - } - - -def _select_champion_policy(payload: dict[str, Any], policy_selector: str) -> dict[str, Any]: - if policy_selector == "robustness_aware": - return cast( - dict[str, Any], - payload.get("selected_policy_robustness_aware") or payload.get("selected_policy", {}), - ) - if policy_selector == "balanced_robustness": - return cast( - dict[str, Any], - payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}), - ) - if policy_selector == "guardrail_robustness": - return cast( - dict[str, Any], - payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}), - ) - if policy_selector == "explicit_champion_only": - selected = payload.get("selected_policy", {}) - if not selected: - raise ValueError("Champion policy artifact missing selected_policy") - return cast(dict[str, Any], selected) - return cast(dict[str, Any], payload.get("selected_policy", {})) - - -def _policy_from_selected_champion( - selected: dict[str, Any], - *, - max_portfolio_pd: float, - policy_selector: str, -) -> dict[str, Any]: - return { - "source": f"champion_policy_artifact::{policy_selector}", - "risk_tolerance": float(selected.get("risk_tolerance", max_portfolio_pd)), - "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), - "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), - "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), - "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), - "gamma": float(selected.get("gamma", 1.0)), - "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), - } - - -def _resolve_champion_robust_policy( - *, - champion_path: Path, - max_portfolio_pd: float, - policy_selector: str, -) -> dict[str, Any] | None: - if not champion_path.exists(): - if policy_selector == "explicit_champion_only": - raise FileNotFoundError(f"Missing champion portfolio policy artifact: {champion_path}") - return None - try: - payload_raw = json.loads(champion_path.read_text(encoding="utf-8")) - selected = ( - _select_champion_policy(payload_raw, policy_selector) - if isinstance(payload_raw, dict) - else {} - ) - policy = _policy_from_selected_champion( - selected, - max_portfolio_pd=max_portfolio_pd, - policy_selector=policy_selector, - ) - except Exception as exc: - if policy_selector == "explicit_champion_only": - raise - logger.warning( - f"Could not parse champion portfolio policy ({champion_path}): {exc}. " - "Falling back to summary-based policy." - ) - return None - logger.info( - "Resolved robust policy from champion artifact: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"policy_mode={policy['policy_mode']}, gamma={policy['gamma']:.2f}" - ) - return policy - - -def _load_robustness_summary(path: Path) -> pd.DataFrame | None: - if not path.exists(): - logger.warning(f"Robustness summary not found ({path}); using fallback robust policy.") - return None - try: - return pd.read_parquet(path) - except Exception as exc: - logger.warning(f"Could not read robustness summary ({path}): {exc}") - return None - - -def _valid_summary_rows(summary: pd.DataFrame, required_cols: set[str]) -> pd.DataFrame | None: - if summary.empty or not required_cols.issubset(set(summary.columns)): - missing = sorted(required_cols - set(summary.columns)) - logger.warning( - "Robustness summary missing required columns or empty; " - f"missing={missing}. Using fallback robust policy." - ) - return None - work = summary.copy() - for col in required_cols: - work[col] = pd.to_numeric(work[col], errors="coerce") - work = work.dropna(subset=list(required_cols)).reset_index(drop=True) - if work.empty: - logger.warning("No valid numeric robust summary rows; using fallback policy.") - return None - return work - - -def _best_summary_policy_row(work: pd.DataFrame, target: float) -> pd.Series: - lower_eq = work.loc[work["risk_tolerance"] <= target + 1e-12].copy() - candidate_pool = lower_eq if not lower_eq.empty else work - candidate_pool["_distance"] = (candidate_pool["risk_tolerance"] - target).abs() - if "best_robust_return" not in candidate_pool.columns: - return candidate_pool.sort_values(by=["_distance"], ascending=[True]).iloc[0] - candidate_pool["best_robust_return"] = pd.to_numeric( - candidate_pool["best_robust_return"], errors="coerce" - ).fillna(float("-inf")) - return candidate_pool.sort_values( - by=["_distance", "best_robust_return"], - ascending=[True, False], - ).iloc[0] - - -def _policy_from_summary_row(row: pd.Series) -> dict[str, Any]: - return { - "source": "portfolio_robustness_summary", - "risk_tolerance": float(row["risk_tolerance"]), - "uncertainty_aversion": float(row["best_robust_lambda"]), - "min_budget_utilization": float(row["best_robust_min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["best_robust_pd_cap_slack_penalty"]), - "policy_mode": str(row.get("best_robust_policy_mode", "hard_worst_case")), - "gamma": float(row.get("best_robust_gamma", 1.0)), - "delta_cap_quantile": float(row.get("best_robust_delta_cap_quantile", 1.0)), - } - - -def _resolve_summary_robust_policy( - *, - path: Path, - max_portfolio_pd: float, -) -> dict[str, Any] | None: - summary = _load_robustness_summary(path) - if summary is None: - return None - - required_cols = { - "risk_tolerance", - "best_robust_lambda", - "best_robust_min_budget_utilization", - "best_robust_pd_cap_slack_penalty", - } - work = _valid_summary_rows(summary, required_cols) - if work is None: - return None - - policy = _policy_from_summary_row( - _best_summary_policy_row(work, target=float(max_portfolio_pd)) - ) - logger.info( - "Resolved robust policy from summary: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"uncertainty_aversion={policy['uncertainty_aversion']:.4f}, " - f"min_budget_utilization={policy['min_budget_utilization']:.4f}, " - f"pd_cap_slack_penalty={policy['pd_cap_slack_penalty']:.4f}" - ) - return policy - - -def _compute_realized_return( - allocation: dict[int, float], - loan_amnt: np.ndarray, - int_rates: np.ndarray, - default_flag: np.ndarray, - lgd: float = 0.45, -) -> np.ndarray: - """Compute per-loan realized return given actual defaults. - - For funded loans: return = alloc * loan_amnt * (rate*(1-default) - default*lgd) - For unfunded loans: return = 0 - """ - n = len(loan_amnt) - returns = np.zeros(n) - for i in range(n): - alloc = allocation.get(i, 0.0) - if alloc > 0.01: - if default_flag[i] == 1: - returns[i] = alloc * loan_amnt[i] * (-lgd) - else: - returns[i] = alloc * loan_amnt[i] * int_rates[i] - return returns - - -def _parse_percent_series(s: pd.Series, default: float = 0.12) -> np.ndarray: - """Convert percent column to decimal.""" - if pd.api.types.is_numeric_dtype(s): - arr = s.to_numpy(dtype=float) - if np.nanmedian(arr) > 1: - arr = arr / 100.0 - return np.nan_to_num(arr, nan=default) - return ( - s.astype(str) - .str.strip() - .str.rstrip("%") - .pipe(pd.to_numeric, errors="coerce") - .div(100) - .fillna(default) - .to_numpy(dtype=float) - ) - - -def _resolve_robust_policy( - *, - max_portfolio_pd: float, - policy_selector: str = "promotion_first", - summary_path: str = "data/processed/portfolio_robustness_summary.parquet", - champion_policy_path: str = "models/champion_portfolio_policy.json", -) -> dict[str, Any]: - """Resolve robust strategy parameters from tradeoff summary, with fallback defaults.""" - champion_path = _artifact_path(champion_policy_path) - champion_policy = _resolve_champion_robust_policy( - champion_path=champion_path, - max_portfolio_pd=max_portfolio_pd, - policy_selector=policy_selector, - ) - if champion_policy is not None: - return champion_policy - - summary_policy = _resolve_summary_robust_policy( - path=_artifact_path(summary_path), - max_portfolio_pd=max_portfolio_pd, - ) - if summary_policy is not None: - return summary_policy - return _default_robust_policy(max_portfolio_pd) - - -def _apply_candidate_universe( - test_df: pd.DataFrame, - intervals: pd.DataFrame, - *, - candidate_universe_path: str, - max_candidates: int, -) -> tuple[pd.DataFrame, pd.DataFrame, str]: - path = _artifact_path(candidate_universe_path) - max_candidates_norm = None if int(max_candidates) <= 0 else int(max_candidates) - if path.exists() and "id" in test_df.columns and "id" in intervals.columns: - universe = pd.read_parquet(path) - if "id" in universe.columns and not universe.empty: - ordered_ids = universe["id"].astype(str) - if max_candidates_norm is not None: - ordered_ids = ordered_ids.iloc[:max_candidates_norm] - order_df = pd.DataFrame( - { - "_id_join": ordered_ids.values, - "_sample_order": np.arange(len(ordered_ids), dtype=int), - } - ) - test_work = test_df.copy() - ints_work = intervals.copy() - test_work["_id_join"] = test_work["id"].astype(str) - ints_work["_id_join"] = ints_work["id"].astype(str) - test_work = test_work.merge(order_df, on="_id_join", how="inner") - ints_work = ints_work.merge(order_df, on="_id_join", how="inner") - test_work = test_work.sort_values("_sample_order").drop_duplicates("_id_join") - ints_work = ints_work.sort_values("_sample_order").drop_duplicates("_id_join") - merged_n = min(len(test_work), len(ints_work)) - test_out = test_work.iloc[:merged_n].drop(columns=["_id_join", "_sample_order"]) - ints_out = ints_work.iloc[:merged_n].drop(columns=["_id_join", "_sample_order"]) - if merged_n > 0: - logger.info( - "Using champion candidate universe from {} with n={}", - path, - merged_n, - ) - return test_out.reset_index(drop=True), ints_out.reset_index(drop=True), str(path) - - n = min(len(test_df), len(intervals)) - if max_candidates_norm is not None: - n = min(n, max_candidates_norm) - logger.info( - "Using positional candidate cohort with n={} (no shared universe artifact).", - n, - ) - return ( - test_df.iloc[:n].reset_index(drop=True), - intervals.iloc[:n].reset_index(drop=True), - "", - ) - - -def _apply_decision_scenario( - test_df: pd.DataFrame, - intervals: pd.DataFrame, - *, - decision_scenario: str, - set_prediction_path: str = "data/processed/pd_set_prediction_cases.parquet", -) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, object]]: - scenario = str(decision_scenario).strip().lower() - if scenario in {"baseline", "none", "standard"}: - return ( - test_df.reset_index(drop=True), - intervals.reset_index(drop=True), - { - "decision_scenario": "baseline", - "rows_removed": 0, - "rows_remaining": int(min(len(test_df), len(intervals))), - "ambiguity_rate_removed": 0.0, - }, - ) - - if scenario not in {"ambiguity_defer", "selective_ambiguity_defer"}: - raise ValueError(f"Unsupported decision scenario: {decision_scenario}") - - path = _artifact_path(set_prediction_path) - if not path.exists(): - raise FileNotFoundError( - f"Decision scenario '{decision_scenario}' requires set prediction artifact: {path}" - ) - cases = pd.read_parquet(path) - cases = cases.copy() - if "ambiguous" not in cases.columns: - raise KeyError("Expected 'ambiguous' column in pd_set_prediction_cases artifact.") - - # Build defer mask depending on scenario - if scenario == "selective_ambiguity_defer": - # Selective: only defer ambiguous loans that are in LOW-ambiguity grades - # (where ambiguity IS informative) OR have very high conformal uncertainty. - LOW_AMBIGUITY_GRADES = {"A", "F", "G"} # grades with ambiguity_rate < 20% - is_ambiguous = cases["ambiguous"].astype(int) == 1 - in_low_amb_grade = ( - cases["grade"].astype(str).isin(LOW_AMBIGUITY_GRADES) - if "grade" in cases.columns - else pd.Series(False, index=cases.index) - ) - # Join width_90 from intervals if available - has_high_uncertainty = pd.Series(False, index=cases.index) - width_col = next((c for c in ["width_90"] if c in intervals.columns), None) - if width_col is not None and len(intervals) >= len(cases): - width_vals = intervals[width_col].iloc[: len(cases)].reset_index(drop=True) - p90_threshold = float(width_vals.quantile(0.90)) - has_high_uncertainty = width_vals > p90_threshold - defer_mask = is_ambiguous & (in_low_amb_grade | has_high_uncertainty) - keep_mask_cases = ~defer_mask - logger.info( - "Selective defer: {} deferred ({:.1%}) from {} ambiguous ({:.1%})", - int(defer_mask.sum()), - float(defer_mask.mean()), - int(is_ambiguous.sum()), - float(is_ambiguous.mean()), - ) - else: - # Original: defer ALL ambiguous - keep_mask_cases = cases["ambiguous"].astype(int) == 0 - - if "id" in test_df.columns and "id" in cases.columns: - eligible_ids = set(cases.loc[keep_mask_cases, "id"].astype(str)) - test_work = test_df.copy() - int_work = intervals.copy() - test_work["_join_id"] = test_work["id"].astype(str) - if "id" in int_work.columns: - int_work["_join_id"] = int_work["id"].astype(str) - else: - int_work["_join_id"] = test_work["_join_id"].iloc[: len(int_work)].to_numpy() - keep_mask_test = test_work["_join_id"].isin(eligible_ids) - keep_mask_int = int_work["_join_id"].isin(eligible_ids) - test_out = test_work.loc[keep_mask_test].drop(columns=["_join_id"]).reset_index(drop=True) - ints_out = int_work.loc[keep_mask_int].drop(columns=["_join_id"]).reset_index(drop=True) - else: - n = min(len(test_df), len(intervals), len(cases)) - keep_mask = keep_mask_cases.iloc[:n].to_numpy() - test_out = test_df.iloc[:n].loc[keep_mask].reset_index(drop=True) - ints_out = intervals.iloc[:n].loc[keep_mask].reset_index(drop=True) - - rows_initial = int(min(len(test_df), len(intervals))) - rows_remaining = int(min(len(test_out), len(ints_out))) - rows_removed = max(rows_initial - rows_remaining, 0) - ambiguity_rate_removed = float(rows_removed / rows_initial) if rows_initial else 0.0 - logger.info( - "Applied decision scenario '{}': removed {} rows, remaining={}", - scenario, - rows_removed, - rows_remaining, - ) - return ( - test_out, - ints_out, - { - "decision_scenario": scenario, - "rows_removed": rows_removed, - "rows_remaining": rows_remaining, - "ambiguity_rate_removed": ambiguity_rate_removed, - "set_prediction_path": str(path), - }, - ) - - -def _build_common_inputs( - test_df: pd.DataFrame, - intervals: pd.DataFrame, -) -> tuple[dict[str, Any], np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - n = min(len(test_df), len(intervals)) - pd_col = next( - (c for c in ["pd_calibrated", "y_pred"] if c in intervals.columns), intervals.columns[0] - ) - low_col = next((c for c in ["pd_low", "pd_low_90"] if c in intervals.columns), None) - high_col = next((c for c in ["pd_high", "pd_high_90"] if c in intervals.columns), None) - pd_point = intervals[pd_col].values - pd_low = intervals[low_col].values if low_col else pd_point * 0.8 - pd_high = intervals[high_col].values if high_col else pd_point * 1.3 - lgd_val = 0.45 - lgd = np.full(n, lgd_val) - int_rates = ( - _parse_percent_series(test_df["int_rate"]) - if "int_rate" in test_df.columns - else np.full(n, 0.12) - ) - default_flag = ( - test_df["default_flag"].values if "default_flag" in test_df.columns else np.zeros(n) - ) - loan_amnt = ( - test_df["loan_amnt"].values if "loan_amnt" in test_df.columns else np.full(n, 10000.0) - ) - return ( - { - "loans": test_df, - "pd_point": pd_point, - "pd_low": pd_low, - "pd_high": pd_high, - "lgd": lgd, - "int_rates": int_rates, - }, - default_flag, - loan_amnt, - int_rates, - pd_high, - ) - - -def _run_strategy( - *, - common: dict[str, Any], - robust: bool, - total_budget: float, - max_portfolio_pd: float, - solver_backend: str, - robust_policy: dict[str, Any] | None = None, -) -> tuple[dict, np.ndarray]: - pd_point = np.asarray(common["pd_point"], dtype=float) - policy = robust_policy or {} - result = solve_policy_allocation( - loans=cast(pd.DataFrame, common["loans"]), - pd_point=pd_point, - pd_low=np.asarray(common["pd_low"], dtype=float), - pd_high=np.asarray(common["pd_high"], dtype=float), - lgd=np.asarray(common["lgd"], dtype=float), - int_rates=np.asarray(common["int_rates"], dtype=float), - total_budget=total_budget, - risk_tolerance=max_portfolio_pd, - robust=robust, - uncertainty_aversion=float(policy.get("uncertainty_aversion", 0.0)), - min_budget_utilization=float(policy.get("min_budget_utilization", 0.0)), - pd_cap_slack_penalty=float(policy.get("pd_cap_slack_penalty", 0.0)), - policy_mode=str(policy.get("policy_mode", "hard_worst_case")), - gamma=float(policy.get("gamma", 1.0)), - delta_cap_quantile=float(policy.get("delta_cap_quantile", 1.0)), - tail_focus_quantile=float(policy.get("tail_focus_quantile", 1.0)), - solver_backend=solver_backend, - ) - return result.solution, result.effective_pd - - -def _candidate_metrics( - *, - solution: dict, - loan_amnt: np.ndarray, - int_rates: np.ndarray, - default_flag: np.ndarray, - lgd_val: float, -) -> tuple[np.ndarray, dict[str, float | int]]: - returns = _compute_realized_return( - solution["allocation"], loan_amnt, int_rates, default_flag, lgd_val - ) - metrics = { - "total_return": float(returns.sum()), - "n_funded": int(solution["n_funded"]), - "total_allocated": float(solution["total_allocated"]), - "avg_return_per_funded": float(returns[returns != 0].mean()) - if (returns != 0).any() - else 0.0, - } - return returns, metrics - - -def _load_frontier_policy_candidates( - *, - frontier_path: str, - max_portfolio_pd: float, - top_k: int, -) -> list[dict[str, float | str]]: - path = _artifact_path(frontier_path) - if not path.exists(): - return [] - frontier = pd.read_parquet(path) - if frontier.empty: - return [] - work = frontier.loc[(frontier["policy"] != "nonrobust") & (frontier["gamma"] > 0)].copy() - work["risk_gap"] = ( - pd.to_numeric(work["risk_tolerance"], errors="coerce") - max_portfolio_pd - ).abs() - work["ret_rank"] = pd.to_numeric(work["realized_total_return"], errors="coerce").fillna( - float("-inf") - ) - work["por_rank"] = pd.to_numeric(work["price_of_robustness_pct"], errors="coerce").fillna( - -999.0 - ) - work["ab_pass_rank"] = work["ab_pass"].fillna(False).astype(bool) - work = work.sort_values( - ["ab_pass_rank", "risk_gap", "ret_rank", "por_rank", "gamma", "uncertainty_aversion"], - ascending=[False, True, False, False, False, True], - ) - candidates: list[dict[str, float | str]] = [] - for _, row in work.head(int(top_k)).iterrows(): - candidates.append( - { - "source": "frontier_actual_ab_search", - "risk_tolerance": float(row["risk_tolerance"]), - "uncertainty_aversion": float(row["uncertainty_aversion"]), - "min_budget_utilization": float(row["min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["pd_cap_slack_penalty"]), - "policy_mode": str(row["policy_mode"]), - "gamma": float(row["gamma"]), - "delta_cap_quantile": float(row.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(row.get("tail_focus_quantile", 1.0)), - } - ) - return candidates - - -def main( - total_budget: float = 1_000_000, - max_portfolio_pd: float = 0.10, - max_candidates: int = 5_000, - n_boot: int = 1000, - seed: int = 42, - no_regression_tolerance_pct: float = 0.05, - robust_policy_summary_path: str = "data/processed/portfolio_robustness_summary.parquet", - champion_policy_path: str = "models/champion_portfolio_policy.json", - candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", - results_path: str = "data/processed/ab_simulation_results.parquet", - summary_path: str = "data/processed/ab_simulation_summary.parquet", - status_path: str = "models/ab_simulation_status.json", - run_tag: str | None = None, - solver_backend: str = "highs", - policy_selector: str = "promotion_first", - frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", - actual_ab_top_k: int = 12, - decision_scenario: str = "baseline", -) -> None: - """Run the A/B simulation.""" - data_dir = Path("data/processed") - test_path = data_dir / "test_fe.parquet" - intervals_path = data_dir / "conformal_intervals_mondrian.parquet" - - for p in [test_path, intervals_path]: - if not p.exists(): - raise FileNotFoundError(f"Missing: {p}") - - test_df = pd.read_parquet(test_path) - intervals = pd.read_parquet(intervals_path) - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - - max_candidates_norm = None if int(max_candidates) <= 0 else int(max_candidates) - test_df, intervals, universe_source = _apply_candidate_universe( - test_df, - intervals, - candidate_universe_path=candidate_universe_path, - max_candidates=max_candidates, - ) - test_df, intervals, scenario_meta = _apply_decision_scenario( - test_df, - intervals, - decision_scenario=decision_scenario, - ) - n = min(len(test_df), len(intervals)) - logger.info( - f"Using {n} candidates " - f"(max_candidates={'full' if max_candidates_norm is None else max_candidates_norm})" - ) - - common, default_flag, loan_amnt, int_rates, _ = _build_common_inputs(test_df, intervals) - lgd_val = 0.45 - - robust_policy = _resolve_robust_policy( - max_portfolio_pd=float(max_portfolio_pd), - policy_selector=str(policy_selector), - summary_path=str(robust_policy_summary_path), - champion_policy_path=str(champion_policy_path), - ) - effective_max_portfolio_pd = float(robust_policy.get("risk_tolerance", max_portfolio_pd)) - - # Strategy A: non-robust - logger.info("Strategy A (control): non-robust portfolio") - sol_a, _ = _run_strategy( - common=common, - robust=False, - total_budget=total_budget, - max_portfolio_pd=effective_max_portfolio_pd, - solver_backend=solver_backend, - ) - returns_a, metrics_a = _candidate_metrics( - solution=sol_a, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=lgd_val, - ) - - # Strategy B: robust - logger.info("Strategy B (treatment): robust portfolio") - policy_search: list[dict[str, object]] = [] - if policy_selector == "actual_ab_guarded": - search_candidates = _load_frontier_policy_candidates( - frontier_path=frontier_path, - max_portfolio_pd=float(effective_max_portfolio_pd), - top_k=int(actual_ab_top_k), - ) - chosen_policy: dict[str, Any] | None = None - chosen_sol: dict[str, Any] | None = None - returns_b: np.ndarray | None = None - metrics_b: dict[str, float | int] | None = None - no_regression_result: dict[str, Any] | None = None - for idx, candidate in enumerate(search_candidates, start=1): - sol_candidate, _ = _run_strategy( - common=common, - robust=True, - robust_policy=candidate, - total_budget=total_budget, - max_portfolio_pd=float(candidate["risk_tolerance"]), - solver_backend=solver_backend, - ) - cand_returns, cand_metrics = _candidate_metrics( - solution=sol_candidate, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=lgd_val, - ) - diff_total_return = float(cand_metrics["total_return"] - metrics_a["total_return"]) - tolerance_total_return = abs(float(metrics_a["total_return"])) * float( - no_regression_tolerance_pct - ) - passed = bool(diff_total_return >= -tolerance_total_return) - policy_search.append( - { - "rank": idx, - "policy": candidate, - "metrics_b": cand_metrics, - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "passed": passed, - } - ) - if passed: - chosen_policy = candidate - chosen_sol = sol_candidate - returns_b = cand_returns - metrics_b = cand_metrics - no_regression_result = { - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "tolerance_pct_of_control": float(no_regression_tolerance_pct), - "passed": True, - "selected_from_search_rank": float(idx), - } - logger.info( - "actual_ab_guarded selected robust policy at rank {}: gamma={} lambda={}", - idx, - candidate["gamma"], - candidate["uncertainty_aversion"], - ) - break - if chosen_policy is None: - logger.warning( - "No robust policy passed actual A/B guardrail. Falling back to nonrobust-equivalent champion." - ) - chosen_policy = { - "source": "actual_ab_guarded_fallback_nonrobust", - "risk_tolerance": effective_max_portfolio_pd, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "blended_uncertainty", - "gamma": 0.0, - "delta_cap_quantile": 1.0, - } - chosen_sol, _ = _run_strategy( - common=common, - robust=True, - robust_policy=chosen_policy, - total_budget=total_budget, - max_portfolio_pd=float(chosen_policy["risk_tolerance"]), - solver_backend=solver_backend, - ) - returns_b, metrics_b = _candidate_metrics( - solution=chosen_sol, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=lgd_val, - ) - diff_total_return = float(metrics_b["total_return"] - metrics_a["total_return"]) - tolerance_total_return = abs(float(metrics_a["total_return"])) * float( - no_regression_tolerance_pct - ) - no_regression_result = { - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "tolerance_pct_of_control": float(no_regression_tolerance_pct), - "passed": bool(diff_total_return >= -tolerance_total_return), - "selected_from_search_rank": None, - "fallback_nonrobust": True, - } - robust_policy = chosen_policy - assert chosen_sol is not None - assert returns_b is not None - assert metrics_b is not None - assert no_regression_result is not None - sol_b = chosen_sol - else: - sol_b, _ = _run_strategy( - common=common, - robust=True, - robust_policy=robust_policy, - total_budget=total_budget, - max_portfolio_pd=effective_max_portfolio_pd, - solver_backend=solver_backend, - ) - returns_b, metrics_b = _candidate_metrics( - solution=sol_b, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=lgd_val, - ) - diff_total_return = float(metrics_b["total_return"] - metrics_a["total_return"]) - tolerance_total_return = abs(float(metrics_a["total_return"])) * float( - no_regression_tolerance_pct - ) - no_regression_result = { - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "tolerance_pct_of_control": float(no_regression_tolerance_pct), - "passed": bool(diff_total_return >= -tolerance_total_return), - } - - comparison = compare_strategies( - returns_a, returns_b, method="bootstrap", n_boot=n_boot, seed=seed - ) - - summary = ab_summary( - {key: float(value) for key, value in metrics_a.items()}, - {key: float(value) for key, value in metrics_b.items()}, - ) - - # Save results - results_df = pd.DataFrame( - [ - { - "strategy_a_return": metrics_a["total_return"], - "strategy_b_return": metrics_b["total_return"], - "diff": comparison["diff"], - "ci_low": comparison["ci_low"], - "ci_high": comparison["ci_high"], - "p_value": comparison["p_value"], - "significant": comparison["significant"], - "n_funded_a": sol_a["n_funded"], - "n_funded_b": sol_b["n_funded"], - } - ] - ) - results_out = _artifact_path(results_path) - results_out.parent.mkdir(parents=True, exist_ok=True) - results_df.to_parquet(results_out, index=False) - logger.info(f"Saved results: {results_out}") - - summary_out = _artifact_path(summary_path) - summary_out.parent.mkdir(parents=True, exist_ok=True) - summary.to_parquet(summary_out, index=False) - - status: dict[str, Any] = { - "strategy_a": "non_robust", - "strategy_b": "robust_selected_for_champion", - "comparison": comparison, - "metrics_a": metrics_a, - "metrics_b": metrics_b, - "n_candidates_available": int(min(len(test_df), len(intervals))), - "n_candidates_used": int(n), - "max_candidates_requested": None if max_candidates_norm is None else max_candidates_norm, - "dataset_scope": "full_candidates" if max_candidates_norm is None else "sampled_candidates", - "solver_backend": str(solver_backend), - "policy_selector": str(policy_selector), - "decision_scenario": str(decision_scenario), - "max_portfolio_pd_requested": float(max_portfolio_pd), - "max_portfolio_pd_effective": float(effective_max_portfolio_pd), - "robust_policy": robust_policy, - "champion_policy_path": str(champion_policy_path), - "candidate_universe_path": universe_source or str(candidate_universe_path), - "gate_contract": { - "gate": "no_regression", - "significance_role": "diagnostic", - }, - "diagnostics": { - "p_value": float(comparison["p_value"]), - "significant": bool(comparison["significant"]), - "n_boot": int(n_boot), - "seed": int(seed), - }, - } - status["policy_search"] = policy_search - status["frontier_path"] = str(_artifact_path(frontier_path)) - status["no_regression"] = no_regression_result - status["decision_scenario_meta"] = scenario_meta - status["baseline_comparison_context"] = { - "artifact_truth_role": "current_run_status", - "official_truth_may_live_in_comparison_json": True, - } - status.update( - build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ) - ) - status_out = _artifact_path(status_path) - status_out.parent.mkdir(parents=True, exist_ok=True) - with open(status_out, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2, default=str) - logger.info(f"Saved status: {status_out}") - - logger.info( - f"A/B result: A(non-robust)={metrics_a['total_return']:,.2f}, " - f"B(robust)={metrics_b['total_return']:,.2f}, " - f"diff={comparison['diff']:,.2f}, p={comparison['p_value']:.4f}" - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="A/B simulation: robust vs non-robust") - parser.add_argument("--total_budget", type=float, default=1_000_000) - parser.add_argument("--max_portfolio_pd", type=float, default=0.10) - parser.add_argument("--max_candidates", type=int, default=5_000) - parser.add_argument("--n_boot", type=int, default=1000) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--no_regression_tolerance_pct", type=float, default=0.05) - parser.add_argument( - "--robust_policy_summary_path", - default="data/processed/portfolio_robustness_summary.parquet", - ) - parser.add_argument( - "--champion_policy_path", - default="models/champion_portfolio_policy.json", - ) - parser.add_argument( - "--candidate_universe_path", - default="data/processed/champion_candidate_universe.parquet", - ) - parser.add_argument("--results_path", default="data/processed/ab_simulation_results.parquet") - parser.add_argument("--summary_path", default="data/processed/ab_simulation_summary.parquet") - parser.add_argument("--status_path", default="models/ab_simulation_status.json") - parser.add_argument("--run-tag", default=None) - parser.add_argument("--solver_backend", choices=["highs", "cuopt"], default="highs") - parser.add_argument( - "--frontier_path", - default="data/processed/portfolio_robustness_frontier.parquet", - ) - parser.add_argument("--actual_ab_top_k", type=int, default=12) - parser.add_argument( - "--policy_selector", - choices=[ - "promotion_first", - "robustness_aware", - "balanced_robustness", - "guardrail_robustness", - "actual_ab_guarded", - "explicit_champion_only", - ], - default="promotion_first", - ) - parser.add_argument( - "--decision-scenario", - default="baseline", - choices=["baseline", "ambiguity_defer", "selective_ambiguity_defer"], - ) - args = parser.parse_args() - main( - total_budget=args.total_budget, - max_portfolio_pd=args.max_portfolio_pd, - max_candidates=args.max_candidates, - n_boot=args.n_boot, - seed=args.seed, - no_regression_tolerance_pct=args.no_regression_tolerance_pct, - robust_policy_summary_path=args.robust_policy_summary_path, - champion_policy_path=args.champion_policy_path, - candidate_universe_path=args.candidate_universe_path, - results_path=args.results_path, - summary_path=args.summary_path, - status_path=args.status_path, - run_tag=args.run_tag, - solver_backend=args.solver_backend, - policy_selector=args.policy_selector, - frontier_path=args.frontier_path, - actual_ab_top_k=args.actual_ab_top_k, - decision_scenario=args.decision_scenario, - ) diff --git a/scripts/train_pd_model.py b/scripts/train_pd_model.py index 8dc23ae..5aed508 100644 --- a/scripts/train_pd_model.py +++ b/scripts/train_pd_model.py @@ -2078,7 +2078,6 @@ def _train_hpo_catboost( n_startup_trials=int(hpo_cfg.get("n_startup_trials", 40)), multivariate_tpe=bool(hpo_cfg.get("multivariate_tpe", True)), group_tpe=bool(hpo_cfg.get("group_tpe", True)), - warn_independent_sampling=bool(hpo_cfg.get("warn_independent_sampling", True)), constant_liar=bool(hpo_cfg.get("constant_liar", False)), pruner_n_startup_trials=int(hpo_cfg.get("pruner_n_startup_trials", 20)), pruner_n_warmup_steps=int(hpo_cfg.get("pruner_n_warmup_steps", 50)), diff --git a/scripts/validate_alpha_gamma_bound.py b/scripts/validate_alpha_gamma_bound.py deleted file mode 100644 index 4f93269..0000000 --- a/scripts/validate_alpha_gamma_bound.py +++ /dev/null @@ -1,565 +0,0 @@ -"""Validate the α-CP → Γ-robustness bound (Theorem 1, CRPTO).""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Any - -import matplotlib -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from loguru import logger - -ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT)) - -from scripts.optimize_portfolio_tradeoff import ( # noqa: E402 - _align_loans_and_intervals, - _load_candidates, - _parse_percent_series, -) -from src.models.conformal_artifacts import load_conformal_intervals # noqa: E402 -from src.optimization.certificate_semantics import ( # noqa: E402 - compute_funded_certificate_metrics, -) -from src.optimization.policy import policy_segment_labels # noqa: E402 -from src.optimization.portfolio_model import ( # noqa: E402 - compute_effective_pd, - optimize_portfolio_allocation, - solution_allocation_vector, -) - -DEFAULT_ALPHAS = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] -DEFAULT_MAX_CANDIDATES = 5_000 -DEFAULT_RANDOM_STATE = 42 -DEFAULT_T_EVAL = 0.05 -DEFAULT_BUDGET = 1_000_000.0 -DEFAULT_MAX_CONCENTRATION = 0.25 -DEFAULT_LGD = 0.45 -DEFAULT_TIME_LIMIT = 300 -DEFAULT_THREADS = 4 -DEFAULT_SOLVER = "highs" -FIG_DIR = ROOT / "reports" / "crpto" / "figures" -DEFAULT_FIG_PREFIX = FIG_DIR / "crpto_fig_alpha_gamma_bound" -DEFAULT_EXACT_JSON = ROOT / "data" / "processed" / "alpha_gamma_bound_validation_exact.json" -DEFAULT_PROXY_JSON = ROOT / "data" / "processed" / "alpha_gamma_bound_validation.json" -DEFAULT_COMPARISON_PATH = ROOT / "data" / "processed" / "alpha_gamma_bound_proxy_vs_exact.parquet" - - -def _coerce_alpha_grid(raw: str | None) -> list[float]: - if not raw: - return list(DEFAULT_ALPHAS) - values = [] - for token in str(raw).split(","): - token = token.strip() - if not token: - continue - values.append(float(token)) - if not values: - raise ValueError("alpha-grid cannot be empty") - return values - - -def _load_policy(policy_path: str | Path) -> dict[str, Any]: - payload = json.loads(Path(policy_path).read_text(encoding="utf-8")) - selected = payload.get("selected_policy", payload) - return { - "source_path": str(policy_path), - "risk_tolerance": float(selected.get("risk_tolerance", 0.10)), - "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), - "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), - "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), - "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), - "gamma": float(selected.get("gamma", 1.0)), - "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), - "solver_backend": str(selected.get("solver_backend", DEFAULT_SOLVER)), - } - - -def _resolve_interval_columns(intervals: pd.DataFrame) -> tuple[str, str, str]: - col_point = "y_pred" if "y_pred" in intervals.columns else "pd_point" - col_low = "pd_low_90" if "pd_low_90" in intervals.columns else "pd_low" - col_high = "pd_high_90" if "pd_high_90" in intervals.columns else "pd_high" - return col_point, col_low, col_high - - -def _load_aligned_dataset( - *, - conformal_intervals_path: str | None, - max_candidates: int, - random_state: int, -) -> pd.DataFrame: - candidates = _load_candidates().reset_index(drop=True) - intervals, path, is_legacy = load_conformal_intervals( - override_path=conformal_intervals_path, - ) - logger.info( - "Loaded conformal intervals from {} (legacy={}, rows={:,})", - path, - is_legacy, - len(intervals), - ) - loans, ints = _align_loans_and_intervals( - candidates=candidates, - intervals=intervals.reset_index(drop=True), - max_candidates=max_candidates, - random_state=random_state, - ) - aligned = loans.reset_index(drop=True).copy() - for column in ints.columns: - aligned[column] = ints[column].reset_index(drop=True) - logger.info( - "Aligned loans and intervals for bound validation: n={:,}, override={}", - len(aligned), - str(conformal_intervals_path or path), - ) - return aligned - - -def _compute_intervals_at_alpha( - frame: pd.DataFrame, - alpha: float, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - col_point, col_low, col_high = _resolve_interval_columns(frame) - pd_point = pd.to_numeric(frame[col_point], errors="coerce").to_numpy(dtype=float) - pd_low_90 = pd.to_numeric(frame[col_low], errors="coerce").to_numpy(dtype=float) - pd_high_90 = pd.to_numeric(frame[col_high], errors="coerce").to_numpy(dtype=float) - radius_90 = (pd_high_90 - pd_low_90) / 2.0 - - sweep_path = ROOT / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" - if sweep_path.exists(): - sweep = pd.read_parquet(sweep_path) - row_base = sweep[np.isclose(sweep["alpha"], 0.10)] - row_target = sweep[np.isclose(sweep["alpha"], alpha)] - if len(row_base) > 0 and len(row_target) > 0: - w_base = float(row_base["avg_width"].values[0]) - w_target = float(row_target["avg_width"].values[0]) - scale = w_target / max(w_base, 1e-8) - radius = radius_90 * scale - pd_high = np.clip(pd_point + radius, 0, 1) - pd_low = np.clip(pd_point - radius, 0, 1) - return pd_point, pd_low, pd_high - return pd_point, pd_low_90, pd_high_90 - - -def _compute_effective_pd_vector( - loans: pd.DataFrame, - pd_point: np.ndarray, - pd_high: np.ndarray, - policy: dict[str, Any], -) -> np.ndarray: - return compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=str(policy["policy_mode"]), - gamma=float(policy["gamma"]), - delta_cap_quantile=float(policy["delta_cap_quantile"]), - tail_focus_quantile=float(policy["tail_focus_quantile"]), - segment_labels=policy_segment_labels(loans, str(policy["policy_mode"])), - ) - - -def _compute_proxy_weights( - loans: pd.DataFrame, - *, - pd_point: np.ndarray, - pd_high: np.ndarray, - effective_pd: np.ndarray, - policy: dict[str, Any], - budget: float, -) -> tuple[np.ndarray, dict[str, Any]]: - int_rates = ( - _parse_percent_series(loans["int_rate"]) - if "int_rate" in loans.columns - else np.full(len(loans), 0.12) - ) - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(len(loans), dtype=float) - ) - lgd = np.full(len(loans), DEFAULT_LGD, dtype=float) - net_return = int_rates - effective_pd * lgd - order = np.argsort(-net_return) - alloc = np.zeros(len(loans), dtype=float) - cum_exposure = 0.0 - cum_weighted_pd = 0.0 - - for j in order: - new_exposure = cum_exposure + loan_amounts[j] - new_weighted_pd = cum_weighted_pd + loan_amounts[j] * effective_pd[j] - if new_exposure > budget: - break - if new_weighted_pd / max(new_exposure, 1e-6) > float(policy["risk_tolerance"]): - continue - alloc[j] = 1.0 - cum_exposure = new_exposure - cum_weighted_pd = new_weighted_pd - - if cum_exposure < 1e-6: - top_n = min(100, len(order)) - alloc[order[:top_n]] = 1.0 - cum_exposure = float(np.sum(alloc * loan_amounts)) - - weights = (alloc * loan_amounts) / max(cum_exposure, 1e-6) - return weights, { - "solver_status": "proxy_greedy", - "total_allocated": float(cum_exposure), - "n_funded": int(np.sum(weights > 1e-8)), - "weighted_pd_constraint_used": float(np.sum(weights * effective_pd)), - "weighted_pd_high": float(np.sum(weights * pd_high)), - "weighted_pd_point": float(np.sum(weights * pd_point)), - } - - -def _compute_exact_weights( - loans: pd.DataFrame, - *, - pd_point: np.ndarray, - pd_low: np.ndarray, - pd_high: np.ndarray, - effective_pd: np.ndarray, - policy: dict[str, Any], - budget: float, - threads: int = DEFAULT_THREADS, -) -> tuple[np.ndarray, dict[str, Any]]: - int_rates = ( - _parse_percent_series(loans["int_rate"]) - if "int_rate" in loans.columns - else np.full(len(loans), 0.12) - ) - lgd = np.full(len(loans), DEFAULT_LGD, dtype=float) - solution = optimize_portfolio_allocation( - loans=loans, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - lgd=lgd, - int_rates=int_rates, - total_budget=budget, - max_concentration=DEFAULT_MAX_CONCENTRATION, - max_portfolio_pd=float(policy["risk_tolerance"]), - robust=True, - uncertainty_aversion=float(policy["uncertainty_aversion"]), - min_budget_utilization=float(policy["min_budget_utilization"]), - pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), - pd_constraint_override=effective_pd, - time_limit=DEFAULT_TIME_LIMIT, - threads=max(1, int(threads)), - solver_backend=str(policy["solver_backend"]), - ) - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(len(loans), dtype=float) - ) - alloc = solution_allocation_vector(solution, len(loans)) - total_allocated = float(np.sum(alloc * loan_amounts)) - weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) - return weights, { - "solver_status": str(solution.get("solver_status", "unknown")), - "solver_backend": str(solution.get("solver_backend", policy["solver_backend"])), - "native_solver_error": str(solution.get("native_solver_error", "")), - "total_allocated": total_allocated, - "n_funded": int(solution.get("n_funded", int(np.sum(weights > 1e-8)))), - "weighted_pd_constraint_used": float(np.sum(weights * effective_pd)), - "weighted_pd_high": float(np.sum(weights * pd_high)), - "weighted_pd_point": float(np.sum(weights * pd_point)), - "pd_cap_slack": float(solution.get("pd_cap_slack", 0.0)), - } - - -def _validate_single_alpha( - aligned: pd.DataFrame, - *, - alpha: float, - policy: dict[str, Any], - allocator_mode: str, - budget: float, - t_eval: float, - threads: int = DEFAULT_THREADS, -) -> dict[str, Any]: - pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) - y_true = ( - pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0).to_numpy(dtype=float) - if "y_true" in aligned.columns - else pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=float) - ) - effective_pd = _compute_effective_pd_vector(aligned, pd_point, pd_high, policy) - mode = str(allocator_mode).strip().lower() - if mode == "exact": - weights, alloc_meta = _compute_exact_weights( - aligned, - pd_point=pd_point, - pd_low=pd_low, - pd_high=pd_high, - effective_pd=effective_pd, - policy=policy, - budget=budget, - threads=max(1, int(threads)), - ) - elif mode == "proxy": - weights, alloc_meta = _compute_proxy_weights( - aligned, - pd_point=pd_point, - pd_high=pd_high, - effective_pd=effective_pd, - policy=policy, - budget=budget, - ) - else: - raise ValueError(f"Unsupported allocator-mode={allocator_mode!r}") - - certificate = compute_funded_certificate_metrics( - weights, - outcomes=y_true, - pd_point=pd_point, - pd_high=pd_high, - pd_effective=effective_pd, - alpha=alpha, - risk_tolerance=float(policy["risk_tolerance"]), - pd_cap_slack=float(alloc_meta.get("pd_cap_slack", 0.0)), - ) - bound_b_value = min(1.0, alpha / max(t_eval, 1e-8)) - risk_excess = round(certificate.realized_risk_tolerance_excess, 6) - empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) - markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) - - return { - "alpha": float(alpha), - "confidence": float(1.0 - alpha), - "gamma_cp": round(certificate.gamma_cp, 6), - "gamma_internalized": round(certificate.gamma_internalized, 6), - "gamma_residual": round(certificate.gamma_residual, 6), - "n_funded": certificate.n_funded, - "weighted_pd_true": round(certificate.weighted_outcome, 6), - "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), - "weighted_pd_high": round(certificate.endpoint_budget, 6), - "weighted_pd_point": round(certificate.weighted_pd_point, 6), - "endpoint_budget": round(certificate.endpoint_budget, 9), - "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), - "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), - "markov_loss_cap": round(certificate.markov_loss_cap, 9), - "tau": float(policy["risk_tolerance"]), - "realized_risk_tolerance_excess": risk_excess, - "violation": risk_excess, - "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), - "weighted_coverage_funded": round(certificate.weighted_coverage, 6), - "sqrt_alpha": round(certificate.sqrt_alpha, 6), - "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), - "empirical_risk_excess_leq_alpha": empirical_risk_screen, - "bound_a_expected_violation_leq_alpha": empirical_risk_screen, - "bound_b_prob_violation_gt_t": round(bound_b_value, 4), - "bound_b_t_eval": float(t_eval), - "bound_b_is_vacuous": bool(bound_b_value >= 1.0), - "markov_miscoverage_screen_pass": markov_screen, - "bound_c_V_leq_sqrt_alpha": markov_screen, - "certificate_screen_pass": empirical_risk_screen and markov_screen, - "all_bounds_hold": empirical_risk_screen and markov_screen, - "allocator_mode": mode, - "solver_status": str(alloc_meta.get("solver_status", "unknown")), - "allocator_solver_backend": str(alloc_meta.get("solver_backend", policy["solver_backend"])), - "allocator_native_solver_error": str(alloc_meta.get("native_solver_error", "")), - "total_allocated": round(float(alloc_meta.get("total_allocated", 0.0)), 2), - "pd_cap_slack": round(float(alloc_meta.get("pd_cap_slack", 0.0)), 6), - } - - -def _plot_validation(results: list[dict[str, Any]], figure_prefix: Path) -> None: - matplotlib.rcParams.update( - { - "font.family": "serif", - "font.size": 10, - "axes.labelsize": 11, - "axes.titlesize": 12, - "figure.dpi": 150, - } - ) - - df = pd.DataFrame(results) - fig, axes = plt.subplots(1, 3, figsize=(14, 4.2)) - - ax = axes[0] - ax.plot(df["alpha"], df["gamma_cp"], "o-", color="#2c3e50", linewidth=2, markersize=6) - ax.set_xlabel(r"$\alpha$ (miscoverage level)") - ax.set_ylabel(r"$\Gamma_{\mathrm{CP}}(\alpha)$") - ax.set_title("(A) Presupuesto conformal de robustez") - ax.invert_xaxis() - ax.grid(True, alpha=0.3) - - ax = axes[1] - ax.bar( - df["alpha"], - df["realized_risk_tolerance_excess"], - width=0.012, - color="#27ae60", - alpha=0.7, - label="Realized risk-tolerance excess", - ) - ax.plot( - df["alpha"], - df["alpha"], - "r--", - linewidth=2, - label=r"Declared empirical screen: excess $\leq \alpha$", - ) - ax.set_xlabel(r"$\alpha$") - ax.set_ylabel("Excess above risk tolerance") - ax.set_title("(B) Realized risk-tolerance screen") - ax.legend(fontsize=8, loc="upper left") - ax.grid(True, alpha=0.3) - - ax = axes[2] - ax.bar( - df["alpha"], - df["weighted_miscoverage_V"], - width=0.012, - color="#3498db", - alpha=0.7, - label=r"$V = \sum w_i Z_i$ (empírico)", - ) - ax.plot(df["alpha"], df["sqrt_alpha"], "r--", linewidth=2, label=r"Cota $\sqrt{\alpha}$") - ax.set_xlabel(r"$\alpha$") - ax.set_ylabel(r"No-cobertura ponderada $V$") - ax.set_title(r"(C) Teorema 1(c): $V \leq \sqrt{\alpha}$") - ax.legend(fontsize=8, loc="upper left") - ax.grid(True, alpha=0.3) - - plt.tight_layout() - figure_prefix.parent.mkdir(parents=True, exist_ok=True) - for ext in ("pdf", "png"): - path = figure_prefix.with_suffix(f".{ext}") - fig.savefig(path, bbox_inches="tight", dpi=300 if ext == "png" else None) - logger.info("Saved figure: {}", path) - plt.close(fig) - - -def _print_summary(results: list[dict[str, Any]], allocator_mode: str) -> None: - all_pass = all(bool(r["all_bounds_hold"]) for r in results) - print("\n" + "=" * 96) - print(f"CERTIFICATE SUMMARY ({allocator_mode.upper()} allocator)") - print("=" * 96) - header = ( - f"{'α':>6} {'1-α':>6} {'Γ_CP':>8} {'Risk excess':>12} {'V':>8} " - f"{'√α':>8} {'Mode':>8} {'Pass':>6}" - ) - print(header) - print("-" * len(header)) - for r in results: - status = " ✓" if r["all_bounds_hold"] else " ✗" - print( - f"{r['alpha']:6.2f} {r['confidence']:6.2f} {r['gamma_cp']:8.4f} " - f"{r['realized_risk_tolerance_excess']:10.6f} {r['weighted_miscoverage_V']:8.4f} " - f"{r['sqrt_alpha']:8.4f} {r['allocator_mode']:>8} {status}" - ) - print("=" * 96) - print(f"Result: {'ALL SCREENS PASS' if all_pass else 'SOME SCREENS FAILED'}") - - -def main(argv: list[str] | None = None) -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--conformal-intervals-path", default=None) - parser.add_argument("--portfolio-policy-path", default="models/champion_portfolio_policy.json") - parser.add_argument("--allocator-mode", choices=["proxy", "exact"], default="exact") - parser.add_argument("--alpha-grid", default=None) - parser.add_argument("--max-candidates", type=int, default=DEFAULT_MAX_CANDIDATES) - parser.add_argument("--random-state", type=int, default=DEFAULT_RANDOM_STATE) - parser.add_argument("--output-json", default=None) - parser.add_argument("--comparison-output", default=str(DEFAULT_COMPARISON_PATH)) - parser.add_argument("--figure-prefix", default=str(DEFAULT_FIG_PREFIX)) - parser.add_argument("--t-eval", type=float, default=DEFAULT_T_EVAL) - parser.add_argument("--budget", type=float, default=DEFAULT_BUDGET) - args = parser.parse_args(argv) - - logger.info("=== Validating α-CP → Γ-robustness bound (Theorem 1) ===") - policy = _load_policy(args.portfolio_policy_path) - aligned = _load_aligned_dataset( - conformal_intervals_path=args.conformal_intervals_path, - max_candidates=args.max_candidates, - random_state=args.random_state, - ) - alpha_grid = _coerce_alpha_grid(args.alpha_grid) - allocator_mode = str(args.allocator_mode) - - results = [] - for alpha in alpha_grid: - logger.info("Validating α = {} with allocator={}", alpha, allocator_mode) - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode=allocator_mode, - budget=float(args.budget), - t_eval=float(args.t_eval), - ) - results.append(result) - status = "✓" if result["all_bounds_hold"] else "✗" - logger.info( - " α={:.2f} Γ_CP={:.4f} risk_excess={:.6f} V={:.4f} √α={:.4f} {}", - alpha, - result["gamma_cp"], - result["realized_risk_tolerance_excess"], - result["weighted_miscoverage_V"], - result["sqrt_alpha"], - status, - ) - - all_pass = all(bool(r["all_bounds_hold"]) for r in results) - output_json = Path( - args.output_json - or (DEFAULT_EXACT_JSON if allocator_mode == "exact" else DEFAULT_PROXY_JSON) - ) - summary = { - "certificate": "Funded-set Markov accounting and empirical screens", - "theorem": "Distribution-free Markov bound under weighted funded-set validity", - "paper": "CRPTO (CRPTO)", - "allocator_mode": allocator_mode, - "n_test_observations": len(aligned), - "policy": policy, - "conformal_intervals_path": str(args.conformal_intervals_path or ""), - "alphas_tested": alpha_grid, - "all_bounds_hold": all_pass, - "all_certificate_screens_pass": all_pass, - "results": results, - } - output_json.parent.mkdir(parents=True, exist_ok=True) - output_json.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8") - logger.info("Saved validation artifact: {}", output_json) - - if allocator_mode == "exact": - proxy_results = [ - _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="proxy", - budget=float(args.budget), - t_eval=float(args.t_eval), - ) - for alpha in alpha_grid - ] - exact_df = pd.DataFrame(results).add_prefix("exact_") - proxy_df = pd.DataFrame(proxy_results).add_prefix("proxy_") - comparison = pd.concat([exact_df, proxy_df], axis=1) - comparison_path = Path(args.comparison_output) - comparison_path.parent.mkdir(parents=True, exist_ok=True) - comparison.to_parquet(comparison_path, index=False) - logger.info("Saved proxy-vs-exact comparison: {}", comparison_path) - _plot_validation(results, Path(args.figure_prefix)) - - if all_pass: - logger.success("ALL BOUNDS HOLD across all alpha levels.") - else: - failed = [r["alpha"] for r in results if not r["all_bounds_hold"]] - logger.warning("Bounds failed at alpha = {}", failed) - - _print_summary(results, allocator_mode=allocator_mode) - - -if __name__ == "__main__": - main() diff --git a/scripts/write_book_build_info.py b/scripts/write_book_build_info.py deleted file mode 100644 index 1ccf675..0000000 --- a/scripts/write_book_build_info.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import subprocess -from datetime import datetime -from pathlib import Path -from zoneinfo import ZoneInfo - -from loguru import logger - -ROOT = Path(__file__).resolve().parents[1] -TARGET = ROOT / "book" / "includes" / "_build-info.qmd" - - -def _git_value(*args: str, default: str) -> str: - result = subprocess.run( - ["git", *args], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - value = result.stdout.strip() - return value or default - - -def main() -> None: - commit = _git_value("rev-parse", "--short", "HEAD", default="local") - branch = _git_value("branch", "--show-current", default="local") - rendered_at = datetime.now(ZoneInfo("America/Bogota")).strftime("%Y-%m-%d") - - TARGET.write_text( - "\n".join( - [ - "::: {.build-info}", - f"Build: `{commit}` | Rama: `{branch}` | Actualizado: `{rendered_at}`", - ":::", - "", - ] - ), - encoding="utf-8", - ) - logger.info("Wrote {}", TARGET) - - -if __name__ == "__main__": - main() diff --git a/src/data/make_dataset.py b/src/data/make_dataset.py index 2426964..944a84c 100644 --- a/src/data/make_dataset.py +++ b/src/data/make_dataset.py @@ -55,9 +55,10 @@ LGD_SNAPSHOT_DATE = pd.Timestamp("2020-09-30") -# Default-indicating statuses +# Snapshot labels. Status variants such as "Does not meet the credit policy" +# are normalized by pattern rather than silently discarded. DEFAULT_STATUSES = ["Charged Off", "Default"] -CURRENT_STATUSES = ["Fully Paid", "Current"] +NONDEFAULT_STATUSES = ["Fully Paid"] def _to_numeric_series(df: pd.DataFrame, col: str, default: float = 0.0) -> pd.Series: @@ -74,6 +75,21 @@ def _parse_issue_dates(series: pd.Series) -> pd.Series: return parsed +def _snapshot_default_target(statuses: pd.Series) -> pd.Series: + """Map resolved snapshot statuses to a nullable binary target. + + Unresolved loans remain missing. Their membership is an origination-time + fact, while their eventual outcome is not available at the snapshot. + """ + normalized = statuses.fillna("").astype(str).str.strip().str.lower() + is_default = normalized.eq("default") | normalized.str.contains("charged off", regex=False) + is_nondefault = normalized.str.contains("fully paid", regex=False) + target = pd.Series(pd.NA, index=statuses.index, dtype="Int8", name="default_flag") + target.loc[is_nondefault] = 0 + target.loc[is_default] = 1 + return target + + def _compute_lgd(df: pd.DataFrame) -> pd.Series: """Compute realized LGD in [0, 1] using principal recovery components. @@ -98,10 +114,16 @@ def _compute_lgd(df: pd.DataFrame) -> pd.Series: lgd = 1.0 - (recovered_principal / exposure) lgd = lgd.clip(lower=0.0, upper=1.0) - if "default_flag" in df.columns: - lgd = lgd.where(df["default_flag"].astype(int) == 1, 0.0) - lgd = lgd.fillna(1.0).clip(lower=0.0, upper=1.0) - return lgd.astype(float) + if "default_flag" not in df.columns: + return lgd.fillna(1.0).clip(lower=0.0, upper=1.0).astype(float) + + target = pd.to_numeric(df["default_flag"], errors="coerce") + default_mask = target.eq(1).fillna(False) + nondefault_mask = target.eq(0).fillna(False) + result = pd.Series(pd.NA, index=df.index, dtype="Float64", name="lgd") + result.loc[default_mask] = lgd.loc[default_mask].fillna(1.0).clip(0.0, 1.0) + result.loc[nondefault_mask] = 0.0 + return result def load_raw_data(filepath: str | Path) -> pd.DataFrame: @@ -112,17 +134,31 @@ def load_raw_data(filepath: str | Path) -> pd.DataFrame: return df -def initial_clean(df: pd.DataFrame) -> pd.DataFrame: - """Remove leakage columns and filter to resolved loans.""" - # Filter to resolved loans only (Fully Paid or Default/Charged Off) - resolved_statuses = DEFAULT_STATUSES + ["Fully Paid"] - mask = df["loan_status"].isin(resolved_statuses) - df = df[mask].copy() - logger.info(f"Filtered to {len(df):,} resolved loans") +def initial_clean(df: pd.DataFrame, *, legacy_resolved_only: bool = False) -> pd.DataFrame: + """Remove leakage columns while preserving the origination-time universe. + + ``legacy_resolved_only`` exists solely to reproduce the frozen historical + lane. New research must retain unresolved rows and handle label availability + through a declared temporal protocol. + """ + if "loan_status" not in df.columns: + raise KeyError("Raw Lending Club data must contain loan_status.") + df = df.copy() + df["default_flag"] = _snapshot_default_target(df["loan_status"]) + df["outcome_observed"] = df["default_flag"].notna() + if legacy_resolved_only: + df = df.loc[df["outcome_observed"]].copy() + logger.warning("Applied explicit legacy resolved-only filter: {:,} rows", len(df)) + else: + logger.info( + "Retained status-independent universe: {:,} rows ({:,} unresolved)", + len(df), + int(df["default_flag"].isna().sum()), + ) - # Create binary target - df["default_flag"] = df["loan_status"].isin(DEFAULT_STATUSES).astype(int) - logger.info(f"Default rate: {df['default_flag'].mean():.2%}") + observed = df.loc[df["outcome_observed"], "default_flag"].astype(float) + if not observed.empty: + logger.info("Observed-outcome default rate: {:.2%}", float(observed.mean())) # Build LGD target before leakage fields are dropped. df["lgd"] = _compute_lgd(df) @@ -160,11 +196,14 @@ def save_interim(df: pd.DataFrame, output_dir: str | Path) -> Path: def main( - input_path: str = "data/raw/Loan_status_2007-2020Q3.csv", output_dir: str = "data/interim/" + input_path: str = "data/raw/Loan_status_2007-2020Q3.csv", + output_dir: str = "data/interim/", + *, + legacy_resolved_only: bool = False, ) -> None: """Run full make_dataset pipeline.""" df = load_raw_data(input_path) - df = initial_clean(df) + df = initial_clean(df, legacy_resolved_only=legacy_resolved_only) save_interim(df, output_dir) @@ -174,5 +213,10 @@ def main( parser = argparse.ArgumentParser(description="Make dataset from raw Lending Club CSV") parser.add_argument("--input", default="data/raw/Loan_status_2007-2020Q3.csv") parser.add_argument("--output", default="data/interim/") + parser.add_argument( + "--legacy-resolved-only", + action="store_true", + help="Reproduce the frozen historical status filter; invalid for new prospective analyses.", + ) args = parser.parse_args() - main(args.input, args.output) + main(args.input, args.output, legacy_resolved_only=args.legacy_resolved_only) diff --git a/src/data/outcome_observability.py b/src/data/outcome_observability.py new file mode 100644 index 0000000..e708509 --- /dev/null +++ b/src/data/outcome_observability.py @@ -0,0 +1,524 @@ +"""Status-independent loan universes and maturity-aware temporal blocks.""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +LABEL_REQUIRED_SPLITS = frozenset( + { + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + } +) +DECISION_SPLITS = frozenset({"policy_development", "primary_oot", "censored_extension"}) + + +def normalize_loan_status(statuses: pd.Series) -> pd.Series: + """Normalize Lending Club snapshot statuses without changing membership.""" + return ( + statuses.astype("string") + .fillna("") + .str.strip() + .str.casefold() + .str.replace(r"\s+", " ", regex=True) + ) + + +def archive_status_default_from_status(statuses: pd.Series) -> pd.Series: + """Map the archive's observed status field to a nullable binary outcome. + + This is a retrospective classification of the status stored in the raw + archive. It is not the date-reconstructed evaluation endpoint below. + """ + normalized = normalize_loan_status(statuses) + positive = normalized.str.contains("charged off", regex=False) | normalized.eq("default") + negative = normalized.str.contains("fully paid", regex=False) + if bool((positive & negative).any()): + raise ValueError("A loan status cannot be both default and fully paid.") + target = pd.Series(pd.NA, index=statuses.index, dtype="Int8", name="snapshot_default") + target.loc[negative] = 0 + target.loc[positive] = 1 + return target + + +def archive_status_resolution_from_status(statuses: pd.Series) -> pd.Series: + """Classify the raw archive status as default, nondefault, or unresolved.""" + target = archive_status_default_from_status(statuses) + resolution = pd.Series("unresolved", index=statuses.index, dtype="string") + resolution.loc[target.eq(0).fillna(False)] = "nondefault" + resolution.loc[target.eq(1).fillna(False)] = "default" + return resolution.rename("snapshot_resolution") + + +def snapshot_default_from_status(statuses: pd.Series) -> pd.Series: + """Compatibility alias for :func:`archive_status_default_from_status`.""" + return archive_status_default_from_status(statuses) + + +def snapshot_resolution_from_status(statuses: pd.Series) -> pd.Series: + """Compatibility alias for :func:`archive_status_resolution_from_status`.""" + return archive_status_resolution_from_status(statuses) + + +def terminal_outcome_from_status(statuses: pd.Series) -> pd.Series: + """Map terminal repayment statuses to the pre-freeze nullable outcome. + + Charged-off status variants are defaults and fully-paid variants are + nondefaults. ``Default`` remains unresolved because the snapshot status + alone does not establish the protocol's terminal charged-off label. + """ + normalized = normalize_loan_status(statuses) + charged_off = normalized.str.contains("charged off", regex=False) + fully_paid = normalized.str.contains("fully paid", regex=False) + if bool((charged_off & fully_paid).any()): + raise ValueError("A loan status cannot be both charged off and fully paid.") + target = pd.Series(pd.NA, index=statuses.index, dtype="Int8", name="terminal_outcome") + target.loc[fully_paid] = 0 + target.loc[charged_off] = 1 + return target + + +def parse_last_payment_dates(values: pd.Series) -> pd.Series: + """Parse ``last_pymnt_d``, using month-end for month-granularity values.""" + strings = values.astype("string").str.strip() + month_year = strings.str.fullmatch(r"[A-Za-z]{3}-\d{4}", na=False) + parsed = pd.to_datetime(strings, format="%b-%Y", errors="coerce") + fallback = parsed.isna() & strings.notna() + parsed.loc[fallback] = pd.to_datetime(strings.loc[fallback], format="mixed", errors="coerce") + parsed.loc[month_year & parsed.notna()] = parsed.loc[ + month_year & parsed.notna() + ] + pd.offsets.MonthEnd(0) + return parsed.rename("last_pymnt_d_parsed") + + +def conservative_label_available_mask( + statuses: pd.Series, + last_payment_dates: pd.Series, + *, + cutoff: str | pd.Timestamp, + charged_off_lag_months: int = 6, +) -> pd.Series: + """Return labels conservatively observable by an explicit cutoff.""" + if not statuses.index.equals(last_payment_dates.index): + raise ValueError("Status and last-payment series must have identical indices.") + lag_months = int(charged_off_lag_months) + if lag_months < 0: + raise ValueError("charged_off_lag_months must be nonnegative.") + cutoff_date = _require_timestamp(cutoff, context="label-availability cutoff") + outcomes = terminal_outcome_from_status(statuses) + payment_dates = parse_last_payment_dates(last_payment_dates) + available_at = payment_dates.copy() + charged_off = outcomes.eq(1).fillna(False) + available_at.loc[charged_off] = payment_dates.loc[charged_off] + pd.DateOffset( + months=lag_months + ) + available = outcomes.notna() & available_at.notna() & available_at.le(cutoff_date) + return available.astype(bool).rename("label_available") + + +def build_outcome_label_availability( + statuses: pd.Series, + last_payment_dates: pd.Series, + *, + cutoff: str | pd.Timestamp, + charged_off_lag_months: int = 6, +) -> pd.DataFrame: + """Build row-level terminal outcomes and conservative availability dates.""" + outcomes = terminal_outcome_from_status(statuses) + payment_dates = parse_last_payment_dates(last_payment_dates) + available_at = payment_dates.copy().rename("label_available_at") + charged_off = outcomes.eq(1).fillna(False) + available_at.loc[charged_off] = payment_dates.loc[charged_off] + pd.DateOffset( + months=int(charged_off_lag_months) + ) + available = conservative_label_available_mask( + statuses, + last_payment_dates, + cutoff=cutoff, + charged_off_lag_months=charged_off_lag_months, + ) + return pd.concat([outcomes, payment_dates, available_at, available], axis=1) + + +def audit_outcome_label_availability( + frame: pd.DataFrame, + *, + cutoff: str | pd.Timestamp, + charged_off_lag_months: int = 6, + block_column: str = "design_split", + status_column: str = "loan_status", + last_payment_column: str = "last_pymnt_d", +) -> pd.DataFrame: + """Report conservative outcome-label counts and retention by design block.""" + required = {block_column, status_column, last_payment_column} + missing = sorted(required.difference(frame.columns)) + if missing: + raise KeyError(f"Outcome-label audit is missing columns: {missing}") + labels = build_outcome_label_availability( + frame[status_column], + frame[last_payment_column], + cutoff=cutoff, + charged_off_lag_months=charged_off_lag_months, + ) + audited = pd.DataFrame( + { + block_column: frame[block_column], + "terminal_outcome": labels["terminal_outcome"], + "label_available": labels["label_available"], + }, + index=frame.index, + ) + rows: list[dict[str, Any]] = [] + for block, group in audited.groupby(block_column, dropna=False, sort=True): + total = int(len(group)) + resolved = int(group["terminal_outcome"].notna().sum()) + retained = int(group["label_available"].sum()) + rows.append( + { + block_column: block, + "total_rows": total, + "terminal_outcome_rows": resolved, + "unresolved_outcome_rows": total - resolved, + "label_available_rows": retained, + "label_unavailable_rows": total - retained, + "retention_rate": retained / total if total else 0.0, + } + ) + return pd.DataFrame(rows) + + +def validate_minimum_label_retention( + audit: pd.DataFrame, + *, + minimum_retention: float, + block_column: str = "design_split", +) -> None: + """Require every audited block to exceed the declared retention floor.""" + threshold = float(minimum_retention) + if not 0.0 <= threshold < 1.0: + raise ValueError("minimum_retention must lie in [0, 1).") + required = {block_column, "retention_rate"} + missing = sorted(required.difference(audit.columns)) + if missing: + raise KeyError(f"Retention audit is missing columns: {missing}") + rates = pd.to_numeric(audit["retention_rate"], errors="coerce") + failed = rates.isna() | rates.le(threshold) + if bool(failed.any()): + details = ", ".join( + f"{block}={rate:.6f}" + for block, rate in zip( + audit.loc[failed, block_column].astype(str), rates.loc[failed], strict=True + ) + ) + raise RuntimeError( + f"Outcome-label retention must exceed {threshold:.2%} in every block; failed: {details}" + ) + + +def parse_term_months(terms: pd.Series) -> pd.Series: + """Extract integer term months from raw Lending Club values.""" + return pd.to_numeric( + terms.astype("string").str.extract(r"(\d+)", expand=False), + errors="coerce", + ).astype("Int16") + + +def parse_issue_dates(values: pd.Series) -> pd.Series: + """Parse the raw month-year issue date with a conservative fallback.""" + parsed = pd.to_datetime(values, format="%b-%Y", errors="coerce") + missing = parsed.isna() & values.notna() + if bool(missing.any()): + parsed.loc[missing] = pd.to_datetime(values.loc[missing], errors="coerce") + return parsed + + +def _between(dates: pd.Series, start: object, end: object) -> pd.Series: + return dates.between(pd.Timestamp(str(start)), pd.Timestamp(str(end))) + + +def assign_design_split(issue_dates: pd.Series, design: Mapping[str, Any]) -> pd.Series: + """Assign every block from issue date alone; status never affects membership.""" + dates = pd.to_datetime(issue_dates, errors="coerce") + labels = pd.Series("outside_design", index=issue_dates.index, dtype="string") + labels.loc[dates.le(pd.Timestamp(str(design["development_end"])))] = "pd_development" + labels.loc[ + _between( + dates, + design["probability_calibration_start"], + design["probability_calibration_end"], + ) + ] = "probability_calibration" + labels.loc[_between(dates, design["conformal_fit_start"], design["conformal_fit_end"])] = ( + "conformal_fit" + ) + labels.loc[ + _between( + dates, + design["policy_development_start"], + design["policy_development_end"], + ) + ] = "policy_development" + + periods = dates.dt.to_period("M").astype("string") + primary = pd.period_range( + str(design["primary_oot_start_month"]), + str(design["primary_oot_end_month"]), + freq="M", + ).astype(str) + extension = pd.period_range( + str(design["censored_extension_start_month"]), + str(design["censored_extension_end_month"]), + freq="M", + ).astype(str) + labels.loc[periods.isin(primary)] = "primary_oot" + labels.loc[periods.isin(extension)] = "censored_extension" + return labels.rename("design_split") + + +def _require_timestamp(value: Any, *, context: str) -> pd.Timestamp: + timestamp = pd.Timestamp(value) + if not isinstance(timestamp, pd.Timestamp): + raise ValueError(f"{context} is missing or is not a valid timestamp.") + return timestamp + + +def temporal_tail_split( + frame: pd.DataFrame, + *, + tail_fraction: float, + date_column: str = "issue_d", +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Timestamp]: + """Split a development frame at a whole-month temporal boundary.""" + fraction = float(tail_fraction) + if not 0.0 < fraction < 1.0: + raise ValueError("tail_fraction must lie in (0, 1).") + ordered = frame.sort_values([date_column, "id"], kind="mergesort") + if len(ordered) < 2: + raise ValueError("Temporal validation requires at least two rows.") + position = max(1, min(len(ordered) - 1, int(np.floor(len(ordered) * (1 - fraction))))) + cutoff_value = _require_timestamp( + ordered.iloc[position][date_column], context="temporal validation cutoff" + ) + cutoff = cutoff_value.to_period("M").to_timestamp() + train = ordered.loc[ordered[date_column] < cutoff].copy() + validation = ordered.loc[ordered[date_column] >= cutoff].copy() + if train.empty or validation.empty: + raise ValueError("Whole-month temporal tail split produced an empty block.") + train_max = _require_timestamp(train[date_column].max(), context="training maximum") + validation_min = _require_timestamp(validation[date_column].min(), context="validation minimum") + if train_max >= validation_min: + raise AssertionError("Temporal validation blocks overlap or contain invalid dates.") + return train, validation, cutoff + + +def maturity_gap_months(earlier_end: str | pd.Timestamp, later_start: str | pd.Timestamp) -> int: + """Return the difference between two calendar-month indices.""" + earlier = _require_timestamp(earlier_end, context="earlier boundary") + later = _require_timestamp(later_start, context="later boundary") + return int((later.year * 12 + later.month) - (earlier.year * 12 + earlier.month)) + + +def load_design_universe( + config: Mapping[str, Any], + *, + raw_path: Path, + label_required_splits: Collection[str] | None = None, +) -> tuple[pd.DataFrame, dict[str, Any]]: + """Load declared windows from raw data without filtering on loan status. + + ``label_required_splits`` separates blocks that consume outcomes from + outcome-free decision-development menus. The legacy default preserves the + historical maturity-safe selector contract. + """ + source = config["source"] + design = config["design"] + required_columns = [str(value) for value in source["required_raw_columns"]] + header_frame = pd.read_csv(raw_path, nrows=0) + header = [str(column) for column in header_frame.columns] + missing = sorted(set(required_columns).difference(header)) + if missing: + raise KeyError(f"Raw Lending Club CSV is missing columns: {missing}") + + chunks: list[pd.DataFrame] = [] + counters: dict[str, Any] = { + "raw_rows_seen": 0, + "invalid_issue_date_rows": 0, + "term_36_rows_all_dates": 0, + "declared_window_rows_all_terms": 0, + "retained_rows": 0, + } + reader = pd.read_csv( + raw_path, + usecols=required_columns, + dtype={"id": "string", "loan_status": "string", "term": "string"}, + chunksize=int(source["csv_chunksize"]), + low_memory=False, + ) + for chunk in reader: + counters["raw_rows_seen"] += int(len(chunk)) + issue_dates = parse_issue_dates(chunk["issue_d"]) + terms = parse_term_months(chunk["term"]) + design_split = assign_design_split(issue_dates, design) + counters["invalid_issue_date_rows"] += int(issue_dates.isna().sum()) + counters["term_36_rows_all_dates"] += int(terms.eq(36).fillna(False).sum()) + in_window = design_split.ne("outside_design") + counters["declared_window_rows_all_terms"] += int(in_window.sum()) + keep = terms.eq(int(design["term_months"])).fillna(False) & in_window + if not bool(keep.any()): + continue + retained = chunk.loc[keep].copy() + retained["issue_d"] = issue_dates.loc[keep] + retained["term_months"] = terms.loc[keep] + retained["design_split"] = design_split.loc[keep] + chunks.append(retained) + counters["retained_rows"] += int(len(retained)) + + if not chunks: + raise RuntimeError("Raw scan found no loans under the declared design.") + frame = pd.concat(chunks, ignore_index=True) + if bool(frame["id"].isna().any()): + raise ValueError("Retained design universe contains missing loan IDs.") + frame["id"] = frame["id"].astype("string").str.strip() + if bool(frame["id"].duplicated().any()): + examples = frame.loc[frame["id"].duplicated(keep=False), "id"].head(5).tolist() + raise ValueError(f"Retained design universe contains duplicate IDs: {examples}") + frame["snapshot_default"] = archive_status_default_from_status(frame["loan_status"]) + frame["snapshot_resolution"] = archive_status_resolution_from_status(frame["loan_status"]) + frame = frame.sort_values(["issue_d", "id"], kind="mergesort").reset_index(drop=True) + + required_labels = ( + LABEL_REQUIRED_SPLITS + if label_required_splits is None + else frozenset(str(value) for value in label_required_splits) + ) + required_splits = required_labels | DECISION_SPLITS + observed_splits = set(frame["design_split"].astype(str)) + absent = sorted(required_splits.difference(observed_splits)) + if absent: + raise RuntimeError(f"Declared design blocks are empty: {absent}") + label_rows = frame["design_split"].isin(required_labels) + unresolved = int(frame.loc[label_rows, "snapshot_default"].isna().sum()) + if unresolved: + raise RuntimeError(f"Label-required blocks contain {unresolved} unresolved rows.") + + split_inventory = ( + frame.groupby(["design_split", "snapshot_resolution"], observed=True) + .size() + .to_frame("rows") + .reset_index() + ) + status_inventory = ( + frame.assign(normalized_status=normalize_loan_status(frame["loan_status"])) + .groupby(["design_split", "normalized_status"], observed=True) + .size() + .to_frame("rows") + .reset_index() + ) + counters.update( + { + "retained_rows_by_split": { + str(key): int(value) + for key, value in frame["design_split"].value_counts().sort_index().items() + }, + "resolved_rows": int(frame["snapshot_default"].notna().sum()), + "unresolved_rows": int(frame["snapshot_default"].isna().sum()), + "raw_schema_columns": header, + "split_inventory": split_inventory.to_dict(orient="records"), + "status_inventory": status_inventory.to_dict(orient="records"), + "membership_uses_loan_status": False, + "label_required_splits": sorted(required_labels), + } + ) + return frame, counters + + +def _month_set(frame: pd.DataFrame, split: str) -> set[str]: + dates = pd.to_datetime(frame.loc[frame["design_split"].eq(split), "issue_d"], errors="coerce") + return set(dates.dt.to_period("M").astype(str)) + + +def _expected_months(start: object, end: object) -> set[str]: + return set(pd.period_range(str(start), str(end), freq="M").astype(str)) + + +def validate_maturity_contract( + frame: pd.DataFrame, + design: Mapping[str, Any], + source: Mapping[str, Any], +) -> dict[str, Any]: + """Validate chronology, label maturity, and snapshot observability.""" + ordered_blocks = [ + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ] + bounds: dict[str, dict[str, str]] = {} + previous_max: pd.Timestamp | None = None + for block in ordered_blocks: + dates = pd.to_datetime( + frame.loc[frame["design_split"].eq(block), "issue_d"], errors="coerce" + ) + if dates.empty or bool(dates.isna().any()): + raise RuntimeError(f"Chronology block {block} is empty or has invalid dates.") + block_min = _require_timestamp(dates.min(), context=f"{block} minimum") + block_max = _require_timestamp(dates.max(), context=f"{block} maximum") + if previous_max is not None and previous_max >= block_min: + raise RuntimeError(f"Chronology blocks overlap or are out of order at {block}.") + bounds[block] = { + "first_issue_month": str(block_min.to_period("M")), + "last_issue_month": str(block_max.to_period("M")), + } + previous_max = block_max + + expected = { + "primary_oot": _expected_months( + design["primary_oot_start_month"], design["primary_oot_end_month"] + ), + "censored_extension": _expected_months( + design["censored_extension_start_month"], design["censored_extension_end_month"] + ), + } + for split, expected_months in expected.items(): + if _month_set(frame, split) != expected_months: + raise RuntimeError(f"{split} does not cover its complete declared monthly range.") + + policy_end = _require_timestamp( + str(design["policy_development_end"]), context="policy development end" + ) + primary_start = _require_timestamp( + str(design["primary_oot_start_month"]), context="primary OOT start" + ) + gap = maturity_gap_months(policy_end, primary_start) + required = int(design["minimum_maturity_gap_months"]) + if gap < required: + raise RuntimeError(f"Maturity gap is {gap} months; declared minimum is {required}.") + term_months = int(design["term_months"]) + policy_maturity = policy_end + pd.DateOffset(months=term_months) + if policy_maturity >= primary_start: + raise RuntimeError("The latest policy-development loan is not mature before primary OOT.") + + snapshot_date = _require_timestamp(str(source["snapshot_date"]), context="snapshot date") + latest_issue = _require_timestamp(frame["issue_d"].max(), context="latest issue date") + latest_contract_maturity = latest_issue + pd.DateOffset(months=term_months) + return { + "block_boundaries": bounds, + "policy_development_contract_maturity_month": str(policy_maturity.to_period("M")), + "primary_oot_first_issue_month": str(primary_start.to_period("M")), + "maturity_gap_months": gap, + "minimum_maturity_gap_months": required, + "snapshot_date": str(snapshot_date.date()), + "latest_retained_contract_maturity_month": str(latest_contract_maturity.to_period("M")), + "latest_contract_maturity_after_snapshot": bool(latest_contract_maturity > snapshot_date), + "membership_uses_loan_status": False, + "passes": True, + } diff --git a/src/data/prepare_dataset.py b/src/data/prepare_dataset.py index e5fc3d5..1dbd499 100644 --- a/src/data/prepare_dataset.py +++ b/src/data/prepare_dataset.py @@ -15,6 +15,21 @@ CALIBRATION_FRACTION = 0.15 # fraction of train set for conformal calibration +def require_observed_binary_target(df: pd.DataFrame) -> None: + """Reject implicit outcome-based filtering before model splits.""" + if "default_flag" not in df.columns: + raise KeyError("Input data must contain default_flag.") + target = pd.to_numeric(df["default_flag"], errors="coerce") + unresolved = int(target.isna().sum()) + if unresolved: + raise RuntimeError( + f"Input contains {unresolved:,} unresolved outcomes. Do not filter them by loan_status; " + "use a declared maturity/label-availability experiment instead." + ) + if not set(target.astype(int).unique()).issubset({0, 1}): + raise ValueError("default_flag must be binary once label availability is established.") + + def _parse_mixed_date(series: pd.Series, *, primary_format: str | None = None) -> pd.Series: if primary_format: parsed = pd.to_datetime(series, format=primary_format, errors="coerce") @@ -103,6 +118,7 @@ def main( """Run full prepare pipeline.""" df = pd.read_parquet(input_path) df = parse_dates(df) + require_observed_binary_target(df) train, test = out_of_time_split(df, cutoff_date) proper_train, calibration = create_calibration_set(train) save_splits(proper_train, test, calibration, output_dir) diff --git a/src/evaluation/__init__.py b/src/evaluation/__init__.py index 60656f8..ce2379b 100644 --- a/src/evaluation/__init__.py +++ b/src/evaluation/__init__.py @@ -1 +1 @@ -"""Evaluation metrics, backtesting, and IFRS9 calculations.""" +"""Coverage, payoff, and partial-identification evaluation for the IJDS audit.""" diff --git a/src/evaluation/ab_testing.py b/src/evaluation/ab_testing.py deleted file mode 100644 index a894424..0000000 --- a/src/evaluation/ab_testing.py +++ /dev/null @@ -1,184 +0,0 @@ -"""A/B testing simulation framework for portfolio strategy comparison. - -Provides tools for power analysis, stratified treatment assignment, -and statistical comparison of two competing portfolio strategies -applied retroactively to the OOT test set. - -This is a SIMULATION framework — no live randomization is implied. -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -from loguru import logger -from scipy import stats - - -def power_analysis( - effect_size: float, - alpha: float = 0.05, - power: float = 0.80, - n_groups: int = 2, -) -> dict[str, float]: - """Estimate minimum sample size per group for a two-sample proportion test. - - Uses normal approximation for the difference of two proportions. - - Args: - effect_size: Expected difference in means between groups. - alpha: Significance level (Type I error rate). - power: Statistical power (1 - Type II error rate). - n_groups: Number of groups (default 2 for A/B). - - Returns: - Dict with n_per_group, total_n, effect_size, alpha, power. - """ - z_alpha = stats.norm.ppf(1 - alpha / n_groups) - z_beta = stats.norm.ppf(power) - # Standard formula for two-sample mean comparison - # n = ((z_alpha + z_beta) / effect_size)^2 * 2 * sigma^2 - # For proportion test, sigma^2 ≈ p*(1-p) ≈ 0.25 (conservative) - variance = 0.25 # conservative for proportions - n_per_group = int(np.ceil(2 * variance * ((z_alpha + z_beta) / (effect_size + 1e-12)) ** 2)) - - result = { - "n_per_group": float(n_per_group), - "total_n": float(n_per_group * n_groups), - "effect_size": effect_size, - "alpha": alpha, - "power": power, - } - logger.info(f"Power analysis: n_per_group={n_per_group} for effect_size={effect_size:.4f}") - return result - - -def stratified_split( - df: pd.DataFrame, - strata_col: str, - treatment_ratio: float = 0.50, - seed: int = 42, -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Balanced treatment/control assignment within strata. - - Args: - df: DataFrame to split. - strata_col: Column to stratify on (e.g., 'grade'). - treatment_ratio: Proportion assigned to treatment group. - seed: Random seed for reproducibility. - - Returns: - Tuple of (control_df, treatment_df) with no index overlap. - """ - rng = np.random.RandomState(seed) - control_idx: list[int] = [] - treatment_idx: list[int] = [] - - for _, group_df in df.groupby(strata_col): - indices = group_df.index.tolist() - rng.shuffle(indices) - n_treatment = int(np.floor(len(indices) * treatment_ratio)) - treatment_idx.extend(indices[:n_treatment]) - control_idx.extend(indices[n_treatment:]) - - control = df.loc[control_idx].copy() - treatment = df.loc[treatment_idx].copy() - - logger.info( - f"Stratified split on '{strata_col}': control={len(control)}, treatment={len(treatment)}" - ) - return control, treatment - - -def compare_strategies( - returns_a: np.ndarray, - returns_b: np.ndarray, - method: str = "bootstrap", - n_boot: int = 1000, - alpha: float = 0.05, - seed: int = 42, -) -> dict[str, float | bool]: - """Statistical comparison of two return distributions. - - Args: - returns_a: Per-loan realized returns for strategy A (control). - returns_b: Per-loan realized returns for strategy B (treatment). - method: One of 'bootstrap', 'permutation', 'ttest'. - n_boot: Number of bootstrap/permutation resamples. - alpha: Significance level. - seed: Random seed. - - Returns: - Dict with mean_a, mean_b, diff, ci_low, ci_high, p_value, significant. - """ - returns_a = np.asarray(returns_a, dtype=float) - returns_b = np.asarray(returns_b, dtype=float) - rng = np.random.RandomState(seed) - - mean_a = float(np.mean(returns_a)) - mean_b = float(np.mean(returns_b)) - observed_diff = mean_b - mean_a - - if method == "bootstrap": - diffs = np.empty(n_boot) - for i in range(n_boot): - idx_a = rng.randint(0, len(returns_a), len(returns_a)) - idx_b = rng.randint(0, len(returns_b), len(returns_b)) - diffs[i] = np.mean(returns_b[idx_b]) - np.mean(returns_a[idx_a]) - ci_low = float(np.percentile(diffs, 100 * alpha / 2)) - ci_high = float(np.percentile(diffs, 100 * (1 - alpha / 2))) - p_value = float(np.mean(diffs <= 0)) if observed_diff > 0 else float(np.mean(diffs >= 0)) - - elif method == "permutation": - combined = np.concatenate([returns_a, returns_b]) - n_a = len(returns_a) - perm_diffs = np.empty(n_boot) - for i in range(n_boot): - rng.shuffle(combined) - perm_diffs[i] = np.mean(combined[n_a:]) - np.mean(combined[:n_a]) - ci_low = float(np.percentile(perm_diffs, 100 * alpha / 2)) - ci_high = float(np.percentile(perm_diffs, 100 * (1 - alpha / 2))) - p_value = float(np.mean(np.abs(perm_diffs) >= abs(observed_diff))) - - elif method == "ttest": - t_stat, p_value = stats.ttest_ind(returns_b, returns_a, equal_var=False) - se = np.sqrt(np.var(returns_b) / len(returns_b) + np.var(returns_a) / len(returns_a)) - z = stats.norm.ppf(1 - alpha / 2) - ci_low = float(observed_diff - z * se) - ci_high = float(observed_diff + z * se) - p_value = float(p_value) - else: - raise ValueError(f"Unknown method: {method}. Use 'bootstrap', 'permutation', or 'ttest'.") - - return { - "mean_a": mean_a, - "mean_b": mean_b, - "diff": observed_diff, - "ci_low": ci_low, - "ci_high": ci_high, - "p_value": p_value, - "significant": p_value < alpha, - } - - -def ab_summary( - control_metrics: dict[str, float], - treatment_metrics: dict[str, float], -) -> pd.DataFrame: - """Format control vs treatment metrics as a tidy comparison table. - - Args: - control_metrics: Dict of metric_name → value for control. - treatment_metrics: Dict of metric_name → value for treatment. - - Returns: - DataFrame with columns: metric, control, treatment, lift_pct. - """ - all_keys = sorted(set(control_metrics) | set(treatment_metrics)) - rows = [] - for key in all_keys: - c = control_metrics.get(key, 0.0) - t = treatment_metrics.get(key, 0.0) - lift = ((t - c) / (abs(c) + 1e-12)) * 100 - rows.append({"metric": key, "control": c, "treatment": t, "lift_pct": lift}) - return pd.DataFrame(rows) diff --git a/src/evaluation/backtesting.py b/src/evaluation/backtesting.py deleted file mode 100644 index 9978662..0000000 --- a/src/evaluation/backtesting.py +++ /dev/null @@ -1,633 +0,0 @@ -"""Backtesting and out-of-time validation for credit risk models. - -Includes: -- Cohort analysis across vintage periods. -- Population Stability Index (PSI) for distribution drift. -- Kupiec (1995) Proportion of Failures (POF) test for unconditional coverage. -- Christoffersen (1998) test for conditional coverage and independence. -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -from loguru import logger -from scipy import stats -from sklearn.ensemble import HistGradientBoostingClassifier -from sklearn.inspection import permutation_importance -from sklearn.metrics import roc_auc_score -from sklearn.model_selection import train_test_split - -from src.evaluation.coverage_tests import christoffersen_test, kupiec_pof_test -from src.evaluation.explainability import effective_driver_count -from src.evaluation.metrics import classification_metrics - - -def cohort_analysis( - df: pd.DataFrame, - y_true_col: str = "default_flag", - y_prob_col: str = "pd_predicted", - cohort_col: str = "issue_quarter", -) -> pd.DataFrame: - """Evaluate model performance across vintage cohorts.""" - results = [] - for cohort, group in df.groupby(cohort_col): - if len(group) < 50: - continue - metrics: dict[str, object] = dict( - classification_metrics( - group[y_true_col].values, - group[y_prob_col].values, - ) - ) - metrics["cohort"] = str(cohort) - metrics["n_loans"] = len(group) - metrics["default_rate"] = group[y_true_col].mean() - results.append(metrics) - - result = pd.DataFrame(results) - logger.info(f"Cohort analysis: {len(result)} cohorts evaluated") - return result - - -def population_stability_index( - expected: np.ndarray, - actual: np.ndarray, - n_bins: int = 10, -) -> float: - """Compute PSI to detect distribution drift between train and test.""" - expected_arr = np.asarray(expected, dtype=float) - actual_arr = np.asarray(actual, dtype=float) - expected_arr = expected_arr[np.isfinite(expected_arr)] - actual_arr = actual_arr[np.isfinite(actual_arr)] - if expected_arr.size == 0 or actual_arr.size == 0: - return 0.0 - - bin_edges = np.percentile(expected_arr, np.linspace(0, 100, n_bins + 1)) - bin_edges = np.unique(bin_edges) - if bin_edges.size < 2: - # Degenerate feature with near-constant values. - return 0.0 - bin_edges[-1] += 1e-6 - - expected_pct = np.histogram(expected_arr, bins=bin_edges)[0] / len(expected_arr) - actual_pct = np.histogram(actual_arr, bins=bin_edges)[0] / len(actual_arr) - - # Avoid log(0) - expected_pct = np.clip(expected_pct, 1e-6, None) - actual_pct = np.clip(actual_pct, 1e-6, None) - - psi = np.sum((actual_pct - expected_pct) * np.log(actual_pct / expected_pct)) - logger.info( - f"PSI = {psi:.4f} ({'stable' if psi < 0.1 else 'drift detected' if psi < 0.25 else 'significant drift'})" - ) - return psi - - -def ks_two_sample_test( - expected: np.ndarray, - actual: np.ndarray, -) -> dict[str, float]: - """Two-sample Kolmogorov-Smirnov test for train-vs-test drift.""" - expected_arr = np.asarray(expected, dtype=float) - actual_arr = np.asarray(actual, dtype=float) - expected_arr = expected_arr[np.isfinite(expected_arr)] - actual_arr = actual_arr[np.isfinite(actual_arr)] - - if expected_arr.size == 0 or actual_arr.size == 0: - return {"ks_statistic": 0.0, "ks_pvalue": 1.0} - - stat, pvalue = stats.ks_2samp(expected_arr, actual_arr) - return {"ks_statistic": float(stat), "ks_pvalue": float(pvalue)} - - -def cramervonmises_two_sample_test( - expected: np.ndarray, - actual: np.ndarray, -) -> dict[str, float]: - """Two-sample Cramér-von Mises test for train-vs-test drift.""" - expected_arr = np.asarray(expected, dtype=float) - actual_arr = np.asarray(actual, dtype=float) - expected_arr = expected_arr[np.isfinite(expected_arr)] - actual_arr = actual_arr[np.isfinite(actual_arr)] - - if expected_arr.size == 0 or actual_arr.size == 0: - return {"cvm_statistic": 0.0, "cvm_pvalue": 1.0} - - result = stats.cramervonmises_2samp(expected_arr, actual_arr) - return {"cvm_statistic": float(result.statistic), "cvm_pvalue": float(result.pvalue)} - - -def classifier_two_sample_test( - train_df: pd.DataFrame, - test_df: pd.DataFrame, - features: list[str], - *, - max_rows_per_split: int = 50_000, - random_state: int = 42, -) -> dict[str, float | int | str | list[dict[str, float | str]]]: - """Classifier Two-Sample Test (C2ST) / adversarial validation. - - Returns AUC of a binary classifier trying to distinguish train rows (label 0) - from test rows (label 1). Values near 0.5 indicate low distribution shift. - """ - if not features: - return { - "c2st_auc": 0.5, - "n_rows": 0, - "n_features": 0, - "materiality": "none", - "effective_driver_count": 0, - "top_drivers": [], - } - - train = train_df[features].copy() - test = test_df[features].copy() - - if len(train) > max_rows_per_split: - train = train.sample(n=max_rows_per_split, random_state=random_state) - if len(test) > max_rows_per_split: - test = test.sample(n=max_rows_per_split, random_state=random_state) - - train["__c2st_label"] = 0 - test["__c2st_label"] = 1 - all_df = pd.concat([train, test], axis=0, ignore_index=True) - all_df = all_df.sample(frac=1.0, random_state=random_state).reset_index(drop=True) - - y = all_df.pop("__c2st_label").to_numpy(dtype=int) - X = all_df.apply(pd.to_numeric, errors="coerce").fillna(0.0) - if X.shape[0] < 200 or X.shape[1] == 0: - return { - "c2st_auc": 0.5, - "n_rows": int(X.shape[0]), - "n_features": int(X.shape[1]), - "materiality": "none", - "effective_driver_count": 0, - "top_drivers": [], - } - - X_train, X_val, y_train, y_val = train_test_split( - X, - y, - test_size=0.30, - random_state=random_state, - stratify=y, - ) - clf = HistGradientBoostingClassifier( - max_depth=6, - max_iter=250, - learning_rate=0.05, - random_state=random_state, - ) - clf.fit(X_train, y_train) - y_prob = clf.predict_proba(X_val)[:, 1] - auc = roc_auc_score(y_val, y_prob) - materiality = "none" - if float(auc) >= 0.70: - materiality = "severe" - elif float(auc) >= 0.60: - materiality = "high" - elif float(auc) >= 0.55: - materiality = "moderate" - elif float(auc) >= 0.52: - materiality = "low" - - top_drivers: list[dict[str, float | str]] = [] - try: - perm = permutation_importance( - clf, - X_val, - y_val, - n_repeats=3, - random_state=random_state, - scoring="roc_auc", - ) - importances = pd.Series(perm.importances_mean, index=X.columns, dtype=float) - importances = ( - importances.replace([np.inf, -np.inf], np.nan) - .fillna(0.0) - .abs() - .sort_values(ascending=False) - ) - top_drivers = [ - {"feature": str(feature), "importance_auc": float(value)} - for feature, value in importances.head(10).items() - ] - effective_count = int(effective_driver_count(importances)) - except Exception: - top_drivers = [] - effective_count = 0 - - return { - "c2st_auc": float(auc), - "n_rows": int(X.shape[0]), - "n_features": int(X.shape[1]), - "materiality": materiality, - "effective_driver_count": int(effective_count), - "top_drivers": top_drivers, - } - - -def two_sided_exact_binomial_test( - n_defaults: int, - n_obs: int, - pd_ref: float, -) -> dict[str, float]: - """Two-sided exact binomial test for PD backtesting.""" - n = int(max(n_obs, 0)) - k = int(max(n_defaults, 0)) - p = float(np.clip(pd_ref, 1e-9, 1.0 - 1e-9)) - if n <= 0: - return {"p_value": 1.0, "expected_defaults": 0.0} - result = stats.binomtest(k, n=n, p=p, alternative="two-sided") - return {"p_value": float(result.pvalue), "expected_defaults": float(n * p)} - - -def jeffreys_interval( - n_defaults: int, - n_obs: int, - *, - alpha: float = 0.05, -) -> dict[str, float]: - """Jeffreys interval for observed default rate.""" - n = int(max(n_obs, 0)) - k = int(max(n_defaults, 0)) - if n <= 0: - return {"lower": 0.0, "upper": 1.0} - lower = float(stats.beta.ppf(alpha / 2.0, k + 0.5, n - k + 0.5)) - upper = float(stats.beta.ppf(1.0 - alpha / 2.0, k + 0.5, n - k + 0.5)) - return {"lower": lower, "upper": upper} - - -def normal_approximation_backtest( - n_defaults: int, - n_obs: int, - pd_ref: float, -) -> dict[str, float]: - """Normal-approximation z-score PD backtest.""" - n = int(max(n_obs, 0)) - k = int(max(n_defaults, 0)) - p = float(np.clip(pd_ref, 1e-9, 1.0 - 1e-9)) - if n <= 0: - return {"z_score": 0.0, "p_value": 1.0, "expected_defaults": 0.0} - variance = float(n * p * (1.0 - p)) - if variance <= 0.0: - return {"z_score": 0.0, "p_value": 1.0, "expected_defaults": float(n * p)} - z_score = float((k - (n * p)) / np.sqrt(variance)) - p_value = float(2.0 * (1.0 - stats.norm.cdf(abs(z_score)))) - return { - "z_score": z_score, - "p_value": p_value, - "expected_defaults": float(n * p), - } - - -def hosmer_lemeshow_test( - y_true: np.ndarray, - y_prob: np.ndarray, - *, - n_groups: int = 10, -) -> dict[str, float | int]: - """Hosmer-Lemeshow calibration test.""" - y_true_arr = np.asarray(y_true, dtype=float) - y_prob_arr = np.asarray(y_prob, dtype=float) - mask = np.isfinite(y_true_arr) & np.isfinite(y_prob_arr) - y_true_arr = y_true_arr[mask] - y_prob_arr = y_prob_arr[mask] - if y_true_arr.size < max(n_groups * 10, 50): - return {"hl_statistic": 0.0, "hl_p_value": 1.0, "n_groups": 0} - - df = pd.DataFrame({"y_true": y_true_arr, "y_prob": y_prob_arr}) - df["group"] = pd.qcut(df["y_prob"], q=n_groups, duplicates="drop") - grouped = df.groupby("group", observed=True) - if len(grouped) < 2: - return {"hl_statistic": 0.0, "hl_p_value": 1.0, "n_groups": len(grouped)} - - statistic = 0.0 - for _, grp in grouped: - n_obs = float(len(grp)) - observed = float(grp["y_true"].sum()) - expected = float(grp["y_prob"].sum()) - expected_non = float(n_obs - expected) - observed_non = float(n_obs - observed) - if expected > 0.0: - statistic += ((observed - expected) ** 2) / expected - if expected_non > 0.0: - statistic += ((observed_non - expected_non) ** 2) / expected_non - dof = max(int(len(grouped) - 2), 1) - p_value = float(1.0 - stats.chi2.cdf(statistic, dof)) - return { - "hl_statistic": float(statistic), - "hl_p_value": p_value, - "n_groups": len(grouped), - } - - -def bootstrap_gap_materiality(abs_gap_bp: float) -> str: - """Classify the practical size of a PD gap in basis points.""" - gap_bp = float(abs(abs_gap_bp)) - if gap_bp >= 200.0: - return "severe" - if gap_bp >= 100.0: - return "high" - if gap_bp >= 50.0: - return "moderate" - if gap_bp >= 25.0: - return "low" - return "negligible" - - -def bootstrap_pd_gap_test( - y_true: np.ndarray, - y_prob: np.ndarray, - *, - n_boot: int = 2_000, - alpha: float = 0.05, - random_state: int = 42, - max_sample_size: int = 10_000, - batch_size: int = 200, -) -> dict[str, float | int | bool | str]: - """Bootstrap the observed-minus-predicted PD gap. - - This diagnostic is intentionally diagnostic-only. It estimates the - uncertainty around the aggregate calibration gap without relying solely - on asymptotic p-values that can become overly sensitive at very large `N`. - """ - y_true_arr = np.asarray(y_true, dtype=float) - y_prob_arr = np.asarray(y_prob, dtype=float) - mask = np.isfinite(y_true_arr) & np.isfinite(y_prob_arr) - y_true_arr = y_true_arr[mask] - y_prob_arr = y_prob_arr[mask] - n_obs = int(y_true_arr.size) - if n_obs == 0: - return { - "n_obs": 0, - "bootstrap_sample_size": 0, - "n_boot": int(n_boot), - "observed_default_rate": 0.0, - "mean_predicted_pd": 0.0, - "rate_gap": 0.0, - "abs_gap_bp": 0.0, - "ci_lower": 0.0, - "ci_upper": 0.0, - "zero_inside_ci": True, - "bootstrap_p_value": 1.0, - "materiality": "negligible", - } - - diff = y_true_arr - y_prob_arr - observed_gap = float(diff.mean()) - sample_size = int(min(n_obs, max_sample_size)) - rng = np.random.RandomState(random_state) - boot_means: list[np.ndarray] = [] - for start in range(0, int(n_boot), int(max(batch_size, 1))): - current = int(min(batch_size, n_boot - start)) - draws = rng.randint(0, n_obs, size=(current, sample_size)) - boot_means.append(diff[draws].mean(axis=1)) - boot = np.concatenate(boot_means) if boot_means else np.asarray([observed_gap], dtype=float) - lower = float(np.quantile(boot, alpha / 2.0)) - upper = float(np.quantile(boot, 1.0 - (alpha / 2.0))) - p_lower = float(np.mean(boot <= 0.0)) - p_upper = float(np.mean(boot >= 0.0)) - p_value = float(min(1.0, 2.0 * min(p_lower, p_upper))) - abs_gap_bp = float(abs(observed_gap) * 10_000.0) - return { - "n_obs": n_obs, - "bootstrap_sample_size": sample_size, - "n_boot": len(boot), - "observed_default_rate": float(y_true_arr.mean()), - "mean_predicted_pd": float(y_prob_arr.mean()), - "rate_gap": observed_gap, - "abs_gap_bp": abs_gap_bp, - "ci_lower": lower, - "ci_upper": upper, - "zero_inside_ci": bool(lower <= 0.0 <= upper), - "bootstrap_p_value": p_value, - "materiality": bootstrap_gap_materiality(abs_gap_bp), - } - - -def bootstrap_slice_gap_report( - frame: pd.DataFrame, - *, - group_col: str, - target_col: str = "default_flag", - score_col: str = "pd_calibrated", - min_rows: int = 200, - n_boot: int = 1_000, - alpha: float = 0.05, - random_state: int = 42, - max_sample_size: int = 5_000, -) -> pd.DataFrame: - """Bootstrap calibration-gap diagnostics by slice.""" - if ( - group_col not in frame.columns - or target_col not in frame.columns - or score_col not in frame.columns - ): - return pd.DataFrame() - - rows: list[dict[str, float | int | bool | str]] = [] - for group_value, group in frame.groupby(group_col, observed=True): - if len(group) < int(min_rows): - continue - report = bootstrap_pd_gap_test( - group[target_col].to_numpy(dtype=float), - group[score_col].to_numpy(dtype=float), - n_boot=n_boot, - alpha=alpha, - random_state=random_state, - max_sample_size=max_sample_size, - ) - rows.append( - { - "slice_name": str(group_col), - "slice_value": str(group_value), - **report, - } - ) - if not rows: - return pd.DataFrame() - return ( - pd.DataFrame(rows) - .sort_values(["zero_inside_ci", "abs_gap_bp"], ascending=[True, False]) - .reset_index(drop=True) - ) - - -def drift_monitoring_report( - train_df: pd.DataFrame, - test_df: pd.DataFrame, - features: list[str], - *, - psi_threshold: float = 0.25, - ks_pvalue_threshold: float = 0.01, - cvm_pvalue_threshold: float = 0.01, - n_bins: int = 10, -) -> pd.DataFrame: - """Build per-feature drift monitoring table. - - Produces PSI + KS + CvM metrics and per-metric pass/fail flags. - """ - rows: list[dict[str, float | int | str | bool]] = [] - for feature in features: - if feature not in train_df.columns or feature not in test_df.columns: - continue - - tr = pd.to_numeric(train_df[feature], errors="coerce").to_numpy(dtype=float) - te = pd.to_numeric(test_df[feature], errors="coerce").to_numpy(dtype=float) - - tr = tr[np.isfinite(tr)] - te = te[np.isfinite(te)] - if tr.size < 30 or te.size < 30: - continue - - psi = population_stability_index(tr, te, n_bins=n_bins) - ks = ks_two_sample_test(tr, te) - cvm = cramervonmises_two_sample_test(tr, te) - - rows.append( - { - "feature": feature, - "train_n": int(tr.size), - "test_n": int(te.size), - "psi": float(psi), - "ks_statistic": float(ks["ks_statistic"]), - "ks_pvalue": float(ks["ks_pvalue"]), - "cvm_statistic": float(cvm["cvm_statistic"]), - "cvm_pvalue": float(cvm["cvm_pvalue"]), - "pass_psi": bool(psi <= psi_threshold), - "pass_ks": bool(ks["ks_pvalue"] >= ks_pvalue_threshold), - "pass_cvm": bool(cvm["cvm_pvalue"] >= cvm_pvalue_threshold), - } - ) - - if not rows: - return pd.DataFrame( - { - "feature": pd.Series(dtype="object"), - "train_n": pd.Series(dtype="int64"), - "test_n": pd.Series(dtype="int64"), - "psi": pd.Series(dtype="float64"), - "ks_statistic": pd.Series(dtype="float64"), - "ks_pvalue": pd.Series(dtype="float64"), - "cvm_statistic": pd.Series(dtype="float64"), - "cvm_pvalue": pd.Series(dtype="float64"), - "pass_psi": pd.Series(dtype="bool"), - "pass_ks": pd.Series(dtype="bool"), - "pass_cvm": pd.Series(dtype="bool"), - } - ) - out = pd.DataFrame(rows).sort_values("psi", ascending=False).reset_index(drop=True) - return out - - -def filter_high_psi_features( - train_df: pd.DataFrame, - test_df: pd.DataFrame, - features: list[str], - psi_threshold: float = 0.25, - n_bins: int = 10, -) -> dict[str, list[str] | pd.DataFrame]: - """Identify and filter features with high PSI (distribution drift). - - Equivalent to feature-engine's DropHighPSIFeatures but using the - project's existing PSI implementation. - - Args: - train_df: Training data (reference distribution). - test_df: Test/production data (current distribution). - features: List of numeric feature names to evaluate. - psi_threshold: Features with PSI above this are flagged for removal. - n_bins: Number of bins for PSI computation. - - Returns: - Dict with: - - 'stable_features': Features with PSI <= threshold. - - 'drifted_features': Features with PSI > threshold. - - 'psi_table': DataFrame with per-feature PSI values. - """ - psi_records: list[dict[str, float | str]] = [] - for feat in features: - if feat not in train_df.columns or feat not in test_df.columns: - continue - tr = pd.to_numeric(train_df[feat], errors="coerce").to_numpy(dtype=float) - te = pd.to_numeric(test_df[feat], errors="coerce").to_numpy(dtype=float) - tr = tr[np.isfinite(tr)] - te = te[np.isfinite(te)] - if tr.size < 30 or te.size < 30: - psi_records.append({"feature": feat, "psi": 0.0, "stable": True}) - continue - psi = population_stability_index(tr, te, n_bins=n_bins) - psi_records.append( - { - "feature": feat, - "psi": float(psi), - "stable": bool(psi <= psi_threshold), - } - ) - - psi_table = pd.DataFrame(psi_records).sort_values("psi", ascending=False).reset_index(drop=True) - stable = [str(r["feature"]) for r in psi_records if r["stable"]] - drifted = [str(r["feature"]) for r in psi_records if not r["stable"]] - - if drifted: - logger.warning( - f"PSI filter: {len(drifted)} features drifted (PSI > {psi_threshold}): {drifted}" - ) - else: - logger.info(f"PSI filter: all {len(stable)} features stable (PSI <= {psi_threshold})") - - return {"stable_features": stable, "drifted_features": drifted, "psi_table": psi_table} - - -def interval_violations( - y_true: np.ndarray, - lower: np.ndarray, - upper: np.ndarray, -) -> np.ndarray: - """Return binary violation indicators (1 if y_true outside [lower, upper]).""" - y = np.asarray(y_true, dtype=float) - lo = np.asarray(lower, dtype=float) - hi = np.asarray(upper, dtype=float) - outside = (y < lo) | (y > hi) - return outside.astype(float) - - -def winkler_interval_score( - y_true: np.ndarray, - lower: np.ndarray, - upper: np.ndarray, - alpha: float, -) -> np.ndarray: - """Compute Winkler interval score per observation. - - Lower is better. Inside-interval score is width; outside gets linear penalty. - """ - y = np.asarray(y_true, dtype=float) - lo = np.asarray(lower, dtype=float) - hi = np.asarray(upper, dtype=float) - widths = np.maximum(0.0, hi - lo) - - score = widths.copy() - below = y < lo - above = y > hi - penalty_scale = 2.0 / max(float(alpha), 1e-8) - - score[below] = widths[below] + penalty_scale * (lo[below] - y[below]) - score[above] = widths[above] + penalty_scale * (y[above] - hi[above]) - return score - - -__all__ = [ - "christoffersen_test", - "classifier_two_sample_test", - "cohort_analysis", - "cramervonmises_two_sample_test", - "drift_monitoring_report", - "interval_violations", - "ks_two_sample_test", - "kupiec_pof_test", - "population_stability_index", - "winkler_interval_score", -] diff --git a/src/evaluation/coverage_tests.py b/src/evaluation/coverage_tests.py deleted file mode 100644 index 9a737ce..0000000 --- a/src/evaluation/coverage_tests.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Statistical coverage tests for conformal prediction intervals. - -Includes: -- Kupiec (1995) Proportion of Failures (POF) test for unconditional coverage. -- Christoffersen (1998) test for conditional coverage and independence. -""" - -from __future__ import annotations - -from typing import Any - -import numpy as np -from loguru import logger -from scipy import stats - - -def kupiec_pof_test( - violations: np.ndarray, - alpha: float, - confidence: float = 0.95, -) -> dict[str, Any]: - """Kupiec (1995) Proportion of Failures test for unconditional coverage. - - Tests H0: the true violation rate equals the nominal rate alpha. - Uses a likelihood-ratio statistic distributed as chi2(1). - - Args: - violations: Binary array (1 = observation outside interval). - alpha: Nominal violation rate (e.g., 0.10 for 90% coverage). - confidence: Confidence level for the test (default 0.95). - - Returns: - Dict with lr_statistic, p_value, reject, n_violations, n_total, - violation_rate, nominal_alpha. - """ - violations = np.asarray(violations, dtype=float) - n_total = violations.size - n_violations = int(violations.sum()) - - if n_total == 0: - return { - "lr_statistic": 0.0, - "p_value": 1.0, - "reject": False, - "n_violations": 0, - "n_total": 0, - "violation_rate": 0.0, - "nominal_alpha": alpha, - } - - p_hat = n_violations / n_total - v = n_violations - t = n_total - - # Avoid log(0) edge cases - if p_hat == 0.0 or p_hat == 1.0: - # Degenerate: if p_hat matches alpha exactly (unlikely), LR=0 - if abs(p_hat - alpha) < 1e-10: - lr = 0.0 - else: - # Use large LR to indicate strong rejection - lr = 2 * t * abs(np.log(max(alpha, 1e-15)) - np.log(max(p_hat, 1e-15))) - else: - log_l0 = v * np.log(alpha) + (t - v) * np.log(1 - alpha) - log_l1 = v * np.log(p_hat) + (t - v) * np.log(1 - p_hat) - lr = -2 * (log_l0 - log_l1) - - p_value = float(1 - stats.chi2.cdf(lr, df=1)) - reject = p_value < (1 - confidence) - - logger.info( - f"Kupiec POF: violations={n_violations}/{n_total} " - f"(rate={p_hat:.4f}, nominal={alpha:.4f}), " - f"LR={lr:.4f}, p={p_value:.4f}, reject={reject}" - ) - return { - "lr_statistic": float(lr), - "p_value": p_value, - "reject": reject, - "n_violations": n_violations, - "n_total": n_total, - "violation_rate": p_hat, - "nominal_alpha": alpha, - } - - -def christoffersen_test( - violations: np.ndarray, - alpha: float, - confidence: float = 0.95, -) -> dict[str, Any]: - """Christoffersen (1998) conditional coverage test. - - Combines the Kupiec unconditional coverage test with a test for - independence of violations (no temporal clustering). Uses a joint - likelihood-ratio statistic distributed as chi2(2). - - Args: - violations: Binary array ordered by time (1 = violation). - alpha: Nominal violation rate. - confidence: Confidence level for the test. - - Returns: - Dict with lr_uc, p_uc, lr_ind, p_ind, lr_cc, p_cc, - reject_uc, reject_ind, reject_cc, transition_matrix. - """ - violations = np.asarray(violations, dtype=float) - n_total = violations.size - - # Unconditional coverage component - uc = kupiec_pof_test(violations, alpha, confidence) - - if n_total < 2: - return { - "lr_uc": uc["lr_statistic"], - "p_uc": uc["p_value"], - "reject_uc": uc["reject"], - "lr_ind": 0.0, - "p_ind": 1.0, - "reject_ind": False, - "lr_cc": uc["lr_statistic"], - "p_cc": uc["p_value"], - "reject_cc": uc["reject"], - "transition_matrix": {"n00": 0, "n01": 0, "n10": 0, "n11": 0}, - } - - # Build transition counts - v0 = violations[:-1] - v1 = violations[1:] - n00 = int(((v0 == 0) & (v1 == 0)).sum()) - n01 = int(((v0 == 0) & (v1 == 1)).sum()) - n10 = int(((v0 == 1) & (v1 == 0)).sum()) - n11 = int(((v0 == 1) & (v1 == 1)).sum()) - - # Transition probabilities - row0 = n00 + n01 - row1 = n10 + n11 - - # Overall violation rate under independence (H0) - pi = (n01 + n11) / max(n_total - 1, 1) - - # Row-conditional violation rates (H1) - pi01 = n01 / row0 if row0 > 0 else 0.0 - pi11 = n11 / row1 if row1 > 0 else 0.0 - - # Independence LR - _eps = 1e-15 - - def _safe_log(x: float) -> float: - return np.log(max(x, _eps)) - - # L(pi) under independence - log_l0_ind = 0.0 - if row0 > 0: - log_l0_ind += n00 * _safe_log(1 - pi) + n01 * _safe_log(pi) - if row1 > 0: - log_l0_ind += n10 * _safe_log(1 - pi) + n11 * _safe_log(pi) - - # L(pi01, pi11) under dependence - log_l1_ind = 0.0 - if row0 > 0: - log_l1_ind += n00 * _safe_log(1 - pi01) + n01 * _safe_log(pi01) - if row1 > 0: - log_l1_ind += n10 * _safe_log(1 - pi11) + n11 * _safe_log(pi11) - - lr_ind = -2 * (log_l0_ind - log_l1_ind) - lr_ind = max(lr_ind, 0.0) # Numerical safety - - p_ind = float(1 - stats.chi2.cdf(lr_ind, df=1)) - reject_ind = p_ind < (1 - confidence) - - # Joint conditional coverage: LR_cc = LR_uc + LR_ind - lr_cc = uc["lr_statistic"] + lr_ind - p_cc = float(1 - stats.chi2.cdf(lr_cc, df=2)) - reject_cc = p_cc < (1 - confidence) - - logger.info( - f"Christoffersen: LR_uc={uc['lr_statistic']:.4f} (p={uc['p_value']:.4f}), " - f"LR_ind={lr_ind:.4f} (p={p_ind:.4f}), " - f"LR_cc={lr_cc:.4f} (p={p_cc:.4f}), reject_cc={reject_cc}" - ) - return { - "lr_uc": uc["lr_statistic"], - "p_uc": uc["p_value"], - "reject_uc": uc["reject"], - "lr_ind": float(lr_ind), - "p_ind": p_ind, - "reject_ind": reject_ind, - "lr_cc": float(lr_cc), - "p_cc": p_cc, - "reject_cc": reject_cc, - "transition_matrix": {"n00": n00, "n01": n01, "n10": n10, "n11": n11}, - } diff --git a/src/evaluation/coverage_transport.py b/src/evaluation/coverage_transport.py new file mode 100644 index 0000000..bc5a9ab --- /dev/null +++ b/src/evaluation/coverage_transport.py @@ -0,0 +1,377 @@ +"""Coverage audits and exact selection-transport decompositions.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + assign_conformal_groups, +) + + +def binary_miscoverage_bounds( + outcomes: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Return sharp binary-outcome miscoverage bounds for nullable outcomes.""" + y_true = np.asarray(outcomes, dtype=float) + low = np.asarray(lower, dtype=float) + high = np.asarray(upper, dtype=float) + if not (y_true.shape == low.shape == high.shape): + raise ValueError("Outcome and interval arrays must align.") + observed = np.isfinite(y_true) + miss_observed = (y_true < low) | (y_true > high) + miss_zero = (low > 0.0) | (high < 0.0) + miss_one = (low > 1.0) | (high < 1.0) + bound_low = np.where(observed, miss_observed, np.minimum(miss_zero, miss_one)) + bound_high = np.where(observed, miss_observed, np.maximum(miss_zero, miss_one)) + return bound_low.astype(float), bound_high.astype(float) + + +def _coverage_summary( + subset: pd.DataFrame, + *, + design_split: str, + period: str, + group_label: str, +) -> dict[str, Any]: + y_true = pd.to_numeric(subset["snapshot_default"], errors="coerce").to_numpy(dtype=float) + lower = subset["conformal_lower"].to_numpy(dtype=float) + upper = subset["conformal_upper"].to_numpy(dtype=float) + miss_low, miss_high = binary_miscoverage_bounds(y_true, lower, upper) + resolved = np.isfinite(y_true) + resolved_coverage = float(1.0 - miss_low[resolved].mean()) if bool(resolved.any()) else None + return { + "design_split": design_split, + "period": period, + "conformal_group": group_label, + "rows": int(len(subset)), + "resolved_rows": int(resolved.sum()), + "unresolved_rows": int((~resolved).sum()), + "resolved_empirical_coverage": resolved_coverage, + "all_candidate_coverage_lower": float(1.0 - miss_high.mean()), + "all_candidate_coverage_upper": float(1.0 - miss_low.mean()), + "all_candidate_miscoverage_lower": float(miss_low.mean()), + "all_candidate_miscoverage_upper": float(miss_high.mean()), + "mean_interval_width": float((upper - lower).mean()), + "lower_endpoint_zero_share": float(np.mean(lower <= 1e-12)), + "upper_endpoint_one_share": float(np.mean(upper >= 1.0 - 1e-12)), + "pd_point_min": float(subset["pd_point"].min()), + "pd_point_max": float(subset["pd_point"].max()), + "below_fit_score_range": int(subset["below_fit_score_range"].sum()), + "above_fit_score_range": int(subset["above_fit_score_range"].sum()), + } + + +def build_temporal_conformal_audit( + decision_frame: pd.DataFrame, + outcomes: pd.DataFrame, + recipe: BinaryOutcomeConformalRecipe, +) -> pd.DataFrame: + """Audit marginal coverage by month and score stratum after freezing.""" + merged = decision_frame.merge(outcomes, on="id", how="left", validate="one_to_one") + if bool(merged["snapshot_resolution"].isna().any()): + raise RuntimeError("Temporal conformal audit could not align all outcomes.") + merged["period"] = pd.to_datetime(merged["issue_d"]).dt.to_period("M").astype(str) + point = merged["pd_point"].to_numpy(dtype=float) + assigned = assign_conformal_groups(point, recipe.bin_edges) + if not np.array_equal(assigned, merged["conformal_group"].to_numpy(dtype=int)): + raise AssertionError("Stored groups do not match the frozen conformal recipe.") + merged["below_fit_score_range"] = point < float(recipe.bin_edges[0]) + merged["above_fit_score_range"] = point > float(recipe.bin_edges[-1]) + + rows: list[dict[str, Any]] = [] + splits = sorted(merged["design_split"].astype(str).unique()) + for design_split in splits: + split_frame = merged.loc[merged["design_split"].astype(str).eq(design_split)] + periods = sorted(split_frame["period"].astype(str).unique()) + for period in periods: + month = split_frame.loc[split_frame["period"].astype(str).eq(period)] + rows.append( + _coverage_summary( + month, + design_split=design_split, + period=period, + group_label="ALL", + ) + ) + for group in sorted(month["conformal_group"].astype(int).unique()): + group_frame = month.loc[month["conformal_group"].eq(group)] + rows.append( + _coverage_summary( + group_frame, + design_split=design_split, + period=period, + group_label=f"score_q{int(group):02d}", + ) + ) + pooled_period = f"{periods[0]}_to_{periods[-1]}" + rows.append( + _coverage_summary( + split_frame, + design_split=design_split, + period=pooled_period, + group_label="ALL", + ) + ) + for group in sorted(split_frame["conformal_group"].astype(int).unique()): + group_frame = split_frame.loc[split_frame["conformal_group"].eq(group)] + rows.append( + _coverage_summary( + group_frame, + design_split=design_split, + period=pooled_period, + group_label=f"score_q{int(group):02d}", + ) + ) + return ( + pd.DataFrame(rows) + .sort_values( + ["design_split", "period", "conformal_group"], + kind="mergesort", + ) + .reset_index(drop=True) + ) + + +def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: + total = float(weights.sum()) + if total <= 0.0: + raise ValueError("Transport decomposition weights must sum to a positive value.") + return float(weights @ values / total) + + +def metric_transport_decomposition( + *, + metric_name: str, + candidate_values: np.ndarray, + candidate_exposure: np.ndarray, + candidate_groups: np.ndarray, + funded_values: np.ndarray, + funded_exposure: np.ndarray, + funded_groups: np.ndarray, + reference: float | None = None, +) -> dict[str, float | str]: + """Decompose row, exposure, group-composition, and within-group shifts.""" + values = np.asarray(candidate_values, dtype=float) + exposure = np.asarray(candidate_exposure, dtype=float) + groups = np.asarray(candidate_groups, dtype=int) + selected_values = np.asarray(funded_values, dtype=float) + selected_exposure = np.asarray(funded_exposure, dtype=float) + selected_groups = np.asarray(funded_groups, dtype=int) + if not (len(values) == len(exposure) == len(groups)): + raise ValueError("Candidate transport arrays must align.") + if not (len(selected_values) == len(selected_exposure) == len(selected_groups)): + raise ValueError("Funded transport arrays must align.") + arrays = [values, exposure, selected_values, selected_exposure] + if any(not bool(np.isfinite(array).all()) for array in arrays): + raise ValueError("Exact transport decomposition requires finite values.") + + row_value = float(values.mean()) + exposure_value = _weighted_mean(values, exposure) + funded_value = _weighted_mean(selected_values, selected_exposure) + funded_total = float(selected_exposure.sum()) + group_mix_value = 0.0 + for group in sorted(set(selected_groups.tolist())): + candidate_mask = groups == group + funded_mask = selected_groups == group + if not bool(candidate_mask.any()): + raise ValueError(f"Funded conformal group {group} is absent from candidates.") + candidate_group_rate = _weighted_mean(values[candidate_mask], exposure[candidate_mask]) + funded_group_share = float(selected_exposure[funded_mask].sum() / funded_total) + group_mix_value += funded_group_share * candidate_group_rate + + reference_value = row_value if reference is None else float(reference) + row_minus_reference = row_value - reference_value + row_to_exposure = exposure_value - row_value + group_composition = group_mix_value - exposure_value + within_group_selection = funded_value - group_mix_value + total = funded_value - reference_value + reconstructed = ( + row_minus_reference + row_to_exposure + group_composition + within_group_selection + ) + return { + "metric": metric_name, + "reference": reference_value, + "candidate_row": row_value, + "candidate_exposure_weighted": exposure_value, + "funded_group_mix_counterfactual": group_mix_value, + "funded_exposure_weighted": funded_value, + "row_minus_reference": row_minus_reference, + "row_to_exposure": row_to_exposure, + "group_composition": group_composition, + "within_group_selection": within_group_selection, + "total_minus_reference": total, + "identity_residual": total - reconstructed, + } + + +def coverage_and_default_transport_decomposition( + candidates_with_outcomes: pd.DataFrame, + funded_allocations: pd.DataFrame, + *, + alpha: float, +) -> pd.DataFrame: + """Return exact miscoverage and default transport identities.""" + candidate_y = pd.to_numeric( + candidates_with_outcomes["snapshot_default"], errors="coerce" + ).to_numpy(dtype=float) + funded_y = pd.to_numeric(funded_allocations["snapshot_default"], errors="coerce").to_numpy( + dtype=float + ) + if not bool(np.isfinite(candidate_y).all()) or not bool(np.isfinite(funded_y).all()): + raise ValueError("Exact transport decomposition cannot include unresolved outcomes.") + candidate_miss, _ = binary_miscoverage_bounds( + candidate_y, + candidates_with_outcomes["conformal_lower"].to_numpy(dtype=float), + candidates_with_outcomes["conformal_upper"].to_numpy(dtype=float), + ) + funded_miss, _ = binary_miscoverage_bounds( + funded_y, + funded_allocations["conformal_lower"].to_numpy(dtype=float), + funded_allocations["conformal_upper"].to_numpy(dtype=float), + ) + candidate_exposure = candidates_with_outcomes["loan_amnt"].to_numpy(dtype=float) + candidate_groups = candidates_with_outcomes["conformal_group"].to_numpy(dtype=int) + funded_exposure = funded_allocations["exposure"].to_numpy(dtype=float) + funded_groups = funded_allocations["conformal_group"].to_numpy(dtype=int) + records = [ + metric_transport_decomposition( + metric_name="binary_miscoverage", + candidate_values=candidate_miss, + candidate_exposure=candidate_exposure, + candidate_groups=candidate_groups, + funded_values=funded_miss, + funded_exposure=funded_exposure, + funded_groups=funded_groups, + reference=float(alpha), + ), + metric_transport_decomposition( + metric_name="snapshot_default", + candidate_values=candidate_y, + candidate_exposure=candidate_exposure, + candidate_groups=candidate_groups, + funded_values=funded_y, + funded_exposure=funded_exposure, + funded_groups=funded_groups, + ), + ] + result = pd.DataFrame(records) + if bool((result["identity_residual"].abs() > 1e-12).any()): + raise AssertionError("Transport decomposition identity did not reconcile.") + return result + + +def coverage_and_default_transport_bounds( + candidates_with_outcomes: pd.DataFrame, + funded_allocations: pd.DataFrame, + *, + alpha: float, +) -> pd.DataFrame: + """Return transport identities under sharp lower and upper completions. + + The funded endpoint in each row is a sharp aggregate bound. Intermediate + decomposition terms describe the corresponding extremal completion; they + are not asserted to be component-wise confidence bounds. + """ + candidate_y = pd.to_numeric( + candidates_with_outcomes["snapshot_default"], errors="coerce" + ).to_numpy(dtype=float) + funded_y = pd.to_numeric(funded_allocations["snapshot_default"], errors="coerce").to_numpy( + dtype=float + ) + candidate_miss_low, candidate_miss_high = binary_miscoverage_bounds( + candidate_y, + candidates_with_outcomes["conformal_lower"].to_numpy(dtype=float), + candidates_with_outcomes["conformal_upper"].to_numpy(dtype=float), + ) + funded_miss_low, funded_miss_high = binary_miscoverage_bounds( + funded_y, + funded_allocations["conformal_lower"].to_numpy(dtype=float), + funded_allocations["conformal_upper"].to_numpy(dtype=float), + ) + candidate_default_low = np.nan_to_num(candidate_y, nan=0.0) + candidate_default_high = np.where(np.isfinite(candidate_y), candidate_y, 1.0) + funded_default_low = np.nan_to_num(funded_y, nan=0.0) + funded_default_high = np.where(np.isfinite(funded_y), funded_y, 1.0) + candidate_exposure = candidates_with_outcomes["loan_amnt"].to_numpy(dtype=float) + candidate_groups = candidates_with_outcomes["conformal_group"].to_numpy(dtype=int) + funded_exposure = funded_allocations["exposure"].to_numpy(dtype=float) + funded_groups = funded_allocations["conformal_group"].to_numpy(dtype=int) + + records: list[dict[str, float | str | bool]] = [] + scenarios = ( + ( + "binary_miscoverage", + "lower", + candidate_miss_low, + funded_miss_low, + float(alpha), + ), + ( + "binary_miscoverage", + "upper", + candidate_miss_high, + funded_miss_high, + float(alpha), + ), + ( + "snapshot_default", + "lower", + candidate_default_low, + funded_default_low, + None, + ), + ( + "snapshot_default", + "upper", + candidate_default_high, + funded_default_high, + None, + ), + ) + for metric, completion, candidate_values, funded_values, reference in scenarios: + record = metric_transport_decomposition( + metric_name=metric, + candidate_values=candidate_values, + candidate_exposure=candidate_exposure, + candidate_groups=candidate_groups, + funded_values=funded_values, + funded_exposure=funded_exposure, + funded_groups=funded_groups, + reference=reference, + ) + records.append( + { + **record, + "completion": completion, + "funded_endpoint_is_sharp_bound": True, + "components_are_completion_specific": True, + } + ) + result = pd.DataFrame(records) + if bool((result["identity_residual"].abs() > 1e-12).any()): + raise AssertionError("Bounded transport decomposition identity did not reconcile.") + return result + + +def weighted_group_shares( + groups: Sequence[int], + exposure: Sequence[float], +) -> dict[str, float]: + """Return deterministic exposure shares by conformal group.""" + group_array = np.asarray(tuple(groups), dtype=int) + exposure_array = np.asarray(tuple(exposure), dtype=float) + total = float(exposure_array.sum()) + if total <= 0.0 or len(group_array) != len(exposure_array): + raise ValueError("Group shares require aligned positive exposure.") + return { + f"score_q{int(group):02d}": float(exposure_array[group_array == group].sum() / total) + for group in sorted(set(group_array.tolist())) + } diff --git a/src/evaluation/explainability.py b/src/evaluation/explainability.py deleted file mode 100644 index ee2b6f8..0000000 --- a/src/evaluation/explainability.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Reusable explainability utilities for operational governance artifacts.""" - -from __future__ import annotations - -from itertools import combinations -from typing import Any - -import numpy as np -import pandas as pd - - -def rank_overlap_ratio(reference: list[str], comparison: list[str], top_k: int = 10) -> float: - """Return normalized overlap between two ranked feature lists.""" - ref = [str(x) for x in reference[:top_k]] - cmp = [str(x) for x in comparison[:top_k]] - if not ref or not cmp: - return 0.0 - denom = max(min(len(ref), len(cmp)), 1) - return float(len(set(ref).intersection(cmp)) / denom) - - -def effective_driver_count( - importances: pd.Series | dict[str, float], - *, - coverage: float = 0.80, -) -> int: - """Count features needed to explain the target importance mass.""" - if isinstance(importances, dict): - series = pd.Series(importances, dtype=float) - else: - series = pd.Series(importances, dtype=float) - series = series.replace([np.inf, -np.inf], np.nan).dropna().abs().sort_values(ascending=False) - total = float(series.sum()) - if total <= 0 or series.empty: - return 0 - cumulative = series.cumsum() / total - return int((cumulative <= float(coverage)).sum() + 1) - - -def compute_ale_curve( - model: Any, - X: pd.DataFrame, - feature: str, - *, - n_bins: int = 10, -) -> pd.DataFrame: - """Compute first-order ALE for a numeric feature using model probabilities.""" - if feature not in X.columns: - return pd.DataFrame() - - values = pd.to_numeric(X[feature], errors="coerce") - valid_mask = values.notna() - if int(valid_mask.sum()) < max(n_bins * 5, 50): - return pd.DataFrame() - - valid_values = values.loc[valid_mask] - quantiles = np.linspace(0.0, 1.0, n_bins + 1) - edges = np.quantile(valid_values.to_numpy(dtype=float), quantiles) - edges = np.unique(np.asarray(edges, dtype=float)) - if edges.size < 3: - return pd.DataFrame() - - rows: list[dict[str, float | int | str]] = [] - diffs: list[float] = [] - - work = X.loc[valid_mask].copy() - work_values = valid_values.to_numpy(dtype=float) - bin_ids = np.digitize(work_values, edges[1:-1], right=True) - - for bin_id in range(len(edges) - 1): - lower = float(edges[bin_id]) - upper = float(edges[bin_id + 1]) - mask = bin_ids == bin_id - n_obs = int(mask.sum()) - if n_obs == 0: - diffs.append(0.0) - rows.append( - { - "feature": feature, - "bin_id": int(bin_id), - "lower_bound": lower, - "upper_bound": upper, - "midpoint": float((lower + upper) / 2.0), - "ale_value": 0.0, - "n_obs": 0, - } - ) - continue - - x_low = work.loc[mask].copy() - x_high = work.loc[mask].copy() - x_low[feature] = lower - x_high[feature] = upper - pred_low = np.asarray(model.predict_proba(x_low)[:, 1], dtype=float) - pred_high = np.asarray(model.predict_proba(x_high)[:, 1], dtype=float) - delta = float(np.mean(pred_high - pred_low)) - diffs.append(delta) - rows.append( - { - "feature": feature, - "bin_id": int(bin_id), - "lower_bound": lower, - "upper_bound": upper, - "midpoint": float((lower + upper) / 2.0), - "ale_value": delta, - "n_obs": n_obs, - } - ) - - if not rows: - return pd.DataFrame() - - out = pd.DataFrame(rows) - out["ale_value"] = out["ale_value"].cumsum() - mean_ale = float(np.average(out["ale_value"], weights=np.maximum(out["n_obs"], 1))) - out["ale_value"] = out["ale_value"] - mean_ale - return out - - -def pairwise_shap_redundancy( - shap_df: pd.DataFrame, - features: list[str], - *, - max_features: int = 10, -) -> pd.DataFrame: - """Approximate SHAP interactions using pairwise dependence between contributions.""" - top_features = [str(f) for f in features[:max_features]] - rows: list[dict[str, float | str | bool]] = [] - for feature_a, feature_b in combinations(top_features, 2): - shap_a = f"shap_{feature_a}" - shap_b = f"shap_{feature_b}" - val_a = f"val_{feature_a}" - val_b = f"val_{feature_b}" - if shap_a not in shap_df.columns or shap_b not in shap_df.columns: - continue - subset = shap_df[[shap_a, shap_b]].copy() - shap_corr = subset.corr(method="spearman").iloc[0, 1] - value_corr = np.nan - if val_a in shap_df.columns and val_b in shap_df.columns: - values = shap_df[[val_a, val_b]].apply(pd.to_numeric, errors="coerce") - if values.notna().sum().min() > 10: - value_corr = values.corr(method="spearman").iloc[0, 1] - rows.append( - { - "feature_a": feature_a, - "feature_b": feature_b, - "shap_spearman": float(np.nan_to_num(shap_corr)), - "value_spearman": float(np.nan_to_num(value_corr)), - "redundancy_flag": bool( - abs(float(np.nan_to_num(shap_corr))) >= 0.35 - or abs(float(np.nan_to_num(value_corr))) >= 0.60 - ), - "relation_type": ( - "synergy" if float(np.nan_to_num(shap_corr)) >= 0 else "tradeoff" - ), - } - ) - if not rows: - return pd.DataFrame( - { - "feature_a": pd.Series(dtype="object"), - "feature_b": pd.Series(dtype="object"), - "shap_spearman": pd.Series(dtype="float64"), - "value_spearman": pd.Series(dtype="float64"), - "redundancy_flag": pd.Series(dtype="bool"), - "relation_type": pd.Series(dtype="object"), - } - ) - return ( - pd.DataFrame(rows) - .sort_values( - ["redundancy_flag", "shap_spearman"], - ascending=[False, False], - ) - .reset_index(drop=True) - ) - - -def extract_top_reason_feature( - row: pd.Series, - features: list[str], - *, - direction: str, -) -> str: - """Return the dominant positive or negative reason feature for a row.""" - pairs: list[tuple[str, float]] = [] - for feature in features: - col = f"shap_{feature}" - if col not in row.index: - continue - raw_value = pd.to_numeric(pd.Series([row[col]]), errors="coerce").iloc[0] - value = 0.0 if pd.isna(raw_value) else float(raw_value) - if (direction == "positive" and value > 0) or (direction == "negative" and value < 0): - pairs.append((feature, value)) - - if not pairs: - fallback = [] - for feature in features: - col = f"shap_{feature}" - if col not in row.index: - continue - raw_value = pd.to_numeric(pd.Series([row[col]]), errors="coerce").iloc[0] - value = 0.0 if pd.isna(raw_value) else float(raw_value) - fallback.append((feature, abs(value))) - if not fallback: - return "" - return max(fallback, key=lambda item: item[1])[0] - - if direction == "positive": - return max(pairs, key=lambda item: item[1])[0] - return min(pairs, key=lambda item: item[1])[0] - - -def dominant_reason_match_rate( - reference_df: pd.DataFrame, - comparison_df: pd.DataFrame, - features: list[str], - *, - pd_col: str = "pd_calibrated", - threshold: float = 0.5, - min_rows_per_band: int = 25, -) -> tuple[float, list[dict[str, str | float | bool]]]: - """Compare dominant reason codes across PD bands between two cohorts.""" - if reference_df.empty or comparison_df.empty: - return 0.0, [] - - bands = [ - ("low", lambda s: s <= threshold * 0.7), - ("near_threshold", lambda s: (s > threshold * 0.7) & (s <= threshold * 1.1)), - ("high", lambda s: s > threshold * 1.1), - ] - - details: list[dict[str, str | float | bool]] = [] - matches = 0 - valid_bands = 0 - - for band_name, selector in bands: - ref_band = reference_df.loc[selector(pd.to_numeric(reference_df[pd_col], errors="coerce"))] - cmp_band = comparison_df.loc[ - selector(pd.to_numeric(comparison_df[pd_col], errors="coerce")) - ] - if len(ref_band) < min_rows_per_band or len(cmp_band) < min_rows_per_band: - continue - - ref_reason = ref_band.apply( - lambda row: extract_top_reason_feature(row, features, direction="positive"), - axis=1, - ) - cmp_reason = cmp_band.apply( - lambda row: extract_top_reason_feature(row, features, direction="positive"), - axis=1, - ) - ref_mode = str(ref_reason.mode().iloc[0]) if not ref_reason.mode().empty else "" - cmp_mode = str(cmp_reason.mode().iloc[0]) if not cmp_reason.mode().empty else "" - matched = ref_mode != "" and ref_mode == cmp_mode - valid_bands += 1 - matches += int(matched) - details.append( - { - "band": band_name, - "reference_reason": ref_mode, - "comparison_reason": cmp_mode, - "matched": bool(matched), - } - ) - - if valid_bands == 0: - return 0.0, details - return float(matches / valid_bands), details - - -def monotonic_violation_rate( - model: Any, - X: pd.DataFrame, - feature: str, - direction: int, - *, - grid_size: int = 7, - sample_size: int = 128, - random_state: int = 42, -) -> float: - """Estimate monotonic violations by varying one feature over a grid.""" - if feature not in X.columns or direction == 0 or X.empty: - return 0.0 - - values = pd.to_numeric(X[feature], errors="coerce").dropna() - if len(values) < max(grid_size * 5, 50): - return 0.0 - - sample = X.copy() - if len(sample) > sample_size: - sample = sample.sample(n=sample_size, random_state=random_state) - - grid = np.quantile(values.to_numpy(dtype=float), np.linspace(0.05, 0.95, grid_size)) - grid = np.unique(grid) - if grid.size < 3: - return 0.0 - - preds: list[np.ndarray] = [] - for value in grid: - x_tmp = sample.copy() - x_tmp[feature] = value - preds.append(np.asarray(model.predict_proba(x_tmp)[:, 1], dtype=float)) - - stacked = np.vstack(preds) - diffs = np.diff(stacked, axis=0) - violations = diffs < -1e-6 if direction > 0 else diffs > 1e-6 - return float(np.mean(violations)) - - -__all__ = [ - "compute_ale_curve", - "dominant_reason_match_rate", - "effective_driver_count", - "extract_top_reason_feature", - "monotonic_violation_rate", - "pairwise_shap_redundancy", - "rank_overlap_ratio", -] diff --git a/src/evaluation/fairness.py b/src/evaluation/fairness.py deleted file mode 100644 index e5d68fc..0000000 --- a/src/evaluation/fairness.py +++ /dev/null @@ -1,419 +0,0 @@ -"""Fairness metrics for credit risk models. - -Computes demographic parity, equalized odds, and disparate impact -across protected attribute groups. Designed for proxy fairness analysis -(Lending Club has no race/gender data). - -Metrics: - - Demographic Parity Difference (DPD): max gap in positive prediction rate - - Equalized Odds Gap (EO): max gap in TPR or FPR across groups - - Disparate Impact Ratio (DIR): min(rate_i / rate_j) across group pairs -""" - -from __future__ import annotations - -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -_EPS = np.finfo(float).eps - - -def demographic_parity_difference( - y_pred: np.ndarray, - groups: np.ndarray, -) -> dict[str, Any]: - """Compute max gap in positive prediction rate across groups. - - Args: - y_pred: Binary predictions (0/1). - groups: Group labels per observation. - - Returns: - Dict with dpd, max_rate_group, min_rate_group, group_rates. - """ - y_pred = np.asarray(y_pred, dtype=float) - groups = np.asarray(groups) - unique_groups = np.unique(groups) - - group_rates: dict[str, float] = {} - for g in unique_groups: - mask = groups == g - group_rates[str(g)] = float(y_pred[mask].mean()) if mask.sum() > 0 else 0.0 - - rates = list(group_rates.values()) - dpd = max(rates) - min(rates) - max_group = max(group_rates, key=lambda key: group_rates[key]) - min_group = min(group_rates, key=lambda key: group_rates[key]) - - return { - "dpd": dpd, - "max_rate_group": max_group, - "min_rate_group": min_group, - "group_rates": group_rates, - } - - -def equalized_odds_gap( - y_true: np.ndarray, - y_pred: np.ndarray, - groups: np.ndarray, -) -> dict[str, Any]: - """Compute max gap in TPR and FPR across groups. - - Args: - y_true: Binary ground truth (0/1). - y_pred: Binary predictions (0/1). - groups: Group labels per observation. - - Returns: - Dict with tpr_gap, fpr_gap, eo_gap (max of both), group_tpr, group_fpr. - """ - y_true = np.asarray(y_true, dtype=float) - y_pred = np.asarray(y_pred, dtype=float) - groups = np.asarray(groups) - unique_groups = np.unique(groups) - - group_tpr: dict[str, float] = {} - group_fpr: dict[str, float] = {} - - for g in unique_groups: - mask = groups == g - yt, yp = y_true[mask], y_pred[mask] - - positives = yt == 1 - negatives = yt == 0 - - tpr = float(yp[positives].mean()) if positives.sum() > 0 else 0.0 - fpr = float(yp[negatives].mean()) if negatives.sum() > 0 else 0.0 - - group_tpr[str(g)] = tpr - group_fpr[str(g)] = fpr - - tpr_values = list(group_tpr.values()) - fpr_values = list(group_fpr.values()) - - tpr_gap = max(tpr_values) - min(tpr_values) if tpr_values else 0.0 - fpr_gap = max(fpr_values) - min(fpr_values) if fpr_values else 0.0 - eo_gap = max(tpr_gap, fpr_gap) - - return { - "tpr_gap": tpr_gap, - "fpr_gap": fpr_gap, - "eo_gap": eo_gap, - "group_tpr": group_tpr, - "group_fpr": group_fpr, - } - - -def disparate_impact_ratio( - y_pred: np.ndarray, - groups: np.ndarray, -) -> dict[str, Any]: - """Compute min(rate_i / rate_j) for all ordered group pairs. - - The 4/5ths rule (DIR >= 0.80) is a common regulatory threshold. - - Args: - y_pred: Binary predictions (0/1). - groups: Group labels per observation. - - Returns: - Dict with dir, numerator_group, denominator_group. - """ - y_pred = np.asarray(y_pred, dtype=float) - groups = np.asarray(groups) - unique_groups = np.unique(groups) - - group_rates: dict[str, float] = {} - for g in unique_groups: - mask = groups == g - group_rates[str(g)] = float(y_pred[mask].mean()) if mask.sum() > 0 else 0.0 - - min_ratio = float("inf") - num_group, den_group = "", "" - - for gi, ri in group_rates.items(): - for gj, rj in group_rates.items(): - if gi == gj: - continue - ratio = ri / (rj + _EPS) - if ratio < min_ratio: - min_ratio = ratio - num_group, den_group = gi, gj - - if min_ratio == float("inf"): - min_ratio = 1.0 - - return { - "dir": min_ratio, - "numerator_group": num_group, - "denominator_group": den_group, - } - - -def fairness_report( - y_true: np.ndarray, - y_pred_proba: np.ndarray, - groups_dict: dict[str, np.ndarray], - threshold: float = 0.5, - dpd_threshold: float = 0.10, - eo_gap_threshold: float = 0.10, - dir_threshold: float = 0.80, -) -> pd.DataFrame: - """Run all fairness metrics for multiple group attribute definitions.""" - y_true = np.asarray(y_true, dtype=float) - y_pred_binary = (np.asarray(y_pred_proba, dtype=float) >= threshold).astype(float) - return fairness_report_from_binary( - y_true=y_true, - y_pred_binary=y_pred_binary, - groups_dict=groups_dict, - dpd_threshold=dpd_threshold, - eo_gap_threshold=eo_gap_threshold, - dir_threshold=dir_threshold, - ) - - -def fairness_report_from_binary( - y_true: np.ndarray, - y_pred_binary: np.ndarray, - groups_dict: dict[str, np.ndarray], - dpd_threshold: float = 0.10, - eo_gap_threshold: float = 0.10, - dir_threshold: float = 0.80, -) -> pd.DataFrame: - """Run all fairness metrics for multiple group attribute definitions. - - Args: - y_true: Binary ground truth (0/1). - y_pred_proba: Predicted probabilities. - groups_dict: Mapping of attribute name to group labels array. - threshold: Probability cutoff for binarization. - dpd_threshold: Maximum acceptable DPD. - eo_gap_threshold: Maximum acceptable EO gap. - dir_threshold: Minimum acceptable DIR (4/5ths rule). - - Returns: - DataFrame with one row per attribute: attribute, dpd, eo_gap, dir, - passed_dpd, passed_eo, passed_dir, passed_all. - """ - y_true = np.asarray(y_true, dtype=float) - y_pred_binary = np.asarray(y_pred_binary, dtype=float) - - rows: list[dict] = [] - for attr_name, groups in groups_dict.items(): - groups = np.asarray(groups) - - dpd_result = demographic_parity_difference(y_pred_binary, groups) - eo_result = equalized_odds_gap(y_true, y_pred_binary, groups) - dir_result = disparate_impact_ratio(y_pred_binary, groups) - - passed_dpd = dpd_result["dpd"] < dpd_threshold - passed_eo = eo_result["eo_gap"] < eo_gap_threshold - passed_dir = dir_result["dir"] > dir_threshold - - rows.append( - { - "attribute": attr_name, - "dpd": dpd_result["dpd"], - "eo_gap": eo_result["eo_gap"], - "dir": dir_result["dir"], - "tpr_gap": eo_result["tpr_gap"], - "fpr_gap": eo_result["fpr_gap"], - "passed_dpd": passed_dpd, - "passed_eo": passed_eo, - "passed_dir": passed_dir, - "passed_all": passed_dpd and passed_eo and passed_dir, - } - ) - - logger.info(f"Fairness report: {len(rows)} attributes evaluated") - return pd.DataFrame(rows) - - -def build_intersectional_groups( - groups_dict: dict[str, np.ndarray], - *, - max_order: int = 2, - min_group_size: int = 200, -) -> dict[str, np.ndarray]: - """Build pairwise intersectional groups from already-resolved attribute arrays.""" - items = [(str(name), np.asarray(values).astype(str)) for name, values in groups_dict.items()] - if not items or max_order < 2: - return {} - - n_rows = len(items[0][1]) - out: dict[str, np.ndarray] = {} - for i in range(len(items)): - for j in range(i + 1, len(items)): - name_i, values_i = items[i] - name_j, values_j = items[j] - if len(values_i) != n_rows or len(values_j) != n_rows: - continue - labels = np.array( - [f"{left} | {right}" for left, right in zip(values_i, values_j, strict=False)], - dtype=object, - ) - counts = pd.Series(labels).value_counts() - allowed = set(counts[counts >= int(min_group_size)].index.astype(str).tolist()) - collapsed = np.array( - [label if label in allowed else "OTHER_SMALL_GROUP" for label in labels], - dtype=object, - ) - out[f"{name_i}__x__{name_j}"] = collapsed - return out - - -def fairness_threshold_frontier( - y_true: np.ndarray, - y_pred_proba: np.ndarray, - groups_dict: dict[str, np.ndarray], - *, - thresholds: list[float], - primary_threshold: float, - dpd_threshold: float = 0.10, - eo_gap_threshold: float = 0.10, - dir_threshold: float = 0.80, -) -> pd.DataFrame: - """Evaluate fairness metrics across multiple thresholds.""" - rows: list[dict[str, float | str | bool]] = [] - for threshold in thresholds: - report = fairness_report( - y_true=y_true, - y_pred_proba=y_pred_proba, - groups_dict=groups_dict, - threshold=float(threshold), - dpd_threshold=dpd_threshold, - eo_gap_threshold=eo_gap_threshold, - dir_threshold=dir_threshold, - ) - if report.empty: - continue - for _, row in report.iterrows(): - rows.append( - { - "attribute": str(row["attribute"]), - "threshold": float(threshold), - "is_primary_threshold": bool( - abs(float(threshold) - float(primary_threshold)) < 1e-9 - ), - "dpd": float(row["dpd"]), - "eo_gap": float(row["eo_gap"]), - "dir": float(row["dir"]), - "passed_dpd": bool(row["passed_dpd"]), - "passed_eo": bool(row["passed_eo"]), - "passed_dir": bool(row["passed_dir"]), - "passed_all": bool(row["passed_all"]), - } - ) - if not rows: - return pd.DataFrame( - { - "attribute": pd.Series(dtype="object"), - "threshold": pd.Series(dtype="float64"), - "is_primary_threshold": pd.Series(dtype="bool"), - "dpd": pd.Series(dtype="float64"), - "eo_gap": pd.Series(dtype="float64"), - "dir": pd.Series(dtype="float64"), - "passed_dpd": pd.Series(dtype="bool"), - "passed_eo": pd.Series(dtype="bool"), - "passed_dir": pd.Series(dtype="bool"), - "passed_all": pd.Series(dtype="bool"), - } - ) - return pd.DataFrame(rows).sort_values(["attribute", "threshold"]).reset_index(drop=True) - - -def conformal_fairness_report( - y_true: np.ndarray, - y_intervals: np.ndarray, - groups_dict: dict[str, np.ndarray], - alpha: float = 0.10, - coverage_disparity_threshold: float = 0.05, - width_ratio_threshold: float = 2.0, -) -> pd.DataFrame: - """Evaluate conformal interval fairness across protected groups. - - Checks whether conformal intervals exhibit disparate coverage or width - across groups. Coverage disparity indicates that some groups receive - weaker uncertainty guarantees. Width disparity indicates that some - groups receive less informative (wider) intervals. - - Args: - y_true: Ground truth values (float). - y_intervals: Prediction intervals array of shape (n, 2) — [low, high]. - groups_dict: Mapping of attribute name to group labels array. - alpha: Nominal significance level (for reference). - coverage_disparity_threshold: Max acceptable coverage gap across groups. - width_ratio_threshold: Max acceptable max_width/min_width ratio. - - Returns: - DataFrame with one row per attribute: attribute, n_groups, - min_coverage, max_coverage, coverage_disparity, min_avg_width, - max_avg_width, width_ratio, passed_coverage, passed_width, passed_all, - group_details (dict). - """ - y_true = np.asarray(y_true, dtype=float) - low = y_intervals[:, 0] - high = y_intervals[:, 1] - covered = (y_true >= low) & (y_true <= high) - widths = high - low - - rows: list[dict] = [] - for attr_name, groups in groups_dict.items(): - groups = np.asarray(groups) - unique_groups = np.unique(groups) - - group_details: dict[str, dict[str, float]] = {} - coverages: list[float] = [] - avg_widths: list[float] = [] - - for g in unique_groups: - mask = groups == g - n_g = int(mask.sum()) - if n_g == 0: - continue - cov_g = float(covered[mask].mean()) - w_g = float(widths[mask].mean()) - group_details[str(g)] = { - "n": n_g, - "coverage": cov_g, - "avg_width": w_g, - } - coverages.append(cov_g) - avg_widths.append(w_g) - - if not coverages: - continue - - min_cov = min(coverages) - max_cov = max(coverages) - cov_disp = max_cov - min_cov - min_w = min(avg_widths) - max_w = max(avg_widths) - w_ratio = max_w / max(min_w, _EPS) - - passed_cov = cov_disp <= coverage_disparity_threshold - passed_w = w_ratio <= width_ratio_threshold - - rows.append( - { - "attribute": attr_name, - "n_groups": len(unique_groups), - "min_coverage": min_cov, - "max_coverage": max_cov, - "coverage_disparity": cov_disp, - "min_avg_width": min_w, - "max_avg_width": max_w, - "width_ratio": w_ratio, - "passed_coverage": passed_cov, - "passed_width": passed_w, - "passed_all": passed_cov and passed_w, - "group_details": group_details, - } - ) - - logger.info(f"Conformal fairness report: {len(rows)} attributes evaluated") - return pd.DataFrame(rows) diff --git a/src/evaluation/maturity_safe_portfolio.py b/src/evaluation/maturity_safe_portfolio.py new file mode 100644 index 0000000..cf18a1b --- /dev/null +++ b/src/evaluation/maturity_safe_portfolio.py @@ -0,0 +1,557 @@ +"""Outcome-isolated portfolio decisions and bounded retrospective evaluation.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.evaluation.coverage_transport import binary_miscoverage_bounds +from src.evaluation.standardized_credit_payoff import ( + PAYOFF_ID, + contractual_rate_decimal, + expected_objective_coefficients, + realized_standardized_payoff_bounds, +) +from src.models.maturity_safe_pd import OUTCOME_COLUMNS +from src.optimization.policy_evaluation import PolicyAllocationResult, solve_policy_allocation +from src.optimization.policy_selection import LinearPolicyCandidate + + +@dataclass(frozen=True) +class SolveResult: + """A solved portfolio plus independently reconciled payoff quantities.""" + + result: PolicyAllocationResult + exposure: np.ndarray + expected_payoff_rate: np.ndarray + expected_objective: float + + +@dataclass(frozen=True) +class MonthlyPolicySpec: + """One frozen policy evaluated under a stable publication label.""" + + candidate: LinearPolicyCandidate + robust: bool + label: str + + +def assert_outcome_free_decision_frame(frame: pd.DataFrame) -> None: + """Fail if a decision frame contains an outcome or outcome-derived field.""" + normalized = {str(column).casefold() for column in frame.columns} + forbidden = sorted(OUTCOME_COLUMNS.intersection(normalized)) + pattern_forbidden = sorted( + column + for column in normalized + if any(token in column for token in ("realized", "miscoverage", "outcome")) + ) + violations = sorted(set(forbidden + pattern_forbidden)) + if violations: + raise ValueError(f"Decision frame contains outcome-derived columns: {violations}") + + +def build_decision_panel( + frame: pd.DataFrame, + *, + pd_point: np.ndarray, + conformal_lower: np.ndarray, + conformal_upper: np.ndarray, + conformal_groups: np.ndarray, +) -> pd.DataFrame: + """Build the only dataframe supplied to policy solving.""" + n_rows = len(frame) + arrays = [pd_point, conformal_lower, conformal_upper, conformal_groups] + if any(len(values) != n_rows for values in arrays): + raise ValueError("Decision-panel prediction arrays must align with candidates.") + loan_amount = pd.to_numeric(frame["loan_amnt"], errors="coerce").to_numpy(dtype=float) + if not bool(np.isfinite(loan_amount).all()) or bool(np.any(loan_amount <= 0.0)): + raise ValueError("Decision candidates require positive finite loan_amnt.") + panel = pd.DataFrame( + { + "id": frame["id"].astype("string").to_numpy(), + "issue_d": pd.to_datetime(frame["issue_d"]).to_numpy(), + "loan_amnt": loan_amount, + "purpose": frame["purpose"].astype("string").fillna("unknown").to_numpy(), + "contractual_rate": contractual_rate_decimal(frame["int_rate"]), + "pd_point": np.asarray(pd_point, dtype=float), + "conformal_lower": np.asarray(conformal_lower, dtype=float), + "conformal_upper": np.asarray(conformal_upper, dtype=float), + "conformal_group": np.asarray(conformal_groups, dtype=int), + } + ) + probability_columns = ["pd_point", "conformal_lower", "conformal_upper"] + probability_values = panel[probability_columns].to_numpy(dtype=float) + if not bool(np.isfinite(probability_values).all()): + raise ValueError("Decision panel contains non-finite probability values.") + if bool(np.any((probability_values < 0.0) | (probability_values > 1.0))): + raise ValueError("Decision scores must lie in [0, 1].") + if bool(np.any(panel["conformal_lower"] > panel["pd_point"] + 1e-12)) or bool( + np.any(panel["pd_point"] > panel["conformal_upper"] + 1e-12) + ): + raise ValueError("Binary-outcome conformal intervals must contain the point score.") + assert_outcome_free_decision_frame(panel) + return panel + + +def build_outcome_panel(frame: pd.DataFrame) -> pd.DataFrame: + """Build the ID-keyed snapshot outcomes kept outside decisions.""" + return pd.DataFrame( + { + "id": frame["id"].astype("string").to_numpy(), + "loan_status": frame["loan_status"].astype("string").to_numpy(), + "snapshot_default": frame["snapshot_default"].astype("Int8").to_numpy(), + "snapshot_resolution": frame["snapshot_resolution"].astype("string").to_numpy(), + } + ) + + +def solve_coherent_policy( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: Mapping[str, Any], + robust: bool, + effective_score_override: np.ndarray | None = None, +) -> SolveResult: + """Optimize and independently reconcile ``(1-p)r-pLGD``.""" + assert_outcome_free_decision_frame(frame) + payoff = config["payoff"] + policy = config["policy"] + execution = config["execution"] + point = frame["pd_point"].to_numpy(dtype=float) + rates = frame["contractual_rate"].to_numpy(dtype=float) + lgd_value = float(payoff["lgd"]) + objective_coefficients = expected_objective_coefficients(point, rates, lgd=lgd_value) + requested_backend = str(execution["solver_backend"]) + result = solve_policy_allocation( + loans=frame, + pd_point=point, + pd_low=frame["conformal_lower"].to_numpy(dtype=float), + pd_high=frame["conformal_upper"].to_numpy(dtype=float), + lgd=np.full(len(frame), lgd_value, dtype=float), + int_rates=rates, + objective_rate_override=objective_coefficients, + total_budget=float(policy["budget"]), + max_concentration=float(policy["max_concentration_by_purpose"]), + risk_tolerance=float(candidate.risk_tolerance), + robust=bool(robust), + pd_constraint_override=effective_score_override, + uncertainty_aversion=0.0, + min_budget_utilization=float(policy["min_budget_utilization_solver"]), + pd_cap_slack_penalty=0.0, + policy_mode=candidate.policy_mode, + gamma=float(candidate.gamma), + delta_cap_quantile=float(candidate.delta_cap_quantile), + tail_focus_quantile=float(candidate.tail_focus_quantile), + time_limit=int(execution["solver_time_limit_seconds"]), + threads=int(execution["threads"]), + solver_backend=requested_backend, + random_seed=int(execution["random_seed"]), + ) + _require_requested_solver_backend( + requested=requested_backend, + actual=str(result.solution.get("solver_backend", "unknown")), + strict=bool(execution.get("strict_solver_backend", False)), + ) + exposure = result.allocation * frame["loan_amnt"].to_numpy(dtype=float) + payoff_rate = objective_coefficients + reconciled = float(exposure @ payoff_rate) + solver_objective = float(result.solution.get("objective_value", np.nan)) + if not np.isfinite(solver_objective) or not np.isclose( + solver_objective, + reconciled, + rtol=0.0, + atol=1e-5, + ): + raise RuntimeError( + "Solver objective does not reconcile to coherent payoff: " + f"solver={solver_objective}, independent={reconciled}." + ) + return SolveResult( + result=result, + exposure=exposure, + expected_payoff_rate=payoff_rate, + expected_objective=reconciled, + ) + + +def _require_requested_solver_backend(*, requested: str, actual: str, strict: bool) -> None: + if not strict: + return + requested_normalized = requested.strip().casefold() + actual_normalized = actual.strip().casefold() + canonical_actual = { + "highs": "highs_sparse", + "scipy_highs": "highs_sparse", + "highs_sparse": "highs_sparse", + "highspy": "highspy", + "highs_native": "highspy", + "native_highs": "highspy", + "highs_pyomo": "highs", + "pyomo_highs": "highs", + "cuopt": "cuopt", + }.get(requested_normalized, requested_normalized) + if actual_normalized != canonical_actual: + raise RuntimeError( + f"Strict solver backend mismatch: requested={requested!r}, actual={actual!r}." + ) + + +def outcome_free_solve_record( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + solved: SolveResult, + *, + allocation_tolerance: float, +) -> dict[str, Any]: + """Summarize one solve using decision-time quantities only.""" + total = float(solved.exposure.sum()) + if total <= 0.0: + raise RuntimeError(f"Policy {candidate.candidate_id} allocated no capital.") + weights = solved.exposure / total + active = solved.exposure > float(allocation_tolerance) + return { + **candidate.to_record(), + "payoff_id": PAYOFF_ID, + "solver_status": str(solved.result.solution.get("solver_status", "unknown")), + "solver_backend_actual": str(solved.result.solution.get("solver_backend", "unknown")), + "expected_objective": solved.expected_objective, + "n_candidates": int(len(frame)), + "n_positive_exposure": int(active.sum()), + "total_allocated": total, + "weighted_pd_point": float(weights @ frame["pd_point"].to_numpy(dtype=float)), + "weighted_pd_effective": float(weights @ solved.result.effective_pd), + "weighted_conformal_upper": float(weights @ frame["conformal_upper"].to_numpy(dtype=float)), + "highs_simplex_iterations": int( + solved.result.solution.get("highs_simplex_iterations", 0) or 0 + ), + "highs_ipm_iterations": int(solved.result.solution.get("highs_ipm_iterations", 0) or 0), + } + + +def solve_outcome_free_allocation( + decision_frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: Mapping[str, Any], + robust: bool, + role: str, + period: str, + policy_label: str, + effective_score_override: np.ndarray | None = None, +) -> tuple[dict[str, Any], pd.DataFrame]: + """Solve and materialize an allocation before any outcome join.""" + assert_outcome_free_decision_frame(decision_frame) + solved = solve_coherent_policy( + decision_frame, + candidate, + config=config, + robust=robust, + effective_score_override=effective_score_override, + ) + tolerance = float(config["execution"]["allocation_tolerance"]) + record = outcome_free_solve_record( + decision_frame, + candidate, + solved, + allocation_tolerance=tolerance, + ) + active = solved.exposure > tolerance + allocation = decision_frame.loc[active].copy() + allocation["allocation_fraction"] = solved.result.allocation[active] + allocation["exposure"] = solved.exposure[active] + total = float(allocation["exposure"].sum()) + allocation["weight"] = allocation["exposure"] / total + allocation["pd_effective"] = solved.result.effective_pd[active] + allocation["expected_payoff_rate"] = solved.expected_payoff_rate[active] + allocation["expected_payoff_contribution"] = ( + allocation["exposure"] * allocation["expected_payoff_rate"] + ) + allocation["role"] = role + allocation["period"] = period + allocation["policy_label"] = policy_label + allocation["candidate_id"] = candidate.candidate_id + return { + **record, + "role": role, + "period": period, + "policy_label": policy_label, + "robust_guardrail": bool(robust), + }, allocation + + +def evaluation_record_and_allocations( + decision_frame: pd.DataFrame, + outcomes: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: Mapping[str, Any], + robust: bool, + role: str, + period: str, + policy_label: str, +) -> tuple[dict[str, Any], pd.DataFrame]: + """Solve outcome-free, then join outcomes for bounded evaluation.""" + base_record, allocation = solve_outcome_free_allocation( + decision_frame, + candidate, + config=config, + robust=robust, + role=role, + period=period, + policy_label=policy_label, + ) + return evaluate_frozen_allocation(base_record, allocation, outcomes, config=config) + + +def evaluate_frozen_allocation( + base_record: Mapping[str, Any], + allocation: pd.DataFrame, + outcomes: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> tuple[dict[str, Any], pd.DataFrame]: + """Join outcomes to an already frozen allocation and compute sharp bounds.""" + assert_outcome_free_decision_frame( + allocation.drop( + columns=[ + "allocation_fraction", + "exposure", + "weight", + "pd_effective", + "expected_payoff_rate", + "expected_payoff_contribution", + "role", + "period", + "policy_label", + "candidate_id", + ], + errors="ignore", + ) + ) + joined = allocation.merge(outcomes, on="id", how="left", validate="one_to_one") + return evaluate_prejoined_frozen_allocation( + base_record, + joined, + config=config, + n_unresolved_candidates=int(outcomes["snapshot_default"].isna().sum()), + ) + + +def evaluate_prejoined_frozen_allocation( + base_record: Mapping[str, Any], + joined_allocation: pd.DataFrame, + *, + config: Mapping[str, Any], + n_unresolved_candidates: int, +) -> tuple[dict[str, Any], pd.DataFrame]: + """Evaluate one frozen allocation after a validated shared outcome join.""" + joined = joined_allocation.copy() + if bool(joined["id"].duplicated().any()): + raise RuntimeError("A frozen policy allocation contains duplicate loan IDs.") + if bool(joined["snapshot_resolution"].isna().any()): + raise RuntimeError("Funded allocation could not be aligned to snapshot outcomes.") + y_true = pd.to_numeric(joined["snapshot_default"], errors="coerce").to_numpy(dtype=float) + payoff_low, payoff_high = realized_standardized_payoff_bounds( + y_true, + joined["contractual_rate"].to_numpy(dtype=float), + lgd=float(config["payoff"]["lgd"]), + ) + miss_low, miss_high = binary_miscoverage_bounds( + y_true, + joined["conformal_lower"].to_numpy(dtype=float), + joined["conformal_upper"].to_numpy(dtype=float), + ) + joined["realized_payoff_rate_lower"] = payoff_low + joined["realized_payoff_rate_upper"] = payoff_high + joined["realized_payoff_lower"] = joined["exposure"] * payoff_low + joined["realized_payoff_upper"] = joined["exposure"] * payoff_high + joined["miscoverage_lower"] = miss_low + joined["miscoverage_upper"] = miss_high + weights = joined["weight"].to_numpy(dtype=float) + unresolved = ~np.isfinite(y_true) + default_lower = np.nan_to_num(y_true, nan=0.0) + default_upper = np.where(unresolved, 1.0, y_true) + realized_lower = float(joined["realized_payoff_lower"].sum()) + realized_upper = float(joined["realized_payoff_upper"].sum()) + role = str(base_record["role"]) + period = str(base_record["period"]) + policy_label = str(base_record["policy_label"]) + robust = bool(base_record["robust_guardrail"]) + record = { + **base_record, + "role": role, + "period": period, + "policy_label": policy_label, + "robust_guardrail": bool(robust), + "n_unresolved_candidates": int(n_unresolved_candidates), + "n_unresolved_positive_exposure": int(unresolved.sum()), + "unresolved_exposure_share": float(weights @ unresolved.astype(float)), + "realized_payoff_lower": realized_lower, + "realized_payoff_upper": realized_upper, + "realized_payoff_exact": ( + realized_lower if np.isclose(realized_lower, realized_upper, atol=1e-12) else None + ), + "weighted_default_lower": float(weights @ default_lower), + "weighted_default_upper": float(weights @ default_upper), + "weighted_miscoverage_lower": float(weights @ miss_low), + "weighted_miscoverage_upper": float(weights @ miss_high), + "full_budget": bool( + np.isclose( + float(base_record["total_allocated"]), + float(config["policy"]["budget"]), + rtol=0.0, + atol=1e-4, + ) + ), + } + return record, joined + + +def aggregate_monthly_evaluation(frame: pd.DataFrame) -> dict[str, Any]: + """Aggregate fixed-policy months using allocated capital as weights.""" + if frame.empty: + raise ValueError("Monthly evaluation frame is empty.") + budget = pd.to_numeric(frame["total_allocated"], errors="raise").to_numpy(dtype=float) + total_budget = float(budget.sum()) + if total_budget <= 0.0: + raise ValueError("Monthly evaluation has no allocated budget.") + month_weights = budget / total_budget + weighted_columns = [ + "weighted_pd_point", + "weighted_pd_effective", + "weighted_conformal_upper", + "weighted_default_lower", + "weighted_default_upper", + "weighted_miscoverage_lower", + "weighted_miscoverage_upper", + "unresolved_exposure_share", + ] + result: dict[str, Any] = { + "policy_label": str(frame["policy_label"].iloc[0]), + "months": int(len(frame)), + "total_budget": total_budget, + "expected_objective": float(frame["expected_objective"].sum()), + "realized_payoff_lower": float(frame["realized_payoff_lower"].sum()), + "realized_payoff_upper": float(frame["realized_payoff_upper"].sum()), + "unresolved_candidates": int(frame["n_unresolved_candidates"].sum()), + "unresolved_positive_exposure": int(frame["n_unresolved_positive_exposure"].sum()), + } + for column in weighted_columns: + values = pd.to_numeric(frame[column], errors="raise").to_numpy(dtype=float) + result[column] = float(month_weights @ values) + return result + + +def evaluate_policy_specs_by_month( + decision_frame: pd.DataFrame, + outcomes: pd.DataFrame, + specs: Sequence[MonthlyPolicySpec], + *, + config: Mapping[str, Any], + role: str, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Evaluate frozen policy specs on each issue-month menu in stable order.""" + assert_outcome_free_decision_frame(decision_frame) + if not specs: + raise ValueError("At least one monthly policy spec is required.") + labels = [spec.label for spec in specs] + if len(labels) != len(set(labels)): + raise ValueError("Monthly policy labels must be unique.") + + periods = sorted(pd.to_datetime(decision_frame["issue_d"]).dt.to_period("M").unique()) + records: list[dict[str, Any]] = [] + allocations: list[pd.DataFrame] = [] + for period_value in periods: + period = str(period_value) + month_mask = pd.to_datetime(decision_frame["issue_d"]).dt.to_period("M").eq(period_value) + month = decision_frame.loc[month_mask].copy() + month_outcomes = outcomes.loc[outcomes["id"].isin(month["id"])].copy() + for spec in specs: + record, funded = evaluation_record_and_allocations( + month, + month_outcomes, + spec.candidate, + config=config, + robust=spec.robust, + role=role, + period=period, + policy_label=spec.label, + ) + records.append(record) + allocations.append(funded) + + evaluation = pd.DataFrame(records) + funded = pd.concat(allocations, ignore_index=True) + expected_rows = len(periods) * len(specs) + if len(evaluation) != expected_rows: + raise RuntimeError( + f"Monthly evaluation produced {len(evaluation)} rows; expected {expected_rows}." + ) + return evaluation, funded + + +def select_policy_on_development( + decision_frame: pd.DataFrame, + outcomes: pd.DataFrame, + candidates: Sequence[tuple[LinearPolicyCandidate, bool, str]], + *, + config: Mapping[str, Any], +) -> tuple[LinearPolicyCandidate, pd.DataFrame, pd.DataFrame]: + """Select once on labeled development months by realized coherent payoff.""" + assert_outcome_free_decision_frame(decision_frame) + periods = sorted(pd.to_datetime(decision_frame["issue_d"]).dt.to_period("M").unique()) + monthly_records: list[dict[str, Any]] = [] + for candidate, robust, label in candidates: + for period_value in periods: + period = str(period_value) + month_mask = ( + pd.to_datetime(decision_frame["issue_d"]).dt.to_period("M").eq(period_value) + ) + month = decision_frame.loc[month_mask].copy() + month_outcomes = outcomes.loc[outcomes["id"].isin(month["id"])].copy() + record, _ = evaluation_record_and_allocations( + month, + month_outcomes, + candidate, + config=config, + robust=robust, + role="policy_development", + period=period, + policy_label=label, + ) + if record["realized_payoff_exact"] is None: + raise RuntimeError("Policy development contains unresolved outcomes.") + monthly_records.append(record) + monthly = pd.DataFrame(monthly_records) + grid = ( + monthly.groupby(["candidate_id", "policy_label"], observed=True, as_index=False) + .agg( + months=("period", "size"), + full_budget_months=("full_budget", "sum"), + total_allocated=("total_allocated", "sum"), + expected_objective=("expected_objective", "sum"), + realized_payoff=("realized_payoff_exact", "sum"), + weighted_default=("weighted_default_lower", "mean"), + weighted_miscoverage=("weighted_miscoverage_lower", "mean"), + ) + .sort_values( + ["realized_payoff", "expected_objective", "candidate_id"], + ascending=[False, False, True], + kind="mergesort", + ) + .reset_index(drop=True) + ) + expected_months = len(periods) + eligible = grid.loc[grid["full_budget_months"].eq(expected_months)] + if eligible.empty: + raise RuntimeError("No development policy used the full budget in every month.") + selected_id = str(eligible.iloc[0]["candidate_id"]) + selected = next( + candidate for candidate, _, _ in candidates if candidate.candidate_id == selected_id + ) + return selected, grid, monthly diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py deleted file mode 100644 index 92c3ebb..0000000 --- a/src/evaluation/metrics.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Comprehensive evaluation metrics for credit risk models. - -Classification: AUC-ROC, KS statistic, Gini, Brier score, ECE. -Regression: MAE, RMSE, R². -Conformal: Coverage, efficiency, singleton rate. -Survival: C-index. -""" - -from __future__ import annotations - -import numpy as np -from sklearn.metrics import ( - average_precision_score, - brier_score_loss, - f1_score, - log_loss, - mean_absolute_error, - mean_squared_error, - r2_score, - recall_score, - roc_auc_score, -) - -try: - from sklearn.metrics import d2_brier_score -except ImportError: # sklearn < 1.8 - d2_brier_score = None - - -def classification_metrics( - y_true: np.ndarray, - y_prob: np.ndarray, - threshold: float = 0.35, -) -> dict[str, float]: - """Compute all classification metrics for PD model. - - Args: - y_true: Binary ground truth (1 = default). - y_prob: Predicted default probability. - threshold: Decision threshold for binary metrics (recall, f1). - Defaults to 0.35, the operational threshold. - """ - from src.models.calibration import expected_calibration_error - - auc = roc_auc_score(y_true, y_prob) - gini = 2 * auc - 1 - brier = brier_score_loss(y_true, y_prob) - ece = expected_calibration_error(y_true, y_prob) - ks = ks_statistic(y_true, y_prob) - pr_auc = average_precision_score(y_true, y_prob) - y_pred_binary = (np.asarray(y_prob) >= threshold).astype(int) - recall_at_t = recall_score(y_true, y_pred_binary, zero_division=0) - f1_at_t = f1_score(y_true, y_pred_binary, zero_division=0) - - logloss = log_loss(y_true, y_prob) - - metrics: dict[str, float] = { - "auc_roc": auc, - "gini": gini, - "brier_score": brier, - "log_loss": logloss, - "ece": ece, - "ks_statistic": ks, - "pr_auc": pr_auc, - f"recall_at_{threshold:.2f}".replace(".", "p"): recall_at_t, - f"f1_at_{threshold:.2f}".replace(".", "p"): f1_at_t, - } - if d2_brier_score is not None: - metrics["d2_brier_score"] = float(d2_brier_score(y_true, y_prob)) - return metrics - - -def ks_statistic(y_true: np.ndarray, y_prob: np.ndarray) -> float: - """Kolmogorov-Smirnov statistic for discriminatory power.""" - from scipy.stats import ks_2samp - - defaults = y_prob[y_true == 1] - non_defaults = y_prob[y_true == 0] - ks_stat, _ = ks_2samp(defaults, non_defaults) - return ks_stat - - -def brier_score_decomposition( - y_true: np.ndarray, - y_prob: np.ndarray, - n_bins: int = 10, -) -> dict[str, float]: - """Decompose Brier score into reliability, resolution, and uncertainty. - - Murphy (1973) decomposition: - Brier = Reliability - Resolution + Uncertainty - - - Reliability (calibration): lower is better. Measures how close - predicted probabilities are to observed frequencies per bin. - - Resolution (discrimination): higher is better. Measures how much - bin-level frequencies deviate from the overall base rate. - - Uncertainty: irreducible, depends only on the base rate. - - Args: - y_true: Binary ground truth. - y_prob: Predicted probabilities. - n_bins: Number of calibration bins. - - Returns: - Dict with brier, reliability, resolution, uncertainty, - and normalized miscalibration_share. - """ - y_true = np.asarray(y_true, dtype=float) - y_prob = np.asarray(y_prob, dtype=float) - n = len(y_true) - - base_rate = y_true.mean() - uncertainty = base_rate * (1 - base_rate) - - bin_edges = np.linspace(0, 1, n_bins + 1) - reliability = 0.0 - resolution = 0.0 - - for i in range(n_bins): - mask = (y_prob >= bin_edges[i]) & (y_prob < bin_edges[i + 1]) - if i == n_bins - 1: - mask = mask | (y_prob == bin_edges[i + 1]) - n_k = mask.sum() - if n_k == 0: - continue - avg_pred = y_prob[mask].mean() - avg_true = y_true[mask].mean() - reliability += n_k * (avg_pred - avg_true) ** 2 - resolution += n_k * (avg_true - base_rate) ** 2 - - reliability /= n - resolution /= n - brier = reliability - resolution + uncertainty - - return { - "brier_decomposed": float(brier), - "reliability": float(reliability), - "resolution": float(resolution), - "uncertainty": float(uncertainty), - "miscalibration_share": float(reliability / max(brier, 1e-10)), - "discrimination_share": float(resolution / max(brier, 1e-10)), - } - - -def regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]: - """Compute regression metrics for LGD/EAD models.""" - return { - "mae": mean_absolute_error(y_true, y_pred), - "rmse": np.sqrt(mean_squared_error(y_true, y_pred)), - "r2": r2_score(y_true, y_pred), - } - - -def conformal_metrics( - y_true: np.ndarray, - y_intervals: np.ndarray, - alpha: float, -) -> dict[str, float]: - """Compute conformal prediction quality metrics. - - Args: - y_intervals: Shape (n, 2) — columns [lower, upper]. - Compatible with MAPIE >=1.3 predict_interval output. - """ - low = y_intervals[:, 0] - high = y_intervals[:, 1] - widths = high - low - - covered = (y_true >= low) & (y_true <= high) - - return { - "empirical_coverage": covered.mean(), - "target_coverage": 1 - alpha, - "coverage_gap": abs(covered.mean() - (1 - alpha)), - "avg_width": widths.mean(), - "median_width": np.median(widths), - "width_std": widths.std(), - "width_90th_pct": np.percentile(widths, 90), - } - - -def compute_all_metrics( - y_true: np.ndarray, - y_prob: np.ndarray, - y_intervals: np.ndarray | None = None, - alpha: float = 0.1, -) -> dict[str, float]: - """Compute all relevant metrics in one call.""" - metrics = classification_metrics(y_true, y_prob) - if y_intervals is not None: - metrics.update(conformal_metrics(y_true, y_intervals, alpha)) - return metrics - - -def forecast_backtest_metrics( - forecast_values: np.ndarray, - actual_values: np.ndarray, - forecast_lo: np.ndarray | None = None, - forecast_hi: np.ndarray | None = None, -) -> dict[str, float]: - """Evaluate time series forecasts against realized actuals. - - Useful for validating IFRS9 forward-looking scenarios against - observed default rates in the OOT test period (2018-2020). - - Args: - forecast_values: Point forecasts. - actual_values: Realized values. - forecast_lo: Lower prediction interval (optional). - forecast_hi: Upper prediction interval (optional). - - Returns: - Dict with MAE, RMSE, directional accuracy, and optionally - interval coverage and width. - """ - forecast_values = np.asarray(forecast_values, dtype=float) - actual_values = np.asarray(actual_values, dtype=float) - - metrics: dict[str, float] = { - "forecast_mae": float(mean_absolute_error(actual_values, forecast_values)), - "forecast_rmse": float(np.sqrt(mean_squared_error(actual_values, forecast_values))), - } - - # Directional accuracy: did forecast correctly predict up/down movement? - if len(actual_values) > 1: - actual_direction = np.sign(np.diff(actual_values)) - forecast_direction = np.sign(np.diff(forecast_values)) - directional_match = actual_direction == forecast_direction - metrics["directional_accuracy"] = float(directional_match.mean()) - - # Mean bias (positive = over-forecast) - metrics["mean_bias"] = float((forecast_values - actual_values).mean()) - - # Interval coverage if bounds provided - if forecast_lo is not None and forecast_hi is not None: - forecast_lo = np.asarray(forecast_lo, dtype=float) - forecast_hi = np.asarray(forecast_hi, dtype=float) - covered = (actual_values >= forecast_lo) & (actual_values <= forecast_hi) - metrics["interval_coverage"] = float(covered.mean()) - metrics["avg_interval_width"] = float((forecast_hi - forecast_lo).mean()) - - return metrics diff --git a/src/evaluation/model_shift.py b/src/evaluation/model_shift.py deleted file mode 100644 index b9f9d57..0000000 --- a/src/evaluation/model_shift.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Interpretive helpers for structural shift and p-value semantics.""" - -from __future__ import annotations - -from typing import Any - - -def interpret_model_shift( - *, - c2st_auc: float, - c2st_materiality: str, - score_psi: float, - auc_delta: float, - brier_increase: float, - calibration_gap_delta: float, - distribution_warning_ratio: float, - score_psi_max: float, - auc_delta_max: float, - brier_increase_max: float, - calibration_gap_delta_max: float, -) -> dict[str, Any]: - """Distinguish structural shift from predictive degradation.""" - structural_level = _structural_shift_level( - c2st_auc=c2st_auc, - score_psi=score_psi, - distribution_warning_ratio=distribution_warning_ratio, - score_psi_max=score_psi_max, - ) - predictive_level = _predictive_degradation_level( - auc_delta=auc_delta, - brier_increase=brier_increase, - calibration_gap_delta=calibration_gap_delta, - auc_delta_max=auc_delta_max, - brier_increase_max=brier_increase_max, - calibration_gap_delta_max=calibration_gap_delta_max, - ) - shift_type = _shift_type(structural_level, predictive_level) - - return { - "shift_type": shift_type, - "structural_shift_level": structural_level, - "predictive_degradation_level": predictive_level, - "governance_posture": _governance_posture(shift_type, predictive_level), - "c2st_materiality": str(c2st_materiality), - "pvalue_interpretation": _pvalue_note(shift_type), - } - - -def _structural_shift_level( - *, - c2st_auc: float, - score_psi: float, - distribution_warning_ratio: float, - score_psi_max: float, -) -> str: - if c2st_auc >= 0.70 or score_psi >= max(score_psi_max * 1.5, 0.20): - return "severe" - if c2st_auc >= 0.60 or score_psi >= score_psi_max: - return "high" - if c2st_auc >= 0.55 or score_psi >= max(score_psi_max * 0.7, 0.10): - return "moderate" - if c2st_auc >= 0.52 or distribution_warning_ratio > 0.05: - return "low" - return "none" - - -def _predictive_degradation_level( - *, - auc_delta: float, - brier_increase: float, - calibration_gap_delta: float, - auc_delta_max: float, - brier_increase_max: float, - calibration_gap_delta_max: float, -) -> str: - metrics = ( - (auc_delta, auc_delta_max), - (brier_increase, brier_increase_max), - (calibration_gap_delta, calibration_gap_delta_max), - ) - if _any_metric_crosses(metrics, multiplier=1.5, strict=False): - return "severe" - if _any_metric_crosses(metrics, multiplier=1.0, strict=True): - return "high" - if _any_metric_crosses(metrics, multiplier=0.75, strict=False): - return "moderate" - if any(value > 0.0 for value, _ in metrics): - return "low" - return "none" - - -def _any_metric_crosses( - metrics: tuple[tuple[float, float], ...], - *, - multiplier: float, - strict: bool, -) -> bool: - for value, limit in metrics: - threshold = limit * multiplier - crosses = value > threshold if strict else value >= threshold - if crosses: - return True - return False - - -def _shift_type(structural_level: str, predictive_level: str) -> str: - if structural_level != "none" and predictive_level in {"none", "low"}: - return "structural_shift_only" - if structural_level == "none" and predictive_level != "none": - return "predictive_degradation" - if structural_level != "none" and predictive_level != "none": - return "mixed_shift" - return "stable" - - -def _governance_posture(shift_type: str, predictive_level: str) -> str: - if shift_type == "mixed_shift" or predictive_level in {"high", "severe"}: - return "candidate_gate" - if shift_type == "structural_shift_only": - return "warning_only" - return "monitor" - - -def _pvalue_note(shift_type: str) -> str: - if shift_type == "structural_shift_only": - return ( - "KS/CvM or C2ST detect structural population change without clear predictive degradation. " - "Treat this as representativeness pressure and monitoring, not an automatic retraining gate." - ) - if shift_type == "predictive_degradation": - return ( - "Predictive degradation is visible even without strong structural shift. " - "Operational metrics dominate p-value interpretation in this case." - ) - if shift_type == "mixed_shift": - return ( - "Both structural shift and predictive degradation are present. " - "This combination deserves the strongest governance posture." - ) - return ( - "Distribution p-values are informative, but they can trigger on small effects at large N; " - "the governance posture should be anchored on materiality, predictive loss, and C2ST/score shift." - ) diff --git a/src/evaluation/pd_validation_interpretation.py b/src/evaluation/pd_validation_interpretation.py deleted file mode 100644 index b492dc7..0000000 --- a/src/evaluation/pd_validation_interpretation.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Interpretation layer for PD calibration and backtesting diagnostics.""" - -from __future__ import annotations - -import numpy as np -import pandas as pd - - -def classify_overall_gap_materiality( - observed_rate: float, predicted_rate: float -) -> dict[str, float | str]: - gap = float(observed_rate - predicted_rate) - gap_bp = gap * 10_000.0 - abs_gap_bp = abs(gap_bp) - if abs_gap_bp < 50.0: - band = "low" - elif abs_gap_bp < 100.0: - band = "moderate" - else: - band = "high" - return { - "gap": gap, - "gap_bp": gap_bp, - "abs_gap_bp": abs_gap_bp, - "materiality_band": band, - } - - -def summarize_slice_materiality( - grade_df: pd.DataFrame, - band_df: pd.DataFrame, - *, - grade_gap_threshold: float = 0.025, - band_gap_threshold: float = 0.015, -) -> dict[str, float | int]: - grade = grade_df.copy() - band = band_df.copy() - if "observed_default_rate" in grade.columns and "mean_predicted_pd" in grade.columns: - grade["abs_gap"] = ( - pd.to_numeric(grade["observed_default_rate"], errors="coerce") - - pd.to_numeric(grade["mean_predicted_pd"], errors="coerce") - ).abs() - else: - grade["abs_gap"] = np.nan - if "rate_gap" not in band.columns and {"observed_default_rate", "mean_predicted_pd"} <= set( - band.columns - ): - band["rate_gap"] = pd.to_numeric( - band["observed_default_rate"], errors="coerce" - ) - pd.to_numeric(band["mean_predicted_pd"], errors="coerce") - band["abs_gap"] = pd.to_numeric(band["rate_gap"], errors="coerce").abs() - return { - "grade_material_breaches": int((grade["abs_gap"] > grade_gap_threshold).sum()) - if not grade.empty - else 0, - "max_grade_gap_bp": float(grade["abs_gap"].max() * 10_000.0) if not grade.empty else 0.0, - "band_material_breaches": int((band["abs_gap"] > band_gap_threshold).sum()) - if not band.empty - else 0, - "max_band_gap_bp": float(band["abs_gap"].max() * 10_000.0) if not band.empty else 0.0, - } - - -def quarter_materiality_report( - predictions: pd.DataFrame, - meta: pd.DataFrame, - *, - min_rows_per_quarter: int = 2000, - score_col: str = "y_prob_final", -) -> pd.DataFrame: - n = min(len(predictions), len(meta)) - preds = predictions.iloc[:n].reset_index(drop=True).copy() - meta_df = meta.iloc[:n].reset_index(drop=True).copy() - issue_q = pd.to_datetime(meta_df["issue_d"], errors="coerce").dt.to_period("Q").astype(str) - frame = pd.DataFrame( - { - "issue_quarter": issue_q, - "y_true": pd.to_numeric(preds["y_true"], errors="coerce"), - "y_prob_final": pd.to_numeric(preds[score_col], errors="coerce"), - } - ).dropna() - report = ( - frame.groupby("issue_quarter", observed=True) - .agg( - n_obs=("y_true", "size"), - observed_default_rate=("y_true", "mean"), - mean_predicted_pd=("y_prob_final", "mean"), - ) - .reset_index() - .sort_values("issue_quarter") - ) - report = report.loc[report["n_obs"] >= int(min_rows_per_quarter)].reset_index(drop=True) - report["rate_gap"] = report["observed_default_rate"] - report["mean_predicted_pd"] - report["abs_gap_bp"] = report["rate_gap"].abs() * 10_000.0 - return report - - -def rare_event_summary(status: dict, report_df: pd.DataFrame) -> dict[str, float | None]: - summary = dict(status.get("summary", {}) or {}) - deciles = ( - report_df.loc[report_df["report_type"] == "score_decile"].copy() - if not report_df.empty - else pd.DataFrame() - ) - max_decile_gap_bp = None - if not deciles.empty and {"prevalence", "mean_score"} <= set(deciles.columns): - max_decile_gap_bp = float( - (deciles["prevalence"] - deciles["mean_score"]).abs().max() * 10_000.0 - ) - return { - "global_brier": summary.get("brier"), - "global_ece": summary.get("ece"), - "worst_protected_group_ece": status.get("worst_protected_group_ece"), - "worst_grade_brier": status.get("worst_grade_brier"), - "max_decile_gap_bp": max_decile_gap_bp, - } - - -def validation_interpretation_status( - *, - overall_backtesting: dict, - slice_materiality: dict, - quarter_report: pd.DataFrame, - rare_event: dict, -) -> dict[str, object]: - gap_meta = classify_overall_gap_materiality( - observed_rate=float(overall_backtesting.get("observed_default_rate", 0.0)), - predicted_rate=float(overall_backtesting.get("mean_predicted_pd", 0.0)), - ) - persistent_quarter_gaps = ( - int((quarter_report["abs_gap_bp"] >= 75.0).sum()) if not quarter_report.empty else 0 - ) - - severity = "pass" - if ( - gap_meta["materiality_band"] in {"moderate", "high"} - or int(slice_materiality.get("grade_material_breaches", 0)) >= 2 - or int(slice_materiality.get("band_material_breaches", 0)) >= 2 - or persistent_quarter_gaps >= 2 - ): - severity = "warning" - if ( - gap_meta["materiality_band"] == "high" - and int(slice_materiality.get("grade_material_breaches", 0)) >= 3 - and persistent_quarter_gaps >= 3 - ): - severity = "fail" - - signal_type = ( - "large_sample_significance" - if severity == "pass" - and float(gap_meta["abs_gap_bp"]) < 50.0 - and not bool(overall_backtesting.get("predicted_pd_inside_jeffreys", False)) - else "material_slice_deviation" - ) - return { - "diagnostic_only": True, - "overall_pass": bool(severity != "fail"), - "severity": severity, - "signal_type": signal_type, - "summary": { - **gap_meta, - **slice_materiality, - **rare_event, - "persistent_quarter_gaps": persistent_quarter_gaps, - "n_quarters_evaluated": len(quarter_report), - "exact_binomial_p_value": overall_backtesting.get("exact_binomial_p_value"), - "hl_p_value": overall_backtesting.get("hl_p_value"), - "predicted_pd_inside_jeffreys": overall_backtesting.get("predicted_pd_inside_jeffreys"), - }, - } diff --git a/src/evaluation/policy_contrast_bounds.py b/src/evaluation/policy_contrast_bounds.py new file mode 100644 index 0000000..af222b0 --- /dev/null +++ b/src/evaluation/policy_contrast_bounds.py @@ -0,0 +1,487 @@ +"""Sharp pairwise policy contrasts with nullable binary outcomes.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +_FACT_COLUMNS = ( + "contractual_rate", + "conformal_lower", + "conformal_upper", + "snapshot_default", +) +_ALLOCATION_COLUMNS = ( + "id", + "role", + "policy_label", + "exposure", + "expected_payoff_contribution", +) + + +def _sharp_binary_sum_bounds( + value_if_zero: np.ndarray, + value_if_one: np.ndarray, + outcomes: np.ndarray, +) -> tuple[float, float]: + observed = np.isfinite(outcomes) + if bool(np.any(observed & ~np.isin(outcomes, [0.0, 1.0]))): + raise ValueError("Observed outcomes must be binary.") + exact = np.where(outcomes == 1.0, value_if_one, value_if_zero) + lower = np.where(observed, exact, np.minimum(value_if_zero, value_if_one)) + upper = np.where(observed, exact, np.maximum(value_if_zero, value_if_one)) + return float(lower.sum()), float(upper.sum()) + + +def _binary_identification_width( + value_if_zero: np.ndarray, + value_if_one: np.ndarray, + outcomes: np.ndarray, +) -> float: + """Return the exact width contributed by unrestricted binary outcomes.""" + unresolved = ~np.isfinite(outcomes) + return float(np.abs(value_if_one[unresolved] - value_if_zero[unresolved]).sum()) + + +def _validated_normalizer(total: float, requested: float | None) -> float: + normalizer = total if requested is None else float(requested) + if not np.isfinite(normalizer): + raise ValueError("Policy normalization capital must be finite.") + if normalizer <= 0.0: + raise ValueError("Policy normalization capital must be positive.") + below_total = normalizer < total and not np.isclose( + normalizer, + total, + rtol=1.0e-10, + atol=1.0e-8, + ) + if below_total: + raise ValueError("Policy normalization capital cannot be below invested capital.") + return normalizer + + +def _validate_policy_bound_inputs( + *, + exposure_a: np.ndarray, + exposure_b: np.ndarray, + outcomes: np.ndarray, + rates: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + expected_difference: float, + lgd: float, + normalization_capital_a: float | None, + normalization_capital_b: float | None, +) -> tuple[float, float, float, float]: + lengths = { + len(exposure_a), + len(exposure_b), + len(outcomes), + len(rates), + len(lower), + len(upper), + } + if len(lengths) != 1: + raise ValueError("Policy contrast arrays are not aligned.") + numeric = { + "policy-a exposure": exposure_a, + "policy-b exposure": exposure_b, + "contractual rate": rates, + "conformal lower endpoint": lower, + "conformal upper endpoint": upper, + } + for label, values in numeric.items(): + if not bool(np.isfinite(values).all()): + raise ValueError(f"{label} values must be finite.") + if bool((exposure_a < 0.0).any()) or bool((exposure_b < 0.0).any()): + raise ValueError("Policy exposures must be non-negative.") + if bool((rates < 0.0).any()): + raise ValueError("Contractual rates must be non-negative.") + if bool((lower < 0.0).any()) or bool((upper > 1.0).any()) or bool((lower > upper).any()): + raise ValueError("Conformal endpoints must satisfy 0 <= lower <= upper <= 1.") + if not np.isfinite(expected_difference): + raise ValueError("Expected objective difference must be finite.") + if not np.isfinite(lgd) or lgd < 0.0: + raise ValueError("LGD must be finite and non-negative.") + total_a = float(exposure_a.sum()) + total_b = float(exposure_b.sum()) + if total_a <= 0.0 or total_b <= 0.0: + raise ValueError("Both policies must allocate positive capital.") + normalizer_a = _validated_normalizer(total_a, normalization_capital_a) + normalizer_b = _validated_normalizer(total_b, normalization_capital_b) + return total_a, total_b, normalizer_a, normalizer_b + + +def _metric_contributions( + *, + exposure_a: np.ndarray, + exposure_b: np.ndarray, + rates: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + lgd: float, + normalizer_a: float, + normalizer_b: float, +) -> dict[str, tuple[np.ndarray, np.ndarray]]: + delta_exposure = exposure_a - exposure_b + delta_weight = exposure_a / normalizer_a - exposure_b / normalizer_b + return { + "realized_payoff": (delta_exposure * rates, delta_exposure * -float(lgd)), + "realized_payoff_rate": (delta_weight * rates, delta_weight * -float(lgd)), + "weighted_default": (np.zeros(len(delta_weight), dtype=float), delta_weight), + "weighted_miscoverage": ( + delta_weight * (lower > 0.0).astype(float), + delta_weight * (upper < 1.0).astype(float), + ), + } + + +def _sharp_metric_results( + contributions: dict[str, tuple[np.ndarray, np.ndarray]], + outcomes: np.ndarray, +) -> tuple[dict[str, float], dict[str, float]]: + bounds: dict[str, float] = {} + widths: dict[str, float] = {} + for metric, (value_if_zero, value_if_one) in contributions.items(): + lower_bound, upper_bound = _sharp_binary_sum_bounds( + value_if_zero, + value_if_one, + outcomes, + ) + bounds[f"{metric}_difference_lower"] = lower_bound + bounds[f"{metric}_difference_upper"] = upper_bound + width_name = f"{metric}_identification_width" + width = _binary_identification_width(value_if_zero, value_if_one, outcomes) + if not np.isclose(width, upper_bound - lower_bound, atol=1.0e-10, rtol=1.0e-12): + raise RuntimeError(f"Sharp-bound width failed direct reconciliation for {width_name}.") + widths[width_name] = width + return bounds, widths + + +def _sharp_policy_bounds_from_arrays( + *, + exposure_a: np.ndarray, + exposure_b: np.ndarray, + outcomes: np.ndarray, + rates: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + expected_difference: float, + policy_a: str, + policy_b: str, + role: str, + lgd: float, + normalization_capital_a: float | None = None, + normalization_capital_b: float | None = None, +) -> dict[str, Any]: + total_a, total_b, normalizer_a, normalizer_b = _validate_policy_bound_inputs( + exposure_a=exposure_a, + exposure_b=exposure_b, + outcomes=outcomes, + rates=rates, + lower=lower, + upper=upper, + expected_difference=expected_difference, + lgd=lgd, + normalization_capital_a=normalization_capital_a, + normalization_capital_b=normalization_capital_b, + ) + contributions = _metric_contributions( + exposure_a=exposure_a, + exposure_b=exposure_b, + rates=rates, + lower=lower, + upper=upper, + lgd=lgd, + normalizer_a=normalizer_a, + normalizer_b=normalizer_b, + ) + bounds, widths = _sharp_metric_results(contributions, outcomes) + return { + "contrast": f"{policy_a}_minus_{policy_b}", + "role": role, + "policy_a": policy_a, + "policy_b": policy_b, + "policy_a_capital": total_a, + "policy_b_capital": total_b, + "policy_a_normalization_capital": normalizer_a, + "policy_b_normalization_capital": normalizer_b, + "funded_union_loans": int(len(outcomes)), + "unresolved_union_loans": int((~np.isfinite(outcomes)).sum()), + "expected_objective_difference": float(expected_difference), + **bounds, + **widths, + "payoff_direction_sign_robust": bool( + bounds["realized_payoff_difference_lower"] > 0.0 + or bounds["realized_payoff_difference_upper"] < 0.0 + ), + "default_direction_sign_robust": bool( + bounds["weighted_default_difference_lower"] > 0.0 + or bounds["weighted_default_difference_upper"] < 0.0 + ), + "miscoverage_direction_sign_robust": bool( + bounds["weighted_miscoverage_difference_lower"] > 0.0 + or bounds["weighted_miscoverage_difference_upper"] < 0.0 + ), + "causal_interpretation": False, + } + + +def _require_columns(frame: pd.DataFrame, columns: tuple[str, ...], *, label: str) -> None: + missing = sorted(set(columns) - set(frame.columns)) + if missing: + raise KeyError(f"{label} is missing required columns: {missing}.") + + +def _require_consistent_loan_facts(frame: pd.DataFrame) -> None: + fact_counts = frame.groupby("id", observed=True, dropna=False)[list(_FACT_COLUMNS)].nunique( + dropna=False + ) + conflicting = fact_counts.gt(1) + if bool(conflicting.to_numpy().any()): + conflict_ids = conflicting.index[conflicting.any(axis=1)].astype(str).tolist() + conflict_columns = conflicting.columns[conflicting.any(axis=0)].tolist() + raise ValueError( + "Conflicting policy facts for the same loan: " + f"ids={conflict_ids[:5]}, columns={conflict_columns}." + ) + + +def _validated_loan_facts( + source: pd.DataFrame, + *, + required_ids: pd.Index, + require_unique_ids: bool, +) -> pd.DataFrame: + _require_columns(source, ("id", *_FACT_COLUMNS), label="Policy loan facts") + facts = source.loc[source["id"].isin(required_ids), ["id", *_FACT_COLUMNS]].copy() + if bool(facts["id"].isna().any()): + raise ValueError("Policy loan facts contain missing IDs.") + _require_consistent_loan_facts(facts) + if require_unique_ids and bool(facts["id"].duplicated().any()): + raise ValueError("External policy loan facts contain duplicate IDs.") + facts = facts.drop_duplicates().set_index("id") + if bool(facts.index.duplicated().any()): + raise ValueError("Policy loan facts do not reduce to one row per ID.") + missing_ids = required_ids.difference(facts.index) + extra_ids = facts.index.difference(required_ids) + if len(missing_ids) or len(extra_ids): + raise ValueError( + f"Policy loan-fact ID mismatch: missing={len(missing_ids)}, extra={len(extra_ids)}." + ) + return facts.reindex(required_ids) + + +def _union_policy_allocations( + allocations: pd.DataFrame, + *, + policy_a: str, + policy_b: str, + role: str, +) -> pd.DataFrame: + subset = allocations.loc[ + allocations["role"].eq(role) & allocations["policy_label"].isin([policy_a, policy_b]) + ].copy() + if subset.empty: + raise ValueError(f"No allocations found for role {role!r}.") + counts = subset.groupby(["id", "policy_label"], observed=True).size() + if bool((counts > 1).any()): + raise ValueError("A loan appears more than once within a policy contrast.") + present = set(subset["policy_label"].astype(str)) + if present != {policy_a, policy_b}: + raise ValueError( + f"Policy contrast is missing labels: {sorted({policy_a, policy_b} - present)}" + ) + + attributes = ["id", *_FACT_COLUMNS] + _require_consistent_loan_facts(subset) + loans = subset[attributes].drop_duplicates() + if bool(loans["id"].duplicated().any()): + raise ValueError("Policy loan facts do not reduce to one row per ID.") + loans = loans.set_index("id") + exposure = subset.pivot(index="id", columns="policy_label", values="exposure").fillna(0.0) + union = loans.join(exposure, how="outer").reset_index() + union[policy_a] = pd.to_numeric(union[policy_a], errors="coerce").fillna(0.0) + union[policy_b] = pd.to_numeric(union[policy_b], errors="coerce").fillna(0.0) + return union + + +class PolicyContrastIndex: + """Validated allocation index for repeated sharp pairwise contrasts. + + The allocation support is indexed once. Each contrast then gathers two sparse + exposure vectors over the exact funded union and restores the deterministic ID + order used by the public single-pair oracle. + """ + + def __init__( + self, + allocations: pd.DataFrame, + *, + role: str, + loan_facts: pd.DataFrame | None = None, + ) -> None: + _require_columns(allocations, _ALLOCATION_COLUMNS, label="Policy allocations") + subset = allocations.loc[allocations["role"].eq(role), list(_ALLOCATION_COLUMNS)].copy() + if subset.empty: + raise ValueError(f"No allocations found for role {role!r}.") + if bool(subset[["id", "policy_label"]].isna().any(axis=None)): + raise ValueError("Policy allocations contain missing IDs or policy labels.") + subset["policy_label"] = subset["policy_label"].astype(str) + duplicate = subset.duplicated(["id", "policy_label"], keep=False) + if bool(duplicate.any()): + raise ValueError("A loan appears more than once within a policy contrast index.") + + ids = pd.Index(subset["id"].drop_duplicates(), name="id") + fact_source = allocations if loan_facts is None else loan_facts + facts = _validated_loan_facts( + fact_source, + required_ids=ids, + require_unique_ids=loan_facts is not None, + ) + self._role = str(role) + self._ids = ids + self._rates = pd.to_numeric(facts["contractual_rate"], errors="raise").to_numpy(dtype=float) + self._lower = pd.to_numeric(facts["conformal_lower"], errors="raise").to_numpy(dtype=float) + self._upper = pd.to_numeric(facts["conformal_upper"], errors="raise").to_numpy(dtype=float) + self._outcomes = pd.to_numeric(facts["snapshot_default"], errors="coerce").to_numpy( + dtype=float + ) + + id_positions = pd.Series(np.arange(len(ids), dtype=np.int64), index=ids) + self._sort_rank = np.empty(len(ids), dtype=np.int64) + sorted_positions = id_positions.loc[ids.sort_values()].to_numpy(dtype=np.int64) + self._sort_rank[sorted_positions] = np.arange(len(ids), dtype=np.int64) + self._positions: dict[str, np.ndarray] = {} + self._exposures: dict[str, np.ndarray] = {} + self._expected: dict[str, float] = {} + for label, frame in subset.groupby("policy_label", observed=True, sort=False): + policy_label = str(label) + positions = id_positions.loc[frame["id"]].to_numpy(dtype=np.int64) + exposure = pd.to_numeric(frame["exposure"], errors="raise").to_numpy(dtype=float) + if not bool(np.isfinite(exposure).all()): + raise ValueError(f"Policy {policy_label!r} contains non-finite exposure.") + if bool((exposure < 0.0).any()): + raise ValueError(f"Policy {policy_label!r} contains negative exposure.") + expected_contributions = pd.to_numeric( + frame["expected_payoff_contribution"], errors="raise" + ).to_numpy(dtype=float) + if not bool(np.isfinite(expected_contributions).all()): + raise ValueError( + f"Policy {policy_label!r} contains non-finite expected contributions." + ) + self._positions[policy_label] = positions + self._exposures[policy_label] = exposure + self._expected[policy_label] = float(expected_contributions.sum()) + self._anchor_cache: dict[str, tuple[np.ndarray, np.ndarray]] = {} + + def _anchor_lookup(self, policy_label: str) -> tuple[np.ndarray, np.ndarray]: + cached = self._anchor_cache.get(policy_label) + if cached is not None: + return cached + positions = self._positions[policy_label] + present = np.zeros(len(self._ids), dtype=bool) + destination = np.full(len(self._ids), -1, dtype=np.int64) + present[positions] = True + destination[positions] = np.arange(len(positions), dtype=np.int64) + cached = (present, destination) + self._anchor_cache[policy_label] = cached + return cached + + def sharp_bounds( + self, + *, + policy_a: str, + policy_b: str, + lgd: float, + normalization_capital_a: float | None = None, + normalization_capital_b: float | None = None, + ) -> dict[str, Any]: + """Return sharp ``policy_a - policy_b`` bounds from the reusable index.""" + missing = sorted({policy_a, policy_b} - set(self._positions)) + if missing: + raise ValueError(f"Policy contrast is missing labels: {missing}") + + positions_a = self._positions[policy_a] + positions_b = self._positions[policy_b] + exposure_values_a = self._exposures[policy_a] + exposure_values_b = self._exposures[policy_b] + present_a, destination_a = self._anchor_lookup(policy_a) + overlap_b = present_a[positions_b] + new_b = ~overlap_b + union_positions = np.concatenate([positions_a, positions_b[new_b]]) + exposure_a = np.zeros(len(union_positions), dtype=float) + exposure_b = np.zeros(len(union_positions), dtype=float) + exposure_a[: len(positions_a)] = exposure_values_a + exposure_b[destination_a[positions_b[overlap_b]]] = exposure_values_b[overlap_b] + exposure_b[len(positions_a) :] = exposure_values_b[new_b] + union_order = np.argsort(self._sort_rank[union_positions], kind="stable") + union_positions = union_positions[union_order] + exposure_a = exposure_a[union_order] + exposure_b = exposure_b[union_order] + + outcomes = self._outcomes[union_positions] + return _sharp_policy_bounds_from_arrays( + exposure_a=exposure_a, + exposure_b=exposure_b, + outcomes=outcomes, + rates=self._rates[union_positions], + lower=self._lower[union_positions], + upper=self._upper[union_positions], + expected_difference=self._expected[policy_a] - self._expected[policy_b], + policy_a=policy_a, + policy_b=policy_b, + role=self._role, + lgd=lgd, + normalization_capital_a=normalization_capital_a, + normalization_capital_b=normalization_capital_b, + ) + + +def sharp_policy_contrast_bounds( + allocations: pd.DataFrame, + *, + policy_a: str, + policy_b: str, + role: str, + lgd: float, +) -> dict[str, Any]: + """Return sharp ``policy_a - policy_b`` bounds on their funded union.""" + union = _union_policy_allocations( + allocations, + policy_a=policy_a, + policy_b=policy_b, + role=role, + ) + exposure_a = union[policy_a].to_numpy(dtype=float) + exposure_b = union[policy_b].to_numpy(dtype=float) + outcomes = pd.to_numeric(union["snapshot_default"], errors="coerce").to_numpy(dtype=float) + rates = union["contractual_rate"].to_numpy(dtype=float) + lower = union["conformal_lower"].to_numpy(dtype=float) + upper = union["conformal_upper"].to_numpy(dtype=float) + expected = ( + allocations.loc[ + allocations["role"].eq(role) & allocations["policy_label"].eq(policy_a), + "expected_payoff_contribution", + ].sum() + - allocations.loc[ + allocations["role"].eq(role) & allocations["policy_label"].eq(policy_b), + "expected_payoff_contribution", + ].sum() + ) + return _sharp_policy_bounds_from_arrays( + exposure_a=exposure_a, + exposure_b=exposure_b, + outcomes=outcomes, + rates=rates, + lower=lower, + upper=upper, + expected_difference=float(expected), + policy_a=policy_a, + policy_b=policy_b, + role=role, + lgd=lgd, + ) diff --git a/src/evaluation/standardized_credit_payoff.py b/src/evaluation/standardized_credit_payoff.py new file mode 100644 index 0000000..2be453d --- /dev/null +++ b/src/evaluation/standardized_credit_payoff.py @@ -0,0 +1,101 @@ +"""Coherent expected and realized standardized credit payoff.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +PAYOFF_ID = "coherent_standardized_binary_payoff_v1" + + +def contractual_rate_decimal(values: pd.Series) -> np.ndarray: + """Convert Lending Club percent-point rates to decimal annual rates once.""" + rates = ( + values.astype("string") + .str.strip() + .str.rstrip("%") + .pipe(pd.to_numeric, errors="coerce") + .to_numpy(dtype=float) + / 100.0 + ) + if not bool(np.isfinite(rates).all()) or bool(np.any((rates < 0.0) | (rates > 1.0))): + raise ValueError("Contractual rates must be finite percent-point values in [0, 100].") + return rates + + +def expected_standardized_payoff_rate( + probabilities: np.ndarray, + contractual_rates: np.ndarray, + *, + lgd: float, +) -> np.ndarray: + """Return ``(1-p)r - p*LGD`` per dollar of exposure.""" + return expected_objective_coefficients(probabilities, contractual_rates, lgd=lgd) + + +def expected_objective_coefficients( + probabilities: np.ndarray, + contractual_rates: np.ndarray, + *, + lgd: float, +) -> np.ndarray: + """Return coherent expected standardized-payoff objective coefficients.""" + point = np.asarray(probabilities, dtype=float) + rates = np.asarray(contractual_rates, dtype=float) + loss_given_default = _validated_payoff_inputs(point, rates, lgd=lgd) + return (1.0 - point) * rates - point * loss_given_default + + +def realized_standardized_payoff_bounds( + outcomes: np.ndarray, + contractual_rates: np.ndarray, + *, + lgd: float, +) -> tuple[np.ndarray, np.ndarray]: + """Return sharp payoff bounds while retaining unresolved outcomes.""" + y_true = np.asarray(outcomes, dtype=float) + rates = np.asarray(contractual_rates, dtype=float) + if y_true.shape != rates.shape: + raise ValueError("outcomes and contractual_rates must align.") + loss_given_default = _validated_lgd(lgd) + _validate_rates(rates) + if bool(np.isinf(y_true).any()): + raise ValueError("Outcomes must be binary or NaN when unresolved.") + observed = np.isfinite(y_true) + if bool(np.any(observed & ~np.isin(y_true, [0.0, 1.0]))): + raise ValueError("Observed outcomes must be binary.") + filled = np.nan_to_num(y_true, nan=0.0) + realized = (1.0 - filled) * rates - filled * loss_given_default + lower = np.where(observed, realized, -loss_given_default) + upper = np.where(observed, realized, rates) + return lower.astype(float), upper.astype(float) + + +def _validated_payoff_inputs( + probabilities: np.ndarray, + contractual_rates: np.ndarray, + *, + lgd: float, +) -> float: + if probabilities.shape != contractual_rates.shape: + raise ValueError("probabilities and contractual_rates must align.") + if not bool(np.isfinite(probabilities).all()) or bool( + np.any((probabilities < 0.0) | (probabilities > 1.0)) + ): + raise ValueError("Probabilities must be finite values in [0, 1].") + _validate_rates(contractual_rates) + return _validated_lgd(lgd) + + +def _validate_rates(contractual_rates: np.ndarray) -> None: + if not bool(np.isfinite(contractual_rates).all()) or bool( + np.any((contractual_rates < 0.0) | (contractual_rates > 1.0)) + ): + raise ValueError("Contractual rates must be finite decimal values in [0, 1].") + + +def _validated_lgd(lgd: float) -> float: + value = float(lgd) + if not np.isfinite(value) or not 0.0 <= value <= 1.0: + raise ValueError("LGD must be a finite value in [0, 1].") + return value diff --git a/src/features/__init__.py b/src/features/__init__.py index eea2693..8b55fc0 100644 --- a/src/features/__init__.py +++ b/src/features/__init__.py @@ -1 +1 @@ -"""Feature engineering and validation schemas.""" +"""Feature engineering used by the active credit-risk specifications.""" diff --git a/src/features/feature_config_io.py b/src/features/feature_config_io.py deleted file mode 100644 index 0dd38f2..0000000 --- a/src/features/feature_config_io.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Read/write helpers for the feature configuration artifact. - -The feature configuration is a plain Python ``dict`` of feature-name lists and -small scalars. The live pipeline stores it as human-readable YAML plus an -inspection-friendly Parquet table. Loading the legacy pickle remains available -only as an explicit audit escape hatch for old artifacts. - -The Parquet representation is deliberately long-form instead of trying to -encode nested Python objects directly: each row stores one list element, dict -entry, or scalar value as JSON with a small Pandera-validated schema. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -import joblib -import pandas as pd -import yaml - -from src.features.schemas import validate_feature_config_table - -DEFAULT_PICKLE_PATH = Path("data/processed/feature_config.pkl") -DEFAULT_YAML_PATH = Path("data/processed/feature_config.yml") -DEFAULT_PARQUET_PATH = Path("data/processed/feature_config.parquet") - - -def load_feature_config( - *, - repo_root: Path | str | None = None, - pickle_path: Path | str | None = None, - yaml_path: Path | str | None = None, - parquet_path: Path | str | None = None, - prefer: str = "yaml", -) -> dict[str, Any]: - """Load the feature configuration dictionary. - - Args: - repo_root: If provided, resolve relative paths against this directory. - pickle_path: Override for the legacy pickle location. - yaml_path: Override for the YAML companion location. - parquet_path: Override for the Parquet table location. - prefer: ``"yaml"`` (strict YAML), ``"parquet"`` (strict Parquet), - ``"pickle"`` (strict legacy pickle), or ``"auto"`` (best-effort - legacy compatibility: YAML, then Parquet, then pickle). - - Returns: - The same ``dict`` structure used by the rest of the pipeline. - """ - root = Path(repo_root) if repo_root is not None else Path.cwd() - pkl = _resolve(root, pickle_path, DEFAULT_PICKLE_PATH) - yml = _resolve(root, yaml_path, DEFAULT_YAML_PATH) - parquet = _resolve(root, parquet_path, DEFAULT_PARQUET_PATH) - - if prefer == "yaml": - return _load_yaml(yml) - if prefer == "parquet": - return _load_parquet(parquet) - if prefer == "pickle": - return _load_pickle(pkl) - if prefer != "auto": - raise ValueError(f"prefer must be 'auto', 'yaml', 'parquet' or 'pickle', got {prefer!r}") - if yml.is_file(): - return _load_yaml(yml) - if parquet.is_file(): - return _load_parquet(parquet) - return _load_pickle(pkl) - - -def save_feature_config( - cfg: Mapping[str, Any], - *, - repo_root: Path | str | None = None, - yaml_path: Path | str | None = None, - parquet_path: Path | str | None = None, - pickle_path: Path | str | None = None, - also_parquet: bool = False, - also_pickle: bool = False, -) -> Path: - """Persist ``cfg`` to YAML and optional companion formats. - - Returns the YAML path that was written. - """ - root = Path(repo_root) if repo_root is not None else Path.cwd() - yml = _resolve(root, yaml_path, DEFAULT_YAML_PATH) - yml.parent.mkdir(parents=True, exist_ok=True) - yml.write_text( - yaml.safe_dump(dict(cfg), sort_keys=True, allow_unicode=True), - encoding="utf-8", - ) - if also_parquet: - parquet = _resolve(root, parquet_path, DEFAULT_PARQUET_PATH) - parquet.parent.mkdir(parents=True, exist_ok=True) - _config_to_frame(cfg).to_parquet(parquet, index=False) - if also_pickle: - pkl = _resolve(root, pickle_path, DEFAULT_PICKLE_PATH) - pkl.parent.mkdir(parents=True, exist_ok=True) - joblib.dump(dict(cfg), pkl) - return yml - - -def pickle_to_yaml( - pickle_path: Path | str = DEFAULT_PICKLE_PATH, - yaml_path: Path | str = DEFAULT_YAML_PATH, -) -> Path: - """One-shot migrator: read an existing pickle and write its YAML companion.""" - cfg = _load_pickle(Path(pickle_path)) - out = Path(yaml_path) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text( - yaml.safe_dump(cfg, sort_keys=True, allow_unicode=True), - encoding="utf-8", - ) - return out - - -# --------------------------------------------------------------------------- -# helpers -# --------------------------------------------------------------------------- - - -def _resolve(root: Path, override: Path | str | None, default: Path) -> Path: - candidate = default if override is None else Path(override) - if candidate.is_absolute(): - return candidate - return root / candidate - - -def _load_pickle(path: Path) -> dict[str, Any]: - if not path.is_file(): - raise FileNotFoundError(path) - cfg = joblib.load(path) - if not isinstance(cfg, dict): - raise TypeError(f"feature_config pickle at {path} is {type(cfg).__name__}, expected dict.") - return cfg - - -def _load_yaml(path: Path) -> dict[str, Any]: - if not path.is_file(): - raise FileNotFoundError(path) - cfg = yaml.safe_load(path.read_text(encoding="utf-8")) - if not isinstance(cfg, dict): - raise TypeError(f"feature_config YAML at {path} is {type(cfg).__name__}, expected dict.") - return cfg - - -def _load_parquet(path: Path) -> dict[str, Any]: - if not path.is_file(): - raise FileNotFoundError(path) - frame = pd.read_parquet(path) - return _frame_to_config(frame) - - -def _config_to_frame(cfg: Mapping[str, Any]) -> pd.DataFrame: - rows: list[dict[str, Any]] = [] - for section in sorted(cfg): - value = cfg[section] - if isinstance(value, list): - rows.extend( - { - "section": section, - "kind": "list", - "ordinal": ordinal, - "key": None, - "value_json": json.dumps(item, sort_keys=True), - } - for ordinal, item in enumerate(value) - ) - elif isinstance(value, dict): - rows.extend( - { - "section": section, - "kind": "dict", - "ordinal": ordinal, - "key": str(key), - "value_json": json.dumps(value[key], sort_keys=True), - } - for ordinal, key in enumerate(sorted(value)) - ) - else: - rows.append( - { - "section": section, - "kind": "scalar", - "ordinal": 0, - "key": None, - "value_json": json.dumps(value, sort_keys=True), - } - ) - return validate_feature_config_table(pd.DataFrame(rows)) - - -def _frame_to_config(frame: pd.DataFrame) -> dict[str, Any]: - validated = validate_feature_config_table(frame).sort_values( - ["section", "ordinal", "key"], - na_position="first", - ) - cfg: dict[str, Any] = {} - for section, group in validated.groupby("section", sort=False): - kinds = set(group["kind"]) - if len(kinds) != 1: - raise ValueError(f"feature_config section {section!r} has mixed kinds: {kinds}") - kind = group["kind"].iloc[0] - if kind == "list": - cfg[str(section)] = [json.loads(raw) for raw in group["value_json"]] - elif kind == "dict": - cfg[str(section)] = { - str(row["key"]): json.loads(str(row["value_json"])) - for row in group.to_dict("records") - } - elif kind == "scalar": - if len(group) != 1: - raise ValueError(f"feature_config scalar section {section!r} has {len(group)} rows") - cfg[str(section)] = json.loads(str(group["value_json"].iloc[0])) - else: - raise ValueError(f"Unsupported feature_config section kind: {kind}") - return cfg diff --git a/src/features/feature_engineering.py b/src/features/feature_engineering.py index 5918ac9..2144963 100644 --- a/src/features/feature_engineering.py +++ b/src/features/feature_engineering.py @@ -248,7 +248,11 @@ def normalize_raw_columns(df: pd.DataFrame) -> pd.DataFrame: if "issue_d" in out.columns: out["issue_d"] = pd.to_datetime(out["issue_d"], errors="coerce") if "earliest_cr_line" in out.columns: - out["earliest_cr_line"] = pd.to_datetime(out["earliest_cr_line"], errors="coerce") + out["earliest_cr_line"] = pd.to_datetime( + out["earliest_cr_line"], + format="%b-%Y", + errors="coerce", + ) if TARGET in out.columns: out[TARGET] = pd.to_numeric(out[TARGET], errors="coerce").fillna(0).astype(int) return out @@ -371,6 +375,7 @@ def create_credit_history_features(df: pd.DataFrame) -> pd.DataFrame: else None ) if recency_source is not None: + out["delinq_recency_observed"] = out[recency_source].notna().astype(int) out["delinq_recency"] = pd.to_numeric(out[recency_source], errors="coerce").fillna(999.0) delinquency_components = [ @@ -402,6 +407,7 @@ def create_flags(df: pd.DataFrame) -> pd.DataFrame: if "pub_rec" in out.columns: out["has_pub_rec"] = (out["pub_rec"] > 0).astype(int) if "pub_rec_bankruptcies" in out.columns: + out["bankruptcy_observed"] = out["pub_rec_bankruptcies"].notna().astype(int) out["has_bankruptcy"] = (out["pub_rec_bankruptcies"] > 0).astype(int) elif "pub_rec" in out.columns: out["has_bankruptcy"] = 0 diff --git a/src/features/schemas.py b/src/features/schemas.py deleted file mode 100644 index ba3f2df..0000000 --- a/src/features/schemas.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Pandera schemas for DataFrame validation at pipeline boundaries.""" - -from __future__ import annotations - -import pandas as pd -import pandera.pandas as pa - -# ── Loan Master Schema ── -loan_master_schema = pa.DataFrameSchema( - columns={ - "loan_amnt": pa.Column(float, pa.Check.greater_than(0), nullable=False), - "annual_inc": pa.Column(float, pa.Check.greater_than_or_equal_to(0), nullable=True), - "loan_to_income": pa.Column(float, pa.Check.in_range(0, 100), nullable=True), - "dti": pa.Column(float, pa.Check.in_range(0, 999), nullable=True), - "default_flag": pa.Column(int, pa.Check.isin([0, 1]), nullable=False), - "int_rate": pa.Column(float, pa.Check.in_range(0, 100), nullable=True), - }, - coerce=True, - strict=False, # Allow extra columns -) - -# ── Time Series Schema (Nixtla-compatible) ── -time_series_schema = pa.DataFrameSchema( - columns={ - "ds": pa.Column("datetime64[ns]", nullable=False), - "unique_id": pa.Column(str, nullable=False), - "y": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), # default_rate - "loan_count": pa.Column(int, pa.Check.greater_than(0), nullable=False), - }, - coerce=True, - strict=False, -) - -# ── EAD Dataset Schema ── -ead_schema = pa.DataFrameSchema( - columns={ - "default_flag": pa.Column(int, pa.Check.equal_to(1), nullable=False), # Only defaults - "loan_amnt": pa.Column(float, pa.Check.greater_than(0), nullable=False), - }, - coerce=True, - strict=False, -) - -# ── Prediction Output Schema ── -prediction_schema = pa.DataFrameSchema( - columns={ - "pd_point": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - "pd_low": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - "pd_high": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - }, - checks=[ - pa.Check( - lambda df: (df["pd_low"] <= df["pd_point"]).all(), error="pd_low must be <= pd_point" - ), - pa.Check( - lambda df: (df["pd_point"] <= df["pd_high"]).all(), error="pd_point must be <= pd_high" - ), - ], - coerce=True, - strict=False, -) - - -# ── Conformal Output Schema ── -conformal_output_schema = pa.DataFrameSchema( - columns={ - "y_pred": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - "pd_low_90": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - "pd_high_90": pa.Column(float, pa.Check.in_range(0, 1), nullable=False), - "grade": pa.Column(str, nullable=True), - "width_90": pa.Column(float, pa.Check.greater_than_or_equal_to(0), nullable=False), - }, - checks=[ - pa.Check( - lambda df: (df["pd_low_90"] <= df["pd_high_90"]).all(), - error="pd_low_90 must be <= pd_high_90", - ), - ], - coerce=True, - strict=False, -) - -feature_config_table_schema = pa.DataFrameSchema( - columns={ - "section": pa.Column(str, nullable=False), - "kind": pa.Column(str, pa.Check.isin(["list", "dict", "scalar"]), nullable=False), - "ordinal": pa.Column(int, pa.Check.greater_than_or_equal_to(0), nullable=False), - "key": pa.Column(str, nullable=True), - "value_json": pa.Column(str, nullable=False), - }, - coerce=True, - strict=True, -) - - -def validate_conformal_output(df: pd.DataFrame) -> pd.DataFrame: - """Validate conformal intervals DataFrame.""" - return conformal_output_schema.validate(df) - - -def validate_loan_master(df: pd.DataFrame) -> pd.DataFrame: - """Validate loan_master DataFrame.""" - return loan_master_schema.validate(df) - - -def validate_time_series(df: pd.DataFrame) -> pd.DataFrame: - """Validate time_series DataFrame.""" - return time_series_schema.validate(df) - - -def validate_predictions(df: pd.DataFrame) -> pd.DataFrame: - """Validate prediction output DataFrame.""" - return prediction_schema.validate(df) - - -def validate_feature_config_table(df: pd.DataFrame) -> pd.DataFrame: - """Validate the long-form feature configuration table.""" - return feature_config_table_schema.validate(df) - - -# --------------------------------------------------------------------------- -# DataFrameModel variants (Pandera 0.20+ class-based style) -# --------------------------------------------------------------------------- -# -# The dict-based ``DataFrameSchema`` above remains the canonical contract used -# by the pipeline. The class-based ``DataFrameModel`` form below is the -# recommended modern style for new schemas: it gives IDE autocomplete, is -# subclass-friendly, and renders cleanly in MRM model cards. - - -class PredictionOutputModel(pa.DataFrameModel): - """Class-based equivalent of :data:`prediction_schema`.""" - - pd_point: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - pd_low: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - pd_high: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - - @pa.dataframe_check - @classmethod - def low_le_point_le_high(cls, df: pd.DataFrame) -> bool: - return bool(((df["pd_low"] <= df["pd_point"]) & (df["pd_point"] <= df["pd_high"])).all()) - - class Config: - coerce = True - strict = False - - -class ConformalOutputModel(pa.DataFrameModel): - """Class-based equivalent of :data:`conformal_output_schema`.""" - - y_pred: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - pd_low_90: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - pd_high_90: pa.typing.Series[float] = pa.Field(ge=0.0, le=1.0) - grade: pa.typing.Series[str] = pa.Field(nullable=True) - width_90: pa.typing.Series[float] = pa.Field(ge=0.0) - - @pa.dataframe_check - @classmethod - def low_le_high(cls, df: pd.DataFrame) -> bool: - return bool((df["pd_low_90"] <= df["pd_high_90"]).all()) - - class Config: - coerce = True - strict = False diff --git a/src/features/tabprep_challenger.py b/src/features/tabprep_challenger.py deleted file mode 100644 index 6d65c55..0000000 --- a/src/features/tabprep_challenger.py +++ /dev/null @@ -1,1003 +0,0 @@ -"""TabPrep-inspired feature generators for isolated CRPTO challengers. - -The implementation is intentionally local to CRPTO instead of importing the -reference TabPrep/AutoGluon stack. That keeps the frozen thesis environment -stable while still exercising the useful feature families described in the -TabPrep paper: arithmetic combinations, relative group-by features, -out-of-fold target encodings, categorical interactions, and random subset -feature compression. -""" - -from __future__ import annotations - -import hashlib -import itertools -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass, field -from typing import Any, cast - -import numpy as np -import pandas as pd -from loguru import logger - -from src.features.feature_engineering import ( - CATEGORICAL_FEATURES, - FLAG_FEATURES, - INTERACTION_FEATURES, - MISSINGNESS_INDICATOR_SUFFIX, - TARGET, - build_feature_config, -) - -MISSING_TOKEN = "__MISSING__" -UNKNOWN_TOKEN = "__UNKNOWN__" -TABPREP_FEATURE_PREFIX = "tp_" - -FORBIDDEN_FEATURES = { - "id", - "url", - "loan_status", - "emp_title", - "title", - "zip_code", - "addr_state", - "next_pymnt_d", - "pymnt_plan", - "application_type", - "verification_status_joint", - "settlement_status", - "hardship_status", - "debt_settlement_flag", - "last_pymnt_d", - "last_pymnt_amnt", - "last_credit_pull_d", - "out_prncp", - "out_prncp_inv", - "total_pymnt", - "total_pymnt_inv", - "total_rec_prncp", - "total_rec_int", - "total_rec_late_fee", - "recoveries", - "collection_recovery_fee", -} -FORBIDDEN_PREFIXES = ("sec_app_",) -FORBIDDEN_CONTAINS = ("url", "title", "emp_title", "next_pymnt") - - -@dataclass(frozen=True) -class TabPrepVariantConfig: - """Feature budgets for one TabPrep challenger variant.""" - - name: str - max_generated_features: int - arithmetic_features: int - groupby_features: int - target_encoding_features: int - interaction_encoding_features: int - rsfc_features: int - max_numeric_base_features: int - max_groupby_numeric_features: int - max_categorical_base_features: int - max_scoring_rows: int = 100_000 - n_oof_folds: int = 5 - smoothing: float = 20.0 - min_group_support: int = 50 - rsfc_candidate_multiplier: int = 2 - - -TABPREP_VARIANTS: dict[str, TabPrepVariantConfig] = { - "safe_500": TabPrepVariantConfig( - name="safe_500", - max_generated_features=500, - arithmetic_features=300, - groupby_features=100, - target_encoding_features=60, - interaction_encoding_features=30, - rsfc_features=10, - max_numeric_base_features=60, - max_groupby_numeric_features=40, - max_categorical_base_features=10, - ), - "balanced_1500": TabPrepVariantConfig( - name="balanced_1500", - max_generated_features=1500, - arithmetic_features=800, - groupby_features=300, - target_encoding_features=100, - interaction_encoding_features=150, - rsfc_features=150, - max_numeric_base_features=100, - max_groupby_numeric_features=75, - max_categorical_base_features=10, - ), - "full_3000": TabPrepVariantConfig( - name="full_3000", - max_generated_features=3000, - arithmetic_features=1500, - groupby_features=700, - target_encoding_features=150, - interaction_encoding_features=300, - rsfc_features=350, - max_numeric_base_features=150, - max_groupby_numeric_features=100, - max_categorical_base_features=10, - ), -} - - -@dataclass(frozen=True) -class FeatureSpec: - """Selected generated feature and its provenance.""" - - name: str - generator: str - operation: str - source_features: tuple[str, ...] - score: float - selected_rank: int - metadata: dict[str, Any] = field(default_factory=dict) - - def to_row(self) -> dict[str, Any]: - """Return a JSON/parquet-friendly manifest row.""" - return { - "feature": self.name, - "generator": self.generator, - "operation": self.operation, - "source_features": "|".join(self.source_features), - "n_source_features": len(self.source_features), - "score": float(self.score), - "selected_rank": int(self.selected_rank), - **{f"meta_{key}": value for key, value in self.metadata.items()}, - } - - -@dataclass -class TargetEncodingState: - """Full-train state used to transform target-encoded features.""" - - sources: tuple[str, ...] - mapping: dict[str, float] - default_value: float - smoothing: float - min_count: int - round_numeric: bool = False - numeric_sources: tuple[str, ...] = () - round_decimals: int = 2 - - -@dataclass -class GroupByState: - """Train-only state for relative group-by features.""" - - category: str - numeric: str - operation: str - group_mean: dict[str, float] - global_mean: float - group_quantiles: dict[str, list[float]] = field(default_factory=dict) - global_quantiles: list[float] = field(default_factory=list) - - -def get_tabprep_variant(name: str) -> TabPrepVariantConfig: - """Return a known TabPrep challenger variant.""" - try: - return TABPREP_VARIANTS[name] - except KeyError as exc: - options = ", ".join(sorted(TABPREP_VARIANTS)) - raise ValueError(f"Unknown TabPrep variant {name!r}. Expected one of: {options}") from exc - - -def is_forbidden_feature(feature: str, *, extra_blacklist: Iterable[str] = ()) -> bool: - """Return True when a feature is unsafe for TabPrep generation.""" - lowered = feature.lower() - exact = FORBIDDEN_FEATURES | {item.lower() for item in extra_blacklist} - return ( - lowered == TARGET - or lowered in exact - or any(lowered.startswith(prefix) for prefix in FORBIDDEN_PREFIXES) - or any(token in lowered for token in FORBIDDEN_CONTAINS) - ) - - -def validate_no_forbidden_features( - features: Sequence[str], - *, - extra_blacklist: Iterable[str] = (), -) -> None: - """Fail fast if any candidate feature violates the TabPrep blacklist.""" - hits = [ - feature - for feature in features - if is_forbidden_feature(feature, extra_blacklist=extra_blacklist) - ] - if hits: - raise ValueError(f"Forbidden TabPrep input features detected: {sorted(hits)}") - - -def resolve_tabprep_input_features( - frame: pd.DataFrame, - *, - feature_config: Mapping[str, Any] | None = None, - extra_blacklist: Iterable[str] = (), -) -> list[str]: - """Resolve vetted CRPTO feature columns for TabPrep generation.""" - cfg = dict(feature_config or build_feature_config(frame)) - pool = list(cfg.get("CHALLENGER_FEATURE_POOL_V2") or []) - if not pool: - pool = list(cfg.get("CATBOOST_FEATURES") or []) - - resolved: list[str] = [] - for feature in pool: - if feature not in frame.columns: - continue - if is_forbidden_feature(str(feature), extra_blacklist=extra_blacklist): - continue - if feature == TARGET: - continue - resolved.append(str(feature)) - return list(dict.fromkeys(resolved)) - - -def resolve_tabprep_categorical_features( - features: Sequence[str], - *, - feature_config: Mapping[str, Any] | None = None, -) -> list[str]: - """Return TabPrep-safe categorical features from the selected feature pool.""" - cfg = dict(feature_config or {}) - configured = set(cfg.get("CATEGORICAL_FEATURES") or CATEGORICAL_FEATURES) - configured.update(cfg.get("INTERACTION_FEATURES") or INTERACTION_FEATURES) - feature_set = set(features) - return [feature for feature in features if feature in configured and feature in feature_set] - - -class TabPrepChallengerTransformer: - """Generate TabPrep-like features with train-only state. - - Target-aware features use out-of-fold values for ``fit_transform`` and - full-train smoothed mappings for later ``transform`` calls. - """ - - def __init__( - self, - *, - variant: str | TabPrepVariantConfig = "safe_500", - input_features: Sequence[str], - categorical_features: Sequence[str], - target: str = TARGET, - random_state: int = 42, - extra_blacklist: Iterable[str] = (), - ) -> None: - self.variant = get_tabprep_variant(variant) if isinstance(variant, str) else variant - self.input_features = list(dict.fromkeys(input_features)) - self.categorical_features = list(dict.fromkeys(categorical_features)) - self.target = target - self.random_state = int(random_state) - self.extra_blacklist = tuple(extra_blacklist) - - self.feature_specs_: list[FeatureSpec] = [] - self.numeric_features_: list[str] = [] - self.rsfc_base_features_: list[str] = [] - self.target_states_: dict[str, TargetEncodingState] = {} - self.groupby_states_: dict[str, GroupByState] = {} - self.generated_features_: list[str] = [] - self.global_target_mean_: float = 0.0 - self._is_fitted = False - - def fit_transform( - self, - frame: pd.DataFrame, - y: pd.Series | np.ndarray, - *, - issue_dates: pd.Series | None = None, - ) -> pd.DataFrame: - """Fit all generators on train-fit data and return OOF-safe generated features.""" - validate_no_forbidden_features(self.input_features, extra_blacklist=self.extra_blacklist) - x = frame.copy() - y_series = pd.Series(y, index=x.index, dtype=float) - self.global_target_mean_ = float(y_series.mean()) - - available = [feature for feature in self.input_features if feature in x.columns] - self.categorical_features = [ - feature for feature in self.categorical_features if feature in available - ][: self.variant.max_categorical_base_features] - self.numeric_features_ = self._select_numeric_features(x, y_series, available) - self.rsfc_base_features_ = self._select_rsfc_base_features(x, available) - - logger.info( - "TabPrep {} input: {} numeric, {} categorical, {} RSFC base features", - self.variant.name, - len(self.numeric_features_), - len(self.categorical_features), - len(self.rsfc_base_features_), - ) - - generated_train: dict[str, pd.Series] = {} - specs: list[FeatureSpec] = [] - - ar_specs = self._select_arithmetic_specs(x, y_series) - specs.extend(ar_specs) - generated_train.update({spec.name: self._apply_arithmetic(x, spec) for spec in ar_specs}) - - group_specs = self._select_groupby_specs(x, y_series) - specs.extend(group_specs) - generated_train.update({spec.name: self._apply_groupby(x, spec) for spec in group_specs}) - - te_specs, te_train = self._select_target_encoding_specs( - x, - y_series, - issue_dates=issue_dates, - ) - specs.extend(te_specs) - generated_train.update(te_train) - - rsfc_specs, rsfc_train = self._select_rsfc_specs( - x, - y_series, - issue_dates=issue_dates, - ) - specs.extend(rsfc_specs) - generated_train.update(rsfc_train) - - ranked = sorted(specs, key=lambda spec: (-spec.score, spec.generator, spec.name)) - ranked = ranked[: self.variant.max_generated_features] - rank_by_name = {spec.name: idx + 1 for idx, spec in enumerate(ranked)} - self.feature_specs_ = [ - FeatureSpec( - name=spec.name, - generator=spec.generator, - operation=spec.operation, - source_features=spec.source_features, - score=spec.score, - selected_rank=rank_by_name[spec.name], - metadata=spec.metadata, - ) - for spec in ranked - ] - self.generated_features_ = [spec.name for spec in self.feature_specs_] - self._is_fitted = True - - return pd.DataFrame( - { - feature: _clean_numeric(generated_train[feature]).astype("float32") - for feature in self.generated_features_ - }, - index=x.index, - ) - - def fit( - self, - frame: pd.DataFrame, - y: pd.Series | np.ndarray, - *, - issue_dates: pd.Series | None = None, - ) -> TabPrepChallengerTransformer: - """Fit the transformer and discard the training matrix.""" - self.fit_transform(frame, y, issue_dates=issue_dates) - return self - - def transform(self, frame: pd.DataFrame) -> pd.DataFrame: - """Transform a new split using train-fit state only.""" - if not self._is_fitted: - raise RuntimeError("TabPrepChallengerTransformer must be fitted before transform().") - return pd.DataFrame( - { - spec.name: _clean_numeric(self._apply_spec(frame, spec)).astype("float32") - for spec in self.feature_specs_ - }, - index=frame.index, - ) - - def feature_manifest(self) -> pd.DataFrame: - """Return selected generated feature provenance.""" - return pd.DataFrame([spec.to_row() for spec in self.feature_specs_]).sort_values( - "selected_rank", - ) - - def state_summary(self) -> dict[str, Any]: - """Return compact serializable metadata about the fitted transformer.""" - return { - "variant": self.variant.name, - "random_state": self.random_state, - "input_features": self.input_features, - "categorical_features": self.categorical_features, - "numeric_features": self.numeric_features_, - "rsfc_base_features": self.rsfc_base_features_, - "generated_features": self.generated_features_, - "global_target_mean": self.global_target_mean_, - } - - def _select_numeric_features( - self, - frame: pd.DataFrame, - y: pd.Series, - available: Sequence[str], - ) -> list[str]: - candidates: list[tuple[float, int, str]] = [] - categorical_set = set(self.categorical_features) - for feature in available: - if feature in categorical_set: - continue - series = pd.to_numeric(frame[feature], errors="coerce") - if series.notna().mean() < 0.20: - continue - nunique = int(series.nunique(dropna=True)) - if nunique < 3: - continue - score = _safe_abs_corr(series, y) - candidates.append((score, nunique, feature)) - candidates.sort(key=lambda item: (-item[0], -item[1], item[2])) - return [feature for _, _, feature in candidates[: self.variant.max_numeric_base_features]] - - def _select_rsfc_base_features( - self, frame: pd.DataFrame, available: Sequence[str] - ) -> list[str]: - categorical_set = set(self.categorical_features) - allowed: list[str] = [] - for feature in available: - if feature in categorical_set or feature in FLAG_FEATURES: - allowed.append(feature) - continue - if feature.endswith(MISSINGNESS_INDICATOR_SUFFIX): - allowed.append(feature) - continue - series = pd.to_numeric(frame[feature], errors="coerce") - nunique = int(series.nunique(dropna=True)) - if 2 <= nunique <= 25: - allowed.append(feature) - return list(dict.fromkeys(allowed))[:150] - - def _sample_for_scoring( - self, - frame: pd.DataFrame, - y: pd.Series, - ) -> tuple[pd.DataFrame, pd.Series]: - if len(frame) <= self.variant.max_scoring_rows: - return frame, y - sample = frame.sample(n=self.variant.max_scoring_rows, random_state=self.random_state) - return sample, y.loc[sample.index] - - def _select_arithmetic_specs(self, frame: pd.DataFrame, y: pd.Series) -> list[FeatureSpec]: - if not self.numeric_features_ or self.variant.arithmetic_features <= 0: - return [] - x_sample, y_sample = self._sample_for_scoring(frame[self.numeric_features_], y) - numeric = self.numeric_features_ - pair_candidates: list[FeatureSpec] = [] - operations = ("ratio", "diff", "product", "sum") - for left, right in itertools.combinations(numeric, 2): - for operation in operations: - source_orders = [(left, right)] - if operation in {"ratio", "diff"}: - source_orders.append((right, left)) - for sources in source_orders: - name = _make_feature_name("tp_ar", operation, sources) - spec = FeatureSpec( - name=name, - generator="arithmetic", - operation=operation, - source_features=sources, - score=0.0, - selected_rank=0, - ) - score = _safe_abs_corr(self._apply_arithmetic(x_sample, spec), y_sample) - if score > 0: - pair_candidates.append(_replace_score(spec, score)) - - triple_candidates: list[FeatureSpec] = [] - triple_base = numeric[: min(20, len(numeric))] - for triple_sources in itertools.combinations(triple_base, 3): - for operation in ("product3", "sum3", "ratio_sum"): - name = _make_feature_name("tp_ar", operation, triple_sources) - spec = FeatureSpec( - name=name, - generator="arithmetic", - operation=operation, - source_features=triple_sources, - score=0.0, - selected_rank=0, - ) - score = _safe_abs_corr(self._apply_arithmetic(x_sample, spec), y_sample) - if score > 0: - triple_candidates.append(_replace_score(spec, score)) - - candidates = pair_candidates + triple_candidates - candidates.sort(key=lambda spec: (-spec.score, spec.name)) - selected = candidates[: self.variant.arithmetic_features] - logger.info("Selected {} arithmetic TabPrep features", len(selected)) - return selected - - def _select_groupby_specs(self, frame: pd.DataFrame, y: pd.Series) -> list[FeatureSpec]: - if ( - not self.categorical_features - or not self.numeric_features_ - or self.variant.groupby_features <= 0 - ): - return [] - groupby_numeric = self.numeric_features_[: self.variant.max_groupby_numeric_features] - x_sample, y_sample = self._sample_for_scoring( - frame[[*self.categorical_features, *groupby_numeric]], - y, - ) - candidates: list[FeatureSpec] = [] - for category in self.categorical_features: - if int(frame[category].nunique(dropna=True)) < 2: - continue - for numeric in groupby_numeric: - for operation in ("mean_ratio", "mean_diff", "pct_rank"): - name = _make_feature_name("tp_gb", operation, (numeric, category)) - spec = FeatureSpec( - name=name, - generator="groupby", - operation=operation, - source_features=(numeric, category), - score=0.0, - selected_rank=0, - metadata={"category": category, "numeric": numeric}, - ) - state = _fit_groupby_state( - x_sample, - category=category, - numeric=numeric, - operation=operation, - min_support=max(2, min(self.variant.min_group_support, len(x_sample))), - ) - score = _safe_abs_corr(_apply_groupby_state(x_sample, state), y_sample) - if score > 0: - candidates.append(_replace_score(spec, score)) - candidates.sort(key=lambda spec: (-spec.score, spec.name)) - selected = candidates[: self.variant.groupby_features] - for spec in selected: - category = str(spec.metadata["category"]) - numeric = str(spec.metadata["numeric"]) - self.groupby_states_[spec.name] = _fit_groupby_state( - frame, - category=category, - numeric=numeric, - operation=spec.operation, - min_support=self.variant.min_group_support, - ) - logger.info("Selected {} group-by TabPrep features", len(selected)) - return selected - - def _select_target_encoding_specs( - self, - frame: pd.DataFrame, - y: pd.Series, - *, - issue_dates: pd.Series | None, - ) -> tuple[list[FeatureSpec], dict[str, pd.Series]]: - if not self.categorical_features: - return [], {} - base_sources = [(feature,) for feature in self.categorical_features] - interaction_sources: list[tuple[str, ...]] = [] - for order in (2, 3): - interaction_sources.extend(itertools.combinations(self.categorical_features, order)) - - selected_specs: list[FeatureSpec] = [] - train_features: dict[str, pd.Series] = {} - selected_base, base_train = self._score_target_encoding_sources( - frame, - y, - base_sources, - generator="target_encoding", - operation="oof_target_mean", - budget=self.variant.target_encoding_features, - issue_dates=issue_dates, - round_numeric=False, - ) - selected_interactions, interaction_train = self._score_target_encoding_sources( - frame, - y, - interaction_sources, - generator="categorical_interaction", - operation="oof_target_mean_interaction", - budget=self.variant.interaction_encoding_features, - issue_dates=issue_dates, - round_numeric=False, - ) - selected_specs.extend(selected_base) - selected_specs.extend(selected_interactions) - train_features.update(base_train) - train_features.update(interaction_train) - logger.info( - "Selected {} target/categorical-interaction TabPrep features", - len(selected_specs), - ) - return selected_specs, train_features - - def _select_rsfc_specs( - self, - frame: pd.DataFrame, - y: pd.Series, - *, - issue_dates: pd.Series | None, - ) -> tuple[list[FeatureSpec], dict[str, pd.Series]]: - if len(self.rsfc_base_features_) < 2 or self.variant.rsfc_features <= 0: - return [], {} - rng = np.random.default_rng(self.random_state) - candidate_count = self.variant.rsfc_features * self.variant.rsfc_candidate_multiplier - source_sets: list[tuple[str, ...]] = [] - seen: set[tuple[str, ...]] = set() - max_size = min(4, len(self.rsfc_base_features_)) - attempts = 0 - while len(source_sets) < candidate_count and attempts < candidate_count * 10: - attempts += 1 - size = int(rng.integers(2, max_size + 1)) - sources = tuple(sorted(rng.choice(self.rsfc_base_features_, size=size, replace=False))) - if sources in seen: - continue - seen.add(sources) - source_sets.append(sources) - selected, train_features = self._score_target_encoding_sources( - frame, - y, - source_sets, - generator="rsfc", - operation="random_subset_target_mean", - budget=self.variant.rsfc_features, - issue_dates=issue_dates, - round_numeric=True, - ) - logger.info("Selected {} RSFC TabPrep features", len(selected)) - return selected, train_features - - def _score_target_encoding_sources( - self, - frame: pd.DataFrame, - y: pd.Series, - sources_list: Sequence[tuple[str, ...]], - *, - generator: str, - operation: str, - budget: int, - issue_dates: pd.Series | None, - round_numeric: bool, - ) -> tuple[list[FeatureSpec], dict[str, pd.Series]]: - if budget <= 0: - return [], {} - candidates: list[tuple[FeatureSpec, TargetEncodingState, pd.Series]] = [] - numeric_sources = tuple(self.numeric_features_) - for sources in sources_list: - if any(source not in frame.columns for source in sources): - continue - name = _make_feature_name( - "tp_te" if generator != "rsfc" else "tp_rsfc", - operation, - sources, - ) - state, oof = _fit_oof_target_encoder( - frame, - y, - sources=sources, - n_folds=self.variant.n_oof_folds, - smoothing=self.variant.smoothing, - min_count=2, - issue_dates=issue_dates, - round_numeric=round_numeric, - numeric_sources=numeric_sources, - ) - score = _safe_abs_corr(oof, y) - if score <= 0: - continue - spec = FeatureSpec( - name=name, - generator=generator, - operation=operation, - source_features=sources, - score=score, - selected_rank=0, - metadata={"round_numeric": round_numeric}, - ) - candidates.append((spec, state, oof.rename(name))) - candidates.sort(key=lambda item: (-item[0].score, item[0].name)) - selected = candidates[:budget] - train_features: dict[str, pd.Series] = {} - specs: list[FeatureSpec] = [] - for spec, state, train_series in selected: - self.target_states_[spec.name] = state - specs.append(spec) - train_features[spec.name] = train_series - return specs, train_features - - def _apply_spec(self, frame: pd.DataFrame, spec: FeatureSpec) -> pd.Series: - if spec.generator == "arithmetic": - return self._apply_arithmetic(frame, spec) - if spec.generator == "groupby": - return self._apply_groupby(frame, spec) - if spec.generator in {"target_encoding", "categorical_interaction", "rsfc"}: - return self._apply_target_encoding(frame, spec) - raise ValueError(f"Unsupported TabPrep generator: {spec.generator}") - - def _apply_arithmetic(self, frame: pd.DataFrame, spec: FeatureSpec) -> pd.Series: - sources = spec.source_features - values = [_numeric_series(frame, source) for source in sources] - if spec.operation == "ratio": - return _safe_divide(values[0], values[1]) - if spec.operation == "diff": - return values[0] - values[1] - if spec.operation == "product": - return values[0] * values[1] - if spec.operation == "sum": - return values[0] + values[1] - if spec.operation == "product3": - return values[0] * values[1] * values[2] - if spec.operation == "sum3": - return values[0] + values[1] + values[2] - if spec.operation == "ratio_sum": - return _safe_divide(values[0], values[1] + values[2]) - raise ValueError(f"Unsupported arithmetic operation: {spec.operation}") - - def _apply_groupby(self, frame: pd.DataFrame, spec: FeatureSpec) -> pd.Series: - state = self.groupby_states_.get(spec.name) - if state is None: - raise RuntimeError(f"Missing group-by state for generated feature {spec.name!r}") - return _apply_groupby_state(frame, state) - - def _apply_target_encoding(self, frame: pd.DataFrame, spec: FeatureSpec) -> pd.Series: - state = self.target_states_.get(spec.name) - if state is None: - raise RuntimeError(f"Missing target encoder state for generated feature {spec.name!r}") - keys = _make_key( - frame, - state.sources, - round_numeric=state.round_numeric, - numeric_sources=set(state.numeric_sources), - round_decimals=state.round_decimals, - ) - return keys.map(state.mapping).fillna(state.default_value).astype(float) - - -def _replace_score(spec: FeatureSpec, score: float) -> FeatureSpec: - return FeatureSpec( - name=spec.name, - generator=spec.generator, - operation=spec.operation, - source_features=spec.source_features, - score=float(score), - selected_rank=spec.selected_rank, - metadata=spec.metadata, - ) - - -def _numeric_series(frame: pd.DataFrame, feature: str) -> pd.Series: - if feature not in frame.columns: - return pd.Series(np.nan, index=frame.index, dtype=float) - return pd.to_numeric(frame[feature], errors="coerce").astype(float) - - -def _clean_numeric(series: pd.Series) -> pd.Series: - return pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan) - - -def _safe_divide(numerator: pd.Series, denominator: pd.Series) -> pd.Series: - den = denominator.replace(0, np.nan) - return numerator / den - - -def _safe_abs_corr(series: pd.Series, y: pd.Series) -> float: - values = _clean_numeric(series) - target = pd.to_numeric(y.loc[values.index], errors="coerce").astype(float) - if values.notna().sum() < 3 or target.nunique(dropna=True) < 2: - return 0.0 - fill = values.median() - if not np.isfinite(fill): - fill = 0.0 - filled = values.fillna(float(fill)) - if filled.nunique(dropna=True) < 2: - return 0.0 - corr = np.corrcoef(filled.to_numpy(dtype=float), target.to_numpy(dtype=float))[0, 1] - if not np.isfinite(corr): - return 0.0 - return float(abs(corr)) - - -def _make_feature_name(prefix: str, operation: str, sources: Sequence[str]) -> str: - cleaned_sources = [_clean_name(source) for source in sources] - base = "__".join([prefix, _clean_name(operation), *cleaned_sources]) - if len(base) <= 120: - return base - digest = hashlib.sha1(base.encode("utf-8")).hexdigest()[:12] - return "__".join([prefix, _clean_name(operation), *cleaned_sources[:3], digest])[:120] - - -def _clean_name(value: str) -> str: - cleaned = "".join(char if char.isalnum() else "_" for char in str(value).lower()) - return "_".join(part for part in cleaned.split("_") if part) - - -def _category_key(series: pd.Series) -> pd.Series: - return series.astype("string").fillna(MISSING_TOKEN).astype(str) - - -def _make_key( - frame: pd.DataFrame, - sources: Sequence[str], - *, - round_numeric: bool = False, - numeric_sources: set[str] | None = None, - round_decimals: int = 2, -) -> pd.Series: - if not sources: - return pd.Series("", index=frame.index, dtype=str) - numeric_sources = numeric_sources or set() - pieces: list[pd.Series] = [] - for source in sources: - if source not in frame.columns: - piece = pd.Series(UNKNOWN_TOKEN, index=frame.index, dtype=str) - elif round_numeric and source in numeric_sources: - rounded = pd.to_numeric(frame[source], errors="coerce").round(round_decimals) - piece = rounded.astype("string").fillna(MISSING_TOKEN).astype(str) - else: - piece = _category_key(frame[source]) - pieces.append(piece) - key = pieces[0] - for piece in pieces[1:]: - key = key + "|" + piece - return key.astype(str) - - -def _fit_mapping( - keys: pd.Series, - y: pd.Series, - *, - smoothing: float, - default_value: float, - min_count: int, -) -> dict[str, float]: - tmp = pd.DataFrame({"key": keys.astype(str), "target": y.astype(float)}) - agg = tmp.groupby("key", dropna=False, observed=True)["target"].agg(["count", "sum"]) - encoded = (agg["sum"] + float(smoothing) * default_value) / (agg["count"] + float(smoothing)) - encoded = encoded[agg["count"] >= int(min_count)] - return {str(key): float(value) for key, value in encoded.items()} - - -def _fit_oof_target_encoder( - frame: pd.DataFrame, - y: pd.Series, - *, - sources: tuple[str, ...], - n_folds: int, - smoothing: float, - min_count: int, - issue_dates: pd.Series | None, - round_numeric: bool, - numeric_sources: tuple[str, ...], -) -> tuple[TargetEncodingState, pd.Series]: - numeric_source_set = set(numeric_sources) - keys = _make_key( - frame, - sources, - round_numeric=round_numeric, - numeric_sources=numeric_source_set, - ) - y_series = y.astype(float) - default_value = float(y_series.mean()) - encoded = pd.Series(default_value, index=frame.index, dtype=float) - folds = _temporal_fold_indices(frame, issue_dates=issue_dates, n_folds=n_folds) - for fold_idx in folds: - if len(fold_idx) == 0 or len(fold_idx) == len(frame): - continue - fold_index = frame.index[fold_idx] - fit_index = frame.index.difference(fold_index) - fold_default = float(y_series.loc[fit_index].mean()) if len(fit_index) else default_value - mapping = _fit_mapping( - keys.loc[fit_index], - y_series.loc[fit_index], - smoothing=smoothing, - default_value=fold_default, - min_count=min_count, - ) - encoded.loc[fold_index] = keys.loc[fold_index].map(mapping).fillna(fold_default) - - mapping = _fit_mapping( - keys, - y_series, - smoothing=smoothing, - default_value=default_value, - min_count=min_count, - ) - state = TargetEncodingState( - sources=sources, - mapping=mapping, - default_value=default_value, - smoothing=float(smoothing), - min_count=int(min_count), - round_numeric=round_numeric, - numeric_sources=numeric_sources, - ) - return state, encoded - - -def _temporal_fold_indices( - frame: pd.DataFrame, - *, - issue_dates: pd.Series | None, - n_folds: int, -) -> list[np.ndarray]: - n_folds = max(2, int(n_folds)) - if issue_dates is not None: - ordered = issue_dates.loc[frame.index].sort_values(kind="mergesort").index - positions = frame.index.get_indexer(ordered) - else: - positions = np.arange(len(frame), dtype=int) - return [fold.astype(int) for fold in np.array_split(positions, n_folds) if len(fold) > 0] - - -def _fit_groupby_state( - frame: pd.DataFrame, - *, - category: str, - numeric: str, - operation: str, - min_support: int, -) -> GroupByState: - groups = ( - _category_key(frame[category]) - if category in frame.columns - else pd.Series(MISSING_TOKEN, index=frame.index) - ) - values = _numeric_series(frame, numeric) - global_mean = float(values.mean()) if values.notna().any() else 0.0 - tmp = pd.DataFrame({"group": groups, "value": values}) - agg = tmp.groupby("group", observed=True)["value"].agg(["count", "mean"]) - supported = agg[agg["count"] >= int(min_support)] - group_mean = {str(key): float(value) for key, value in supported["mean"].dropna().items()} - state = GroupByState( - category=category, - numeric=numeric, - operation=operation, - group_mean=group_mean, - global_mean=global_mean, - ) - if operation == "pct_rank": - state.global_quantiles = _quantile_grid(values) - for group, group_values in tmp.groupby("group", observed=True)["value"]: - group_values = cast(pd.Series, group_values) - if len(group_values) >= int(min_support): - state.group_quantiles[str(group)] = _quantile_grid(group_values) - return state - - -def _apply_groupby_state(frame: pd.DataFrame, state: GroupByState) -> pd.Series: - groups = ( - _category_key(frame[state.category]) - if state.category in frame.columns - else pd.Series(MISSING_TOKEN, index=frame.index) - ) - values = _numeric_series(frame, state.numeric) - if state.operation in {"mean_ratio", "mean_diff"}: - means = groups.map(state.group_mean).fillna(state.global_mean).astype(float) - if state.operation == "mean_ratio": - return _safe_divide(values, means) - return values - means - if state.operation == "pct_rank": - return _apply_quantile_rank(values, groups, state) - raise ValueError(f"Unsupported group-by operation: {state.operation}") - - -def _quantile_grid(values: pd.Series, *, n_quantiles: int = 21) -> list[float]: - clean = _clean_numeric(values).dropna() - if len(clean) < 2: - return [] - quantiles = np.nanquantile(clean.to_numpy(dtype=float), np.linspace(0.0, 1.0, n_quantiles)) - return np.unique(quantiles.astype(float)).tolist() - - -def _apply_quantile_rank(values: pd.Series, groups: pd.Series, state: GroupByState) -> pd.Series: - result = pd.Series(np.nan, index=values.index, dtype=float) - global_bins = np.asarray(state.global_quantiles, dtype=float) - for group, idx in groups.groupby(groups, sort=False).groups.items(): - bins = np.asarray( - state.group_quantiles.get(str(group), state.global_quantiles), dtype=float - ) - if len(bins) < 2: - bins = global_bins - group_values = values.loc[idx].to_numpy(dtype=float) - if len(bins) < 2: - result.loc[idx] = np.nan - continue - ranks = np.searchsorted(bins, group_values, side="right") - 1 - denom = max(len(bins) - 1, 1) - result.loc[idx] = np.clip(ranks / denom, 0.0, 1.0) - return result diff --git a/src/ijds_audit/__init__.py b/src/ijds_audit/__init__.py new file mode 100644 index 0000000..860fce2 --- /dev/null +++ b/src/ijds_audit/__init__.py @@ -0,0 +1,24 @@ +"""Active IJDS binary-geometry and comparator-frontier audit.""" + +from __future__ import annotations + +from typing import Any + +__all__ = ["load_v4_config", "solve_point_portfolio", "summarize_binary_geometry"] + + +def __getattr__(name: str) -> Any: + """Preserve package-level conveniences without eager scientific imports.""" + if name == "load_v4_config": + from src.ijds_audit.config import load_v4_config + + return load_v4_config + if name == "summarize_binary_geometry": + from src.ijds_audit.geometry import summarize_binary_geometry + + return summarize_binary_geometry + if name == "solve_point_portfolio": + from src.ijds_audit.portfolio import solve_point_portfolio + + return solve_point_portfolio + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ijds_audit/allocation_granularity.py b/src/ijds_audit/allocation_granularity.py new file mode 100644 index 0000000..0c8cab4 --- /dev/null +++ b/src/ijds_audit/allocation_granularity.py @@ -0,0 +1,273 @@ +"""Outcome-free allocation granularity transform and sharp evaluation.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from src.evaluation.policy_contrast_bounds import PolicyContrastIndex + +PORTFOLIO_KEYS = ("window_id", "role", "period", "policy_label", "comparator_rule") +TRACK_KEYS = ("window_id", "candidate_id") + + +def floor_allocations_to_lot( + allocations: pd.DataFrame, + *, + lot_size: float, + committed_budget: float, + tolerance: float = 1.0e-8, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Round exposures down to a fixed lot and retain residual capital as cash.""" + required = { + *PORTFOLIO_KEYS, + "candidate_id", + "id", + "loan_amnt", + "exposure", + "expected_payoff_rate", + "pd_point", + "pd_effective", + "conformal_upper", + "frontier_cap", + "purpose", + "scenario_purpose_cap", + } + missing = sorted(required.difference(allocations.columns)) + if missing: + raise KeyError(f"Allocation granularity input is missing columns: {missing}.") + if not np.isfinite(lot_size) or lot_size <= 0.0: + raise ValueError("Lot size must be finite and positive.") + if not np.isfinite(committed_budget) or committed_budget <= 0.0: + raise ValueError("Committed budget must be finite and positive.") + + rounded = allocations.copy() + exposure = pd.to_numeric(rounded["exposure"], errors="raise").to_numpy(dtype=float) + loan_amount = pd.to_numeric(rounded["loan_amnt"], errors="raise").to_numpy(dtype=float) + if not bool(np.isfinite(exposure).all()) or bool((exposure < -tolerance).any()): + raise ValueError("Allocation exposures must be finite and non-negative.") + if bool((exposure > loan_amount + tolerance).any()): + raise ValueError("Allocation exposure exceeds the listed loan amount.") + + lot_exposure = np.floor((np.maximum(exposure, 0.0) + tolerance) / lot_size) * lot_size + lot_exposure = np.minimum(lot_exposure, loan_amount) + rounded["source_exposure"] = exposure + rounded["exposure"] = lot_exposure + rounded["allocation_fraction"] = np.divide( + lot_exposure, + loan_amount, + out=np.zeros_like(lot_exposure), + where=loan_amount > 0.0, + ) + rounded["weight"] = lot_exposure / committed_budget + rounded["expected_payoff_contribution"] = lot_exposure * rounded[ + "expected_payoff_rate" + ].to_numpy(dtype=float) + rounded["granularity_transform"] = f"floor_to_{lot_size:g}_cash_residual" + rounded = rounded.loc[rounded["exposure"].gt(tolerance)].copy() + + original_grouped = allocations.groupby(list(PORTFOLIO_KEYS), observed=True, sort=True) + rounded_grouped = rounded.groupby(list(PORTFOLIO_KEYS), observed=True, sort=True) + original_total = original_grouped["exposure"].sum().rename("source_allocated") + rounded_total = rounded_grouped["exposure"].sum().rename("rounded_allocated") + changed = ( + allocations.assign( + changed=np.abs(exposure - lot_exposure) > tolerance, + source_partial=(exposure > tolerance) & (exposure < loan_amount - tolerance), + ) + .groupby(list(PORTFOLIO_KEYS), observed=True, sort=True) + .agg( + changed_positions=("changed", "sum"), source_partial_positions=("source_partial", "sum") + ) + ) + rounded_partial = ( + rounded.assign( + rounded_partial=( + rounded["exposure"].to_numpy(dtype=float) + < rounded["loan_amnt"].to_numpy(dtype=float) - tolerance + ) + ) + .groupby(list(PORTFOLIO_KEYS), observed=True, sort=True)["rounded_partial"] + .sum() + .rename("rounded_partial_positions") + ) + audit = pd.concat( + [original_total, rounded_total, changed, rounded_partial], axis=1 + ).reset_index() + if bool(audit[["rounded_allocated", "rounded_partial_positions"]].isna().any(axis=None)): + raise RuntimeError("Lot rounding removed an entire portfolio.") + audit["cash_residual"] = committed_budget - audit["rounded_allocated"] + audit["cash_share"] = audit["cash_residual"] / committed_budget + audit["lot_size"] = float(lot_size) + audit["committed_budget"] = float(committed_budget) + if not np.allclose( + audit["source_allocated"].to_numpy(dtype=float), + committed_budget, + atol=1.0e-4, + rtol=0.0, + ): + raise RuntimeError("Source portfolios do not use the declared committed budget.") + if bool((audit["cash_residual"] < -tolerance).any()): + raise RuntimeError("Lot rounding increased invested capital.") + + lot_remainder = np.mod(rounded["exposure"].to_numpy(dtype=float), lot_size) + if not bool(np.all(np.minimum(lot_remainder, lot_size - lot_remainder) <= tolerance)): + raise RuntimeError("Rounded exposures are not exact lot multiples.") + _validate_rounded_constraints( + rounded, + committed_budget=committed_budget, + tolerance=tolerance, + ) + return rounded.reset_index(drop=True), audit.reset_index(drop=True) + + +def rounded_solve_records( + records: pd.DataFrame, + rounded: pd.DataFrame, + audit: pd.DataFrame, + *, + committed_budget: float, +) -> pd.DataFrame: + """Update frozen solve records after the deterministic floor transform.""" + indexed = rounded.groupby(list(PORTFOLIO_KEYS), observed=True, sort=True).agg( + n_positive_exposure=("id", "size"), + total_allocated=("exposure", "sum"), + expected_objective=("expected_payoff_contribution", "sum"), + ) + weighted = ( + rounded.assign( + weighted_pd_point_contribution=rounded["exposure"] * rounded["pd_point"], + weighted_pd_effective_contribution=rounded["exposure"] * rounded["pd_effective"], + weighted_upper_contribution=rounded["exposure"] * rounded["conformal_upper"], + ) + .groupby(list(PORTFOLIO_KEYS), observed=True, sort=True)[ + [ + "weighted_pd_point_contribution", + "weighted_pd_effective_contribution", + "weighted_upper_contribution", + ] + ] + .sum() + ) + indexed["weighted_pd_point"] = weighted["weighted_pd_point_contribution"] / committed_budget + indexed["weighted_pd_effective"] = ( + weighted["weighted_pd_effective_contribution"] / committed_budget + ) + indexed["weighted_conformal_upper"] = weighted["weighted_upper_contribution"] / committed_budget + indexed = indexed.reset_index() + + result = records.merge( + indexed, + on=list(PORTFOLIO_KEYS), + how="left", + validate="one_to_one", + suffixes=("_source", ""), + ).merge( + audit[[*PORTFOLIO_KEYS, "cash_residual", "cash_share", "changed_positions"]], + on=list(PORTFOLIO_KEYS), + how="left", + validate="one_to_one", + ) + if bool(result[["total_allocated", "cash_residual"]].isna().any(axis=None)): + raise RuntimeError("Rounded solve records do not cover every source portfolio.") + result["budget_residual"] = result["cash_residual"] + result["constraint_slack"] = result["risk_tolerance"] - result["weighted_pd_effective"] + result["granularity_transform"] = "floor_to_lot_cash_residual" + return result + + +def granularity_contrast_bounds( + continuous: pd.DataFrame, + rounded: pd.DataFrame, + outcomes: pd.DataFrame, + *, + committed_budget: float, + periods: Sequence[str], + lgd: float, +) -> pd.DataFrame: + """Evaluate rounded-minus-continuous contrasts with cash in the denominator.""" + outcome_columns = ["id", "snapshot_default"] + if bool(outcomes["id"].duplicated().any()): + raise ValueError("Outcome IDs must be unique for granularity evaluation.") + expected_periods = tuple(str(period) for period in periods) + rows: list[dict[str, Any]] = [] + for raw_keys, source in continuous.groupby(list(TRACK_KEYS), observed=True, sort=True): + keys = raw_keys if isinstance(raw_keys, tuple) else (raw_keys,) + window_id, candidate_id = (str(value) for value in keys) + transformed = rounded.loc[ + rounded["window_id"].eq(window_id) & rounded["candidate_id"].eq(candidate_id) + ].copy() + if set(source["period"].astype(str)) != set(expected_periods): + raise RuntimeError( + f"Continuous track {window_id}/{candidate_id} has an incomplete period grid." + ) + if set(transformed["period"].astype(str)) != set(expected_periods): + raise RuntimeError( + f"Rounded track {window_id}/{candidate_id} has an incomplete period grid." + ) + + source = source.copy() + source["policy_label"] = "continuous" + transformed["policy_label"] = "rounded_lot" + combined = pd.concat([source, transformed], ignore_index=True) + facts = ( + combined[["id", "contractual_rate", "conformal_lower", "conformal_upper"]] + .drop_duplicates() + .merge(outcomes[outcome_columns], on="id", how="left", validate="one_to_one") + ) + if len(facts) != combined["id"].nunique(): + raise RuntimeError("Granularity outcome join did not preserve the funded union.") + committed = committed_budget * len(expected_periods) + bounds = PolicyContrastIndex( + combined, + role="primary_oot", + loan_facts=facts, + ).sharp_bounds( + policy_a="rounded_lot", + policy_b="continuous", + lgd=lgd, + normalization_capital_a=committed, + normalization_capital_b=committed, + ) + rows.append( + { + "window_id": window_id, + "candidate_id": candidate_id, + "frontier_ruler": str(source["frontier_ruler"].iloc[0]), + "frontier_coordinate": float(source["frontier_coordinate"].iloc[0]), + "gamma": float(source["gamma"].iloc[0]), + "periods": len(expected_periods), + "cash_residual_total": committed - float(bounds["policy_a_capital"]), + "cash_share": 1.0 - float(bounds["policy_a_capital"]) / committed, + **bounds, + } + ) + return pd.DataFrame(rows).sort_values(list(TRACK_KEYS), kind="stable").reset_index(drop=True) + + +def _validate_rounded_constraints( + rounded: pd.DataFrame, + *, + committed_budget: float, + tolerance: float, +) -> None: + grouped = rounded.groupby(list(PORTFOLIO_KEYS), observed=True, sort=False) + risk = grouped.apply( + lambda frame: float((frame["exposure"] * frame["pd_effective"]).sum()) / committed_budget, + include_groups=False, + ) + risk_cap = grouped["frontier_cap"].first() + if bool((risk > risk_cap + tolerance).any()): + raise RuntimeError("Lot rounding violated a risk constraint.") + purpose = ( + rounded.groupby([*PORTFOLIO_KEYS, "purpose"], observed=True, sort=False)["exposure"].sum() + / committed_budget + ) + purpose_cap = rounded.groupby([*PORTFOLIO_KEYS, "purpose"], observed=True, sort=False)[ + "scenario_purpose_cap" + ].first() + if bool((purpose > purpose_cap + tolerance).any()): + raise RuntimeError("Lot rounding violated a purpose constraint.") diff --git a/src/ijds_audit/allocations.py b/src/ijds_audit/allocations.py new file mode 100644 index 0000000..6ccf059 --- /dev/null +++ b/src/ijds_audit/allocations.py @@ -0,0 +1,450 @@ +"""Outcome-free V4 guardrail, comparator, and exact-frontier construction.""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.evaluation.maturity_safe_portfolio import solve_outcome_free_allocation +from src.evaluation.standardized_credit_payoff import expected_objective_coefficients +from src.ijds_audit.portfolio import ( + PointPortfolioSession, + PointPortfolioSolution, + c2_cap, + verify_c2_dominance, +) +from src.optimization.policy_selection import LinearPolicyCandidate, build_linear_policy_grid + + +@dataclass(frozen=True) +class OutcomeFreePortfolioAudit: + """Frozen solve records, funded rows, and declared comparator supports.""" + + records: pd.DataFrame + allocations: pd.DataFrame + comparator_support: pd.DataFrame + frontier_breakpoints: pd.DataFrame + + +def policy_family(config: Mapping[str, Any]) -> tuple[LinearPolicyCandidate, ...]: + grid = build_linear_policy_grid( + risk_tolerances=[float(value) for value in config["policy"]["risk_tolerances"]], + gammas=[float(value) for value in config["policy"]["gammas"]], + uncertainty_aversions=[float(value) for value in config["policy"]["uncertainty_aversions"]], + ) + if len(grid) != 9: + raise RuntimeError("The V4 guardrail family must contain exactly nine policies.") + return tuple( + LinearPolicyCandidate( + candidate_id=candidate.candidate_id, + risk_tolerance=candidate.risk_tolerance, + gamma=candidate.gamma, + uncertainty_aversion=candidate.uncertainty_aversion, + policy_mode=candidate.policy_mode, + delta_cap_quantile=candidate.delta_cap_quantile, + tail_focus_quantile=candidate.tail_focus_quantile, + min_budget_utilization=float(config["policy"]["min_budget_utilization_solver"]), + pd_cap_slack_penalty=0.0, + ) + for candidate in grid + ) + + +def _cell_config(config: Mapping[str, Any]) -> dict[str, Any]: + cell = copy.deepcopy(dict(config)) + cell["execution"]["random_seed"] = int(config["model"]["canonical_seed"]) + cell["policy"]["max_concentration_by_purpose"] = float( + config["policy"]["max_concentration_by_purpose"] + ) + return cell + + +def _period_frames(panel: pd.DataFrame, role: str) -> tuple[tuple[str, pd.DataFrame], ...]: + frame = panel.loc[panel["design_split"].eq(role)].drop(columns="design_split") + periods = pd.to_datetime(frame["issue_d"]).dt.to_period("M") + return tuple( + (str(period), frame.loc[periods.eq(period)].copy()) for period in sorted(periods.unique()) + ) + + +def declared_menu_counts(config: Mapping[str, Any]) -> tuple[int, int]: + """Return the exact development and primary month counts declared by the design.""" + design = config["design"] + development = pd.period_range( + str(design["policy_development_start"]), + str(design["policy_development_end"]), + freq="M", + ) + primary = pd.period_range( + str(design["primary_oot_start_month"]), + str(design["primary_oot_end_month"]), + freq="M", + ) + return len(development), len(primary) + + +def _guardrail_solve( + month: pd.DataFrame, + *, + candidate: LinearPolicyCandidate, + config: Mapping[str, Any], + role: str, + period: str, + window_id: str, +) -> tuple[dict[str, Any], pd.DataFrame]: + record, allocation = solve_outcome_free_allocation( + month, + candidate, + config=config, + robust=True, + role=role, + period=period, + policy_label=f"guardrail_{candidate.candidate_id}", + ) + metadata = { + "window_id": window_id, + "comparator_rule": "guardrail", + "paired_policy_id": candidate.candidate_id, + "frontier_cap": np.nan, + } + record.update(metadata) + return record, allocation.assign(**metadata) + + +def _solution_allocation( + month: pd.DataFrame, + solution: PointPortfolioSolution, + *, + role: str, + period: str, + window_id: str, + comparator_rule: str, + paired_policy_id: str, + risk_cap: float, + lgd: float, + allocation_tolerance: float, +) -> tuple[dict[str, Any], pd.DataFrame]: + active = solution.exposure > float(allocation_tolerance) + funded = month.loc[active].copy() + funded["allocation_fraction"] = solution.allocation_fraction[active] + funded["exposure"] = solution.exposure[active] + funded["weight"] = funded["exposure"] / solution.total_allocated + funded["pd_effective"] = funded["pd_point"] + funded["expected_payoff_rate"] = expected_objective_coefficients( + funded["pd_point"].to_numpy(dtype=float), + funded["contractual_rate"].to_numpy(dtype=float), + lgd=float(lgd), + ) + funded["expected_payoff_contribution"] = funded["exposure"] * funded["expected_payoff_rate"] + policy_label = f"{comparator_rule}_{paired_policy_id}" + if comparator_rule == "point_cap_frontier": + policy_label = f"point_cap_frontier_{float(risk_cap):.12g}" + metadata = { + "role": role, + "period": period, + "window_id": window_id, + "policy_label": policy_label, + "candidate_id": f"point-{comparator_rule}-{paired_policy_id}-{period}", + "comparator_rule": comparator_rule, + "paired_policy_id": paired_policy_id, + "frontier_cap": float(risk_cap), + } + funded = funded.assign(**metadata) + record: dict[str, Any] = { + **metadata, + "risk_tolerance": float(risk_cap), + "gamma": 0.0, + "uncertainty_aversion": 0.0, + "policy_mode": "point_estimate", + "robust_guardrail": False, + "solver_status": "Optimal", + "solver_backend_actual": "highspy_exact_budget_simplex", + "expected_objective": float(solution.objective_value), + "n_candidates": int(len(month)), + "n_positive_exposure": int(active.sum()), + "total_allocated": float(solution.total_allocated), + "weighted_pd_point": float(solution.weighted_point_score), + "weighted_pd_effective": float(solution.weighted_point_score), + "weighted_conformal_upper": float( + funded["weight"].to_numpy(dtype=float) @ funded["conformal_upper"].to_numpy(dtype=float) + ), + "basis_cap_lower": float(solution.basis_cap_lower), + "basis_cap_upper": float(solution.basis_cap_upper), + "highs_simplex_iterations": int(solution.simplex_iterations), + } + return record, funded + + +def _point_session( + month: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> PointPortfolioSession: + point = month["pd_point"].to_numpy(dtype=float) + objective = expected_objective_coefficients( + point, + month["contractual_rate"].to_numpy(dtype=float), + lgd=float(config["payoff"]["lgd"]), + ) + return PointPortfolioSession( + month, + point_score=point, + objective_rate=objective, + budget=float(config["policy"]["budget"]), + purpose_cap=float(config["policy"]["max_concentration_by_purpose"]), + time_limit=int(config["execution"]["solver_time_limit_seconds"]), + threads=int(config["execution"]["threads"]), + ) + + +def _funded_vector(month: pd.DataFrame, allocation: pd.DataFrame) -> np.ndarray: + exposure = allocation.set_index("id")["exposure"] + if bool(exposure.index.duplicated().any()): + raise RuntimeError("A guardrail allocation contains duplicate IDs.") + return month["id"].map(exposure).fillna(0.0).to_numpy(dtype=float) + + +def _unique_caps(values: list[float], *, tolerance: float = 1e-10) -> tuple[float, ...]: + output: list[float] = [] + for value in sorted(float(item) for item in values): + if not output or value - output[-1] > tolerance: + output.append(value) + return tuple(output) + + +def build_outcome_free_portfolios( + panels: Mapping[str, pd.DataFrame], + config: Mapping[str, Any], +) -> OutcomeFreePortfolioAudit: + """Build all V4 allocations without accepting an outcome dataframe.""" + if set(panels) != {str(window["id"]) for window in config["residual_specification"]["windows"]}: + raise ValueError("Portfolio panels must cover exactly the eight declared windows.") + cell = _cell_config(config) + policies = policy_family(config) + records: list[dict[str, Any]] = [] + allocations: list[pd.DataFrame] = [] + support_rows: list[dict[str, Any]] = [] + support_caps: list[float] = [] + point_cache: dict[tuple[str, str], PointPortfolioSolution] = {} + point_sessions: dict[str, PointPortfolioSession] = {} + expected_development, expected_primary = declared_menu_counts(config) + + for window_id, panel in panels.items(): + development = _period_frames(panel, "policy_development") + primary = _period_frames(panel, "primary_oot") + if len(development) != expected_development or len(primary) != expected_primary: + raise RuntimeError( + f"{window_id} requires {expected_development} development and " + f"{expected_primary} primary OOT monthly menus; observed " + f"{len(development)} and {len(primary)}." + ) + for policy in policies: + development_moments: list[float] = [] + for period, month in development: + record, funded = _guardrail_solve( + month, + candidate=policy, + config=cell, + role="policy_development", + period=period, + window_id=window_id, + ) + moment = c2_cap( + funded["exposure"].to_numpy(dtype=float), + funded["pd_point"].to_numpy(dtype=float), + ) + record["guardrail_funded_point_moment"] = moment + funded["guardrail_funded_point_moment"] = moment + records.append(record) + allocations.append(funded) + development_moments.append(moment) + c1 = float(np.average(development_moments)) + support_lower = float(min(development_moments)) + support_upper = float(max(development_moments)) + support_caps.extend((support_lower, support_upper)) + support_rows.append( + { + "window_id": window_id, + "paired_policy_id": policy.candidate_id, + "development_months": len(development_moments), + "c1_cap": c1, + "support_lower": support_lower, + "support_upper": support_upper, + } + ) + for period, month in primary: + guard_record, guard_funded = _guardrail_solve( + month, + candidate=policy, + config=cell, + role="primary_oot", + period=period, + window_id=window_id, + ) + guard_vector = _funded_vector(month, guard_funded) + point = month["pd_point"].to_numpy(dtype=float) + objective = expected_objective_coefficients( + point, + month["contractual_rate"].to_numpy(dtype=float), + lgd=float(config["payoff"]["lgd"]), + ) + contemporaneous = c2_cap(guard_vector, point) + guard_record["guardrail_funded_point_moment"] = contemporaneous + guard_funded["guardrail_funded_point_moment"] = contemporaneous + records.append(guard_record) + allocations.append(guard_funded) + for rule, cap in ( + ("c0_same_numeric_cap", float(policy.risk_tolerance)), + ("c1_development_mean", c1), + ("c2_contemporaneous", contemporaneous), + ): + cache_key = (period, float(cap).hex()) + solution = point_cache.get(cache_key) + if solution is None: + session = point_sessions.get(period) + if session is None: + session = _point_session(month, config=cell) + point_sessions[period] = session + solution = session.solve(cap) + point_cache[cache_key] = solution + record, funded = _solution_allocation( + month, + solution, + role="primary_oot", + period=period, + window_id=window_id, + comparator_rule=rule, + paired_policy_id=policy.candidate_id, + risk_cap=cap, + lgd=float(config["payoff"]["lgd"]), + allocation_tolerance=float(config["execution"]["allocation_tolerance"]), + ) + record.update( + c1_cap=c1, + c2_cap=contemporaneous, + development_support_lower=support_lower, + development_support_upper=support_upper, + ) + funded = funded.assign( + c1_cap=c1, + c2_cap=contemporaneous, + development_support_lower=support_lower, + development_support_upper=support_upper, + ) + if rule == "c2_contemporaneous": + diagnostics = verify_c2_dominance( + guardrail_exposure=guard_vector, + point_solution=solution, + point_score=point, + objective_rate=objective, + tolerance=float( + config["comparators"]["exact_point_cap_frontier"][ + "objective_dominance_tolerance" + ] + ), + ) + match_residual = float(solution.weighted_point_score - contemporaneous) + if abs(match_residual) > float( + config["comparators"]["exact_point_cap_frontier"]["cap_tolerance"] + ): + raise RuntimeError( + f"C2 funded point-risk match failed for {window_id} " + f"{policy.candidate_id} {period}: {match_residual:.3e}." + ) + record.update(diagnostics, c2_match_residual=match_residual) + funded = funded.assign(c2_match_residual=match_residual) + records.append(record) + allocations.append(funded) + + frontier = config["comparators"]["exact_point_cap_frontier"] + frontier_rows: list[dict[str, Any]] = [] + reference_panels = dict(panels) + reference = next(iter(reference_panels.values())) + reference_months = dict(_period_frames(reference, "primary_oot")) + breakpoints_by_period: dict[str, tuple[float, ...]] = {} + enumerated_lower = min(float(frontier["start"]), min(support_caps)) + enumerated_upper = max(float(frontier["stop"]), max(support_caps)) + for period, reference_month in reference_months.items(): + session = point_sessions.get(period) + if session is None: + session = _point_session(reference_month, config=cell) + point_sessions[period] = session + breakpoints = session.basis_breakpoints( + lower_cap=enumerated_lower, + upper_cap=enumerated_upper, + tolerance=float(frontier["cap_tolerance"]), + ) + breakpoints_by_period[period] = breakpoints + global_caps = _unique_caps( + [ + *support_caps, + *(cap for caps in breakpoints_by_period.values() for cap in caps), + ], + tolerance=float(frontier["cap_tolerance"]), + ) + for period in reference_months: + for cap in global_caps: + cache_key = (period, float(cap).hex()) + solution = point_cache.get(cache_key) + if solution is None: + solution = point_sessions[period].solve(cap) + point_cache[cache_key] = solution + frontier_rows.append( + { + "period": period, + "frontier_cap": cap, + "basis_cap_lower": solution.basis_cap_lower, + "basis_cap_upper": solution.basis_cap_upper, + "is_enumerated_support_breakpoint": any( + abs(cap - item) <= float(frontier["cap_tolerance"]) + for item in breakpoints_by_period[period] + ), + "is_global_breakpoint": any( + abs(cap - item) <= float(frontier["cap_tolerance"]) + for breakpoints in breakpoints_by_period.values() + for item in breakpoints + ), + "objective_value": solution.objective_value, + "weighted_pd_point": solution.weighted_point_score, + } + ) + record, funded = _solution_allocation( + reference_months[period], + solution, + role="primary_oot", + period=period, + window_id="__shared_point_frontier__", + comparator_rule="point_cap_frontier", + paired_policy_id="frontier", + risk_cap=cap, + lgd=float(config["payoff"]["lgd"]), + allocation_tolerance=float(config["execution"]["allocation_tolerance"]), + ) + window_specific = [ + "conformal_lower", + "conformal_upper", + "conformal_group", + "learner", + "taxonomy_groups", + ] + funded = funded.drop(columns=window_specific, errors="ignore") + record["weighted_conformal_upper"] = np.nan + records.append(record) + allocations.append(funded) + + result_records = pd.DataFrame(records) + result_allocations = pd.concat(allocations, ignore_index=True) + forbidden = {"loan_status", "snapshot_default", "terminal_default", "total_pymnt"} + if forbidden.intersection(column.casefold() for column in result_allocations.columns): + raise AssertionError("Outcome-free allocation artifact contains an outcome field.") + return OutcomeFreePortfolioAudit( + records=result_records, + allocations=result_allocations, + comparator_support=pd.DataFrame(support_rows), + frontier_breakpoints=pd.DataFrame(frontier_rows), + ) diff --git a/src/ijds_audit/claim_ledger.py b/src/ijds_audit/claim_ledger.py new file mode 100644 index 0000000..114c783 --- /dev/null +++ b/src/ijds_audit/claim_ledger.py @@ -0,0 +1,230 @@ +"""Nonnumeric claim contracts bound to the single paper evidence manifest.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +from src.utils.artifact_descriptor import relative_artifact_descriptor + +ALLOWED_STATUSES = {"active", "pending", "retired"} +ALLOWED_KINDS = {"empirical", "theorem", "boundary"} +ALLOWED_RULES = {"equals", "documented"} + + +def _surface_registry(value: object) -> dict[str, str]: + if not isinstance(value, Mapping) or not value: + raise TypeError("Claim ledger must declare named surfaces.") + if not all(isinstance(name, str) and isinstance(path, str) for name, path in value.items()): + raise TypeError("Claim-ledger surfaces must map names to paths.") + return {str(name): str(path) for name, path in value.items()} + + +def _validate_result_rule(claim: Mapping[str, Any], *, claim_id: str) -> None: + source = claim.get("result_pointer") + if claim["rule"] == "documented": + if source is not None or "expected" in claim: + raise ValueError(f"Documented claim {claim_id!r} cannot declare a result value.") + return + if not isinstance(source, str) or not source.startswith("/"): + raise ValueError(f"Claim {claim_id!r} requires a JSON result pointer.") + expected = claim.get("expected") + if isinstance(expected, (int, float)) and not isinstance(expected, bool): + raise ValueError(f"Claim {claim_id!r} duplicates a numeric result.") + if not isinstance(expected, (bool, str)) and expected is not None: + raise TypeError(f"Claim {claim_id!r} has a nonportable expected value.") + + +def _claim_surface_contract( + value: object, + *, + claim_id: str, + known_surfaces: set[str], +) -> dict[str, list[str]]: + if not isinstance(value, Mapping): + raise TypeError(f"Claim {claim_id!r} must declare surfaces.") + contract: dict[str, list[str]] = {} + for field in ("required", "allowed"): + names = value.get(field) + if not isinstance(names, list) or not all(isinstance(name, str) for name in names): + raise TypeError(f"Claim {claim_id!r} surfaces.{field} must be a string list.") + contract[field] = [str(name) for name in names] + unknown = sorted(set(names).difference(known_surfaces)) + if unknown: + raise ValueError(f"Claim {claim_id!r} references unknown surfaces: {unknown}.") + if not set(contract["required"]).issubset(contract["allowed"]): + raise ValueError(f"Claim {claim_id!r} requires a forbidden surface.") + return contract + + +def _validated_claim(value: object, *, known_surfaces: set[str]) -> dict[str, Any]: + if not isinstance(value, dict): + raise TypeError("Every claim contract must be a mapping.") + claim = dict(value) + claim_id = claim.get("id") + if not isinstance(claim_id, str) or not claim_id or not claim_id.isascii(): + raise TypeError("Claim IDs must be nonempty ASCII strings.") + for field, allowed in ( + ("status", ALLOWED_STATUSES), + ("kind", ALLOWED_KINDS), + ("rule", ALLOWED_RULES), + ): + if claim.get(field) not in allowed: + raise ValueError(f"Claim {claim_id!r} has an invalid {field}.") + _validate_result_rule(claim, claim_id=claim_id) + claim["surfaces"] = _claim_surface_contract( + claim.get("surfaces"), + claim_id=claim_id, + known_surfaces=known_surfaces, + ) + boundaries = claim.get("forbidden_inference") + if not isinstance(boundaries, list) or not all(isinstance(item, str) for item in boundaries): + raise TypeError(f"Claim {claim_id!r} needs forbidden-inference boundaries.") + return claim + + +def load_claim_ledger(path: Path) -> dict[str, Any]: + """Load and structurally validate a nonnumeric claim ledger.""" + raw_payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw_payload, dict): + raise TypeError("Claim ledger must be a mapping.") + payload = dict(raw_payload) + if payload.get("status") != "active_ijds_claim_contract": + raise ValueError("Unexpected claim-ledger status.") + surfaces = _surface_registry(payload.get("surfaces")) + raw_claims = payload.get("claims") + if not isinstance(raw_claims, list) or not raw_claims: + raise TypeError("Claim ledger must contain claims.") + claims = [_validated_claim(claim, known_surfaces=set(surfaces)) for claim in raw_claims] + identifiers = [str(claim["id"]) for claim in claims] + if len(set(identifiers)) != len(identifiers): + raise ValueError("Claim IDs must be globally unique.") + payload["surfaces"] = surfaces + payload["claims"] = claims + return payload + + +def resolve_json_pointer(document: Mapping[str, Any], pointer: str) -> Any: + """Resolve an RFC 6901 object pointer without array-index extensions.""" + if pointer == "": + return document + if not pointer.startswith("/"): + raise ValueError("JSON pointer must be empty or start with '/'.") + current: Any = document + for raw_token in pointer[1:].split("/"): + token = raw_token.replace("~1", "/").replace("~0", "~") + if not isinstance(current, Mapping) or token not in current: + raise KeyError(f"JSON pointer does not resolve: {pointer}.") + current = current[token] + return current + + +def materialize_claim_ledger( + ledger_path: Path, + *, + evidence: Mapping[str, Any], + repo_root: Path, + check_surfaces: bool = True, +) -> dict[str, Any]: + """Verify claim results and markers, returning a manifest-ready ledger.""" + ledger = load_claim_ledger(ledger_path) + root = repo_root.resolve() + surface_paths = {name: (root / path).resolve() for name, path in ledger["surfaces"].items()} + for path in surface_paths.values(): + path.relative_to(root) + surface_text = _read_surface_text(surface_paths) if check_surfaces else {} + + materialized = [ + _materialize_claim( + claim, + evidence=evidence, + surface_text=surface_text, + check_surfaces=check_surfaces, + ) + for claim in ledger["claims"] + ] + return { + "schema_version": str(ledger["schema_version"]), + "status": str(ledger["status"]), + "numeric_authority": "parent_evidence_manifest_only", + "contract": relative_artifact_descriptor(ledger_path, repo_root=repo_root), + "claims": materialized, + } + + +def _read_surface_text(surface_paths: Mapping[str, Path]) -> dict[str, str]: + text: dict[str, str] = {} + for name, path in surface_paths.items(): + if not path.is_file(): + raise FileNotFoundError(f"Claim surface {name!r} is missing: {path}.") + text[name] = path.read_text(encoding="utf-8") + return text + + +def _active_claim_result(claim: Mapping[str, Any], evidence: Mapping[str, Any]) -> Any: + if claim["status"] != "active" or claim["rule"] != "equals": + return None + claim_id = str(claim["id"]) + result = resolve_json_pointer(evidence, str(claim["result_pointer"])) + if isinstance(result, (int, float)) and not isinstance(result, bool): + raise ValueError(f"Claim {claim_id!r} resolves to a duplicated numeric result.") + if result != claim.get("expected"): + raise RuntimeError(f"Claim {claim_id!r} failed: {result!r} != {claim.get('expected')!r}.") + return result + + +def _check_surface_markers(claim: Mapping[str, Any], surface_text: Mapping[str, str]) -> None: + if claim["status"] != "active": + return + claim_id = str(claim["id"]) + marker = f"claim:{claim_id}" + required = set(claim["surfaces"]["required"]) + allowed = set(claim["surfaces"]["allowed"]) + missing = sorted(name for name in required if marker not in surface_text[name]) + forbidden = sorted( + name for name, text in surface_text.items() if name not in allowed and marker in text + ) + if missing: + raise RuntimeError(f"Claim {claim_id!r} is missing markers on {missing}.") + if forbidden: + raise RuntimeError(f"Claim {claim_id!r} appears on forbidden surfaces {forbidden}.") + + +def _materialize_claim( + claim: Mapping[str, Any], + *, + evidence: Mapping[str, Any], + surface_text: Mapping[str, str], + check_surfaces: bool, +) -> dict[str, Any]: + result = _active_claim_result(claim, evidence) + if check_surfaces: + _check_surface_markers(claim, surface_text) + return { + "id": str(claim["id"]), + "status": str(claim["status"]), + "kind": str(claim["kind"]), + "lineages": list(claim.get("lineages", [])), + "scope": str(claim.get("scope", "")), + "rule": str(claim["rule"]), + "result_pointer": claim.get("result_pointer"), + "result": result if claim["status"] == "active" else None, + "forbidden_inference": list(claim["forbidden_inference"]), + "surfaces": dict(claim["surfaces"]), + } + + +def claim_markers_from_manifest(evidence_path: Path) -> set[str]: + """Return active claim markers from one generated evidence manifest.""" + payload = json.loads(evidence_path.read_text(encoding="utf-8")) + ledger = payload.get("claim_ledger", {}) + claims = ledger.get("claims", []) if isinstance(ledger, Mapping) else [] + return { + f"claim:{claim['id']}" + for claim in claims + if isinstance(claim, Mapping) and claim.get("status") == "active" + } diff --git a/src/ijds_audit/config.py b/src/ijds_audit/config.py new file mode 100644 index 0000000..b49e55d --- /dev/null +++ b/src/ijds_audit/config.py @@ -0,0 +1,572 @@ +"""Configuration loading and closed-design validation for the V4 audit.""" + +from __future__ import annotations + +import copy +import math +from collections.abc import Mapping +from pathlib import Path +from typing import Any, cast + +import pandas as pd +import yaml + +_ALLOWED_TOP_LEVEL_KEYS = frozenset( + { + "schema_version", + "protocol_status", + "protocol_tag", + "run_tag", + "hypothesis", + "source", + "target", + "design", + "model", + "probability_calibration", + "conformal", + "payoff", + "policy", + "comparators", + "analysis", + "simulation", + "execution", + "output", + "protocol_lineage_files", + "residual_specification", + "learner_control", + "resume_outcome_free", + "credit_risk_controls", + "resume_credit_control_freeze", + "evaluation_recovery", + "rolling_origin", + "decision_active_simulation", + "endpoint_reason_recovery", + } +) + +_ALLOWED_SECTION_KEYS = { + "source": frozenset( + { + "charged_off_reporting_lag_months", + "csv_chunksize", + "information_cutoff", + "minimum_label_retention", + "raw_path", + "required_raw_columns", + "snapshot_date", + "snapshot_date_role", + } + ), + "target": frozenset( + { + "evaluation_outcome_contract", + "name", + "negative_definition", + "positive_definition", + "right_censored_definition", + } + ), + "design": frozenset( + { + "censored_extension_end_month", + "censored_extension_start_month", + "conformal_fit_end", + "conformal_fit_start", + "development_end", + "endpoint", + "historical_archive_previously_inspected", + "policy_development_end", + "policy_development_start", + "primary_oot_end_month", + "primary_oot_start_month", + "probability_calibration_end", + "probability_calibration_start", + "term_months", + "validation_tail_fraction", + } + ), + "payoff": frozenset( + { + "expected_formula", + "id", + "lgd", + "realized_formula", + "reoptimization_lgd_grid", + "secondary_cash_metric", + } + ), + "policy": frozenset( + { + "budget", + "canonical_purpose_cap", + "family", + "gammas", + "max_concentration_by_purpose", + "min_budget_utilization_solver", + "outcome_based_selection", + "purpose_cap_sensitivity", + "risk_tolerances", + "score_ablations", + "uncertainty_aversions", + } + ), + "analysis": frozenset( + { + "all_eight_windows_primary", + "all_nine_policies_primary", + "canonical_seed_only_for_v4_portfolio", + "comparator_reversal_requires", + "confirmatory_language_forbidden", + "finite_population_estimand", + "miscoverage_direction_requires", + "nested_scope_counts_are_not_independent", + "no_result_based_promotion", + "report_discrete_set_geometry", + "report_every_cell", + "report_exact_frontier_breakpoints", + "universal_direction_requires", + "unresolved_outcomes", + } + ), + "residual_specification": frozenset( + {"minimum_monthly_label_retention", "window_months", "windows"} + ), + "execution": frozenset( + { + "allocation_tolerance", + "solver_backend", + "solver_time_limit_seconds", + "strict_solver_backend", + "threads", + } + ), + "rolling_origin": frozenset( + { + "audit_id", + "common_primary_months", + "origin_year", + "outcome_based_origin_selection", + "pooled_origin_claims", + "reference_origin_year", + } + ), +} + +_CREDIT_CONTROL_MODELS = ( + "catboost_platt", + "numeric_logistic_platt", + "catboost_monotonic_platt", + "woe_scorecard_platform_platt", + "woe_scorecard_borrower_platt", +) +_FREEZE_RESUME_FIELDS = frozenset( + { + "source_run_tag", + "source_protocol_tag", + "source_protocol_commit", + "source_freeze_sha256", + } +) + + +def _reject_unknown_keys( + payload: Mapping[str, Any], allowed: frozenset[str], *, context: str +) -> None: + unknown = sorted(str(key) for key in payload if str(key) not in allowed) + if unknown: + raise KeyError(f"Unknown {context} keys: {unknown}.") + + +def _validate_known_config_keys(config: Mapping[str, Any]) -> None: + """Reject silent top-level and critical-section YAML typos.""" + _reject_unknown_keys(config, _ALLOWED_TOP_LEVEL_KEYS, context="V4 config") + for section, allowed in _ALLOWED_SECTION_KEYS.items(): + value = config.get(section) + if value is not None: + if not isinstance(value, Mapping): + raise TypeError(f"V4 config section {section} must be a mapping.") + _reject_unknown_keys(value, allowed, context=f"V4 {section}") + target = config.get("target") + if isinstance(target, Mapping): + contract = target.get("evaluation_outcome_contract") + if contract is not None: + if not isinstance(contract, Mapping): + raise TypeError("Evaluation outcome contract must be a mapping.") + _reject_unknown_keys( + contract, + frozenset( + { + "mode", + "cutoff", + "terminal_status_source", + "archive_is_verified_point_in_time_snapshot", + "charged_off_reporting_lag_months", + } + ), + context="evaluation outcome contract", + ) + + +def _deep_merge(base: dict[str, Any], override: Mapping[str, Any]) -> dict[str, Any]: + merged = copy.deepcopy(base) + for key, value in override.items(): + name = str(key) + if value is None: + merged.pop(name, None) + elif isinstance(value, Mapping) and isinstance(merged.get(name), dict): + merged[name] = _deep_merge(merged[name], value) + else: + merged[name] = copy.deepcopy(value) + return merged + + +def _require_lowercase_sha256(value: Any, *, label: str) -> str: + digest = str(value) + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError(f"{label} must be a lowercase hexadecimal SHA-256.") + return digest + + +def _validate_freeze_resume(resume: Mapping[str, Any], *, label: str) -> None: + missing = sorted(_FREEZE_RESUME_FIELDS.difference(resume)) + if missing: + raise KeyError(f"{label} is missing fields: {missing}.") + _require_lowercase_sha256(resume["source_freeze_sha256"], label=f"{label} freeze digest") + + +def load_config_payload(path: Path, seen: frozenset[Path] = frozenset()) -> dict[str, Any]: + """Load one recursively inherited YAML mapping with cycle protection.""" + resolved = path.resolve() + if resolved in seen: + raise ValueError(f"Protocol config inheritance cycle at {resolved}.") + payload = yaml.safe_load(resolved.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Protocol config must be a YAML mapping.") + extends = payload.pop("extends", None) + if extends is None: + return payload + return _deep_merge( + load_config_payload((resolved.parent / str(extends)).resolve(), seen | {resolved}), + payload, + ) + + +def _validate_windows(config: Mapping[str, Any]) -> None: + design = config["design"] + specification = config["residual_specification"] + windows = specification["windows"] + if len(windows) != 8 or int(specification["window_months"]) != 6: + raise ValueError("V4 requires exactly eight six-month residual windows.") + conformal_start = pd.Timestamp(design["conformal_fit_start"]) + conformal_end = pd.Timestamp(design["conformal_fit_end"]) + expected_starts = pd.date_range(conformal_start, periods=8, freq="MS") + identifiers: set[str] = set() + observed_end: pd.Timestamp | None = None + for expected_start, window in zip(expected_starts, windows, strict=True): + identifier = str(window["id"]) + if identifier in identifiers: + raise ValueError(f"Duplicate residual-window id: {identifier}.") + identifiers.add(identifier) + start = cast(pd.Timestamp, pd.Timestamp(window["start"])) + end = cast(pd.Timestamp, pd.Timestamp(window["end"])) + expected_end = start + pd.offsets.MonthEnd(6) + if start != cast(pd.Timestamp, expected_start) or end != expected_end: + raise ValueError( + f"Residual window {identifier} is not the declared consecutive six-month window." + ) + observed_end = end + if observed_end != conformal_end: + raise ValueError("The final residual window must end at design.conformal_fit_end.") + + +def _validate_design_chronology(config: Mapping[str, Any]) -> None: + design = config["design"] + development_end = pd.Timestamp(design["development_end"]) + calibration_start = pd.Timestamp(design["probability_calibration_start"]) + calibration_end = pd.Timestamp(design["probability_calibration_end"]) + conformal_start = pd.Timestamp(design["conformal_fit_start"]) + conformal_end = pd.Timestamp(design["conformal_fit_end"]) + policy_start = pd.Timestamp(design["policy_development_start"]) + policy_end = pd.Timestamp(design["policy_development_end"]) + + if calibration_start != development_end + pd.Timedelta(days=1): + raise ValueError("Probability calibration must start immediately after PD development.") + if calibration_end != calibration_start + pd.offsets.MonthEnd(12): + raise ValueError("Probability calibration must contain exactly twelve complete months.") + if conformal_start != calibration_end + pd.Timedelta(days=1): + raise ValueError("Residual fitting must start immediately after probability calibration.") + if policy_start != conformal_end + pd.Timedelta(days=1): + raise ValueError("Policy development must start immediately after residual fitting.") + if policy_end != policy_start + pd.offsets.MonthEnd(11): + raise ValueError("Policy development must contain exactly eleven complete months.") + + cutoff = pd.Timestamp(config["source"]["information_cutoff"]) + primary_start_period = cast( + pd.Period, pd.Period(str(design["primary_oot_start_month"]), freq="M") + ) + primary_start = primary_start_period.start_time + primary_end = cast(pd.Period, pd.Period(str(design["primary_oot_end_month"]), freq="M")) + extension_start = cast( + pd.Period, pd.Period(str(design["censored_extension_start_month"]), freq="M") + ) + extension_end = cast( + pd.Period, pd.Period(str(design["censored_extension_end_month"]), freq="M") + ) + if cutoff != primary_start - pd.Timedelta(days=1): + raise ValueError("The information cutoff must immediately precede primary OOT.") + if primary_end.ordinal < primary_start_period.ordinal: + raise ValueError("Primary OOT has an invalid month range.") + if ( + extension_start.ordinal != primary_end.ordinal + 1 + or extension_end.ordinal < extension_start.ordinal + ): + raise ValueError("The censored extension must immediately follow primary OOT.") + + +def _validate_evaluation_outcome_contract(config: Mapping[str, Any]) -> None: + contract = config["target"].get("evaluation_outcome_contract") + if contract is None: + return + expected_mode = "conservative_terminal_status_reconstruction" + if contract.get("mode") != expected_mode: + raise ValueError(f"Evaluation outcome mode must be {expected_mode!r}.") + cutoff = pd.Timestamp(contract.get("cutoff")) + if pd.isna(cutoff): + raise ValueError("Evaluation outcome contract requires a valid cutoff.") + if cutoff != pd.Timestamp(config["source"]["snapshot_date"]): + raise ValueError("Endpoint reconstruction cutoff must match source.snapshot_date.") + if contract.get("archive_is_verified_point_in_time_snapshot") is not False: + raise ValueError( + "The distributed archive cannot be declared a verified point-in-time snapshot." + ) + if contract.get("terminal_status_source") != "distributed_archive_final_status": + raise ValueError("Unexpected terminal-status source for endpoint reconstruction.") + if int(contract.get("charged_off_reporting_lag_months", -1)) != int( + config["source"]["charged_off_reporting_lag_months"] + ): + raise ValueError("Evaluation and fitting charge-off lags must agree.") + + +def _validate_endpoint_reason_recovery(config: Mapping[str, Any]) -> None: + recovery = config.get("endpoint_reason_recovery") + if recovery is None: + return + if not isinstance(recovery, Mapping): + raise TypeError("Endpoint reason recovery must be a mapping.") + status = recovery.get("status") + if status == "reason_taxonomy_only_no_scientific_metric_change": + if recovery.get("require_exact_reference_column_equivalence") is not True: + raise ValueError("Endpoint reason recovery must require exact scientific equivalence.") + elif status == "reason_taxonomy_only_machine_tolerance_recovery": + if recovery.get("require_exact_non_float_reference_equivalence") is not True: + raise ValueError("Endpoint recovery must retain exact non-floating equivalence.") + if recovery.get("equivalence_mode") != "exact_non_float_machine_tolerant_float": + raise ValueError("Endpoint recovery has an invalid equivalence mode.") + for field in ("float_atol", "float_rtol"): + tolerance = float(recovery.get(field, -1.0)) + if not math.isfinite(tolerance) or not 0.0 <= tolerance <= 1.0e-12: + raise ValueError(f"Endpoint recovery {field} exceeds the machine-drift ceiling.") + else: + raise ValueError("Unexpected endpoint reason recovery status.") + if not isinstance(recovery.get("reference_json"), Mapping): + raise KeyError("Endpoint reason recovery requires a reference JSON descriptor.") + if recovery.get("artifact_section") not in {"artifacts", "evaluation_artifacts"}: + raise ValueError("Endpoint reason recovery has an invalid artifact section.") + + +def _validate_rolling_origin(config: Mapping[str, Any]) -> None: + rolling = config.get("rolling_origin") + if rolling is None: + return + origin = int(rolling["origin_year"]) + expected_source = {"information_cutoff": f"{origin}-03-31"} + expected_design = { + "development_end": f"{origin - 6}-12-31", + "probability_calibration_start": f"{origin - 5}-01-01", + "probability_calibration_end": f"{origin - 5}-12-31", + "conformal_fit_start": f"{origin - 4}-01-01", + "conformal_fit_end": f"{origin - 3}-01-31", + "policy_development_start": f"{origin - 3}-02-01", + "policy_development_end": f"{origin - 3}-12-31", + "primary_oot_start_month": f"{origin}-04", + "primary_oot_end_month": f"{origin}-06", + "censored_extension_start_month": f"{origin}-07", + "censored_extension_end_month": f"{origin}-09", + } + for field, expected in expected_source.items(): + if str(config["source"][field]) != expected: + raise ValueError(f"Rolling origin {origin} has an asymmetric source.{field}.") + for field, expected in expected_design.items(): + if str(config["design"][field]) != expected: + raise ValueError(f"Rolling origin {origin} has an asymmetric design.{field}.") + if int(rolling.get("common_primary_months", -1)) != 3: + raise ValueError("The rolling-origin common primary horizon must contain three months.") + if int(rolling.get("reference_origin_year", -1)) != 2016: + raise ValueError("The rolling-origin reference year must remain 2016.") + if rolling.get("outcome_based_origin_selection") is not False: + raise ValueError("Outcome-based origin selection is forbidden.") + if rolling.get("pooled_origin_claims") is not False: + raise ValueError("Pooled rolling-origin claims are forbidden.") + + +def load_v4_config(path: Path) -> dict[str, Any]: + """Load V4 and reject any expansion of its closed analysis family.""" + config = load_config_payload(path) + _validate_known_config_keys(config) + required = { + "source", + "target", + "design", + "model", + "probability_calibration", + "conformal", + "residual_specification", + "learner_control", + "payoff", + "policy", + "comparators", + "analysis", + "simulation", + "execution", + "output", + } + missing = sorted(required.difference(config)) + if missing: + raise KeyError(f"V4 protocol config is missing sections: {missing}.") + if config["protocol_status"] != "locked_retrospective_complete_specification_audit": + raise ValueError("Unexpected V4 protocol status.") + if config["design"].get("historical_archive_previously_inspected") is not True: + raise ValueError("The inspected-archive disclosure must remain true.") + if config["policy"].get("outcome_based_selection") is not False: + raise ValueError("Outcome-based policy selection is forbidden.") + if config["comparators"].get("selection_from_outcomes") is not False: + raise ValueError("Outcome-based comparator selection is forbidden.") + if config["analysis"].get("all_eight_windows_primary") is not True: + raise ValueError("All eight residual windows must remain co-primary.") + if config["analysis"].get("all_nine_policies_primary") is not True: + raise ValueError("All nine policies must remain co-primary.") + if [int(value) for value in config["conformal"]["diagnostic_group_counts"]] != [1, 2, 5, 10]: + raise ValueError("The closed taxonomy diagnostic grid is 1/2/5/10.") + if int(config["conformal"]["canonical_groups"]) != 5: + raise ValueError("The canonical taxonomy must contain five groups.") + if [int(value) for value in config["model"]["sensitivity_seeds"]] != [40, 41, 42, 43, 44]: + raise ValueError("Inherited seed lineage changed unexpectedly.") + if int(config["model"]["canonical_seed"]) != 42: + raise ValueError("V4 portfolio optimization is locked to seed 42.") + if len(config["policy"]["risk_tolerances"]) * len(config["policy"]["gammas"]) != 9: + raise ValueError("The closed portfolio family must contain nine policies.") + support = config["comparators"]["exact_point_cap_frontier"] + if not 0.0 <= float(support["start"]) < float(support["stop"]) <= 1.0: + raise ValueError("The broad frontier support must be a nonempty subset of [0, 1].") + if config["learner_control"].get("portfolio_optimization") is not False: + raise ValueError("The logistic learner is coverage-only.") + resume = config.get("resume_outcome_free") + if resume: + if not isinstance(resume, Mapping): + raise TypeError("Outcome-free import must be a mapping.") + _validate_freeze_resume(resume, label="Outcome-free import") + _validate_windows(config) + _validate_design_chronology(config) + _validate_evaluation_outcome_contract(config) + _validate_endpoint_reason_recovery(config) + _validate_rolling_origin(config) + return config + + +def _validate_credit_control_scope(controls: Mapping[str, Any]) -> None: + models = tuple(str(value) for value in controls.get("co_primary_models", [])) + if models != _CREDIT_CONTROL_MODELS: + raise ValueError("The five reported learner controls must remain co-primary.") + required_flags = { + "all_models_reported": True, + "selection_from_oot": False, + "portfolio_optimization": False, + "sampling": "none_all_eligible_rows", + } + for field, expected in required_flags.items(): + if controls.get(field) != expected: + raise ValueError(f"Credit-risk control field {field!r} must equal {expected!r}.") + reference = controls.get("active_score_reference") + if not isinstance(reference, Mapping): + raise KeyError("The active V4 score reference is required.") + _require_lowercase_sha256(reference.get("sha256", ""), label="Active score reference") + + +def _active_model_features(config: Mapping[str, Any]) -> set[str]: + return { + *[str(value) for value in config["model"]["numeric_features"]], + *[str(value) for value in config["model"]["categorical_features"]], + } + + +def _validate_monotonic_control(controls: Mapping[str, Any], *, active_features: set[str]) -> None: + monotonic = controls.get("monotonic_catboost") + if not isinstance(monotonic, Mapping): + raise ValueError("Monotonic CatBoost requires a configuration mapping.") + constraints = monotonic.get("constraints") + if not isinstance(constraints, Mapping) or not constraints: + raise ValueError("Monotonic CatBoost requires a nonempty constraint map.") + if not set(map(str, constraints)).issubset(active_features): + raise ValueError("Monotonic constraints must use active model features only.") + if {int(value) for value in constraints.values()}.difference({-1, 1}): + raise ValueError("Monotonic constraints must be -1 or +1.") + + +def _validate_scorecard_controls(controls: Mapping[str, Any], *, active_features: set[str]) -> None: + scorecards = controls.get("scorecards") + if not isinstance(scorecards, Mapping) or set(scorecards) != {"platform", "borrower"}: + raise ValueError("Exactly the platform and borrower scorecards are required.") + features_by_scorecard: dict[str, set[str]] = {} + for name, raw_specification in scorecards.items(): + if not isinstance(raw_specification, Mapping): + raise TypeError(f"Scorecard {name} must be a mapping.") + features = [str(value) for value in raw_specification.get("features", [])] + if not features or len(features) != len(set(features)): + raise ValueError(f"Scorecard {name} has empty or duplicate features.") + if not set(features).issubset(active_features): + raise ValueError(f"Scorecard {name} uses a feature outside the active contract.") + features_by_scorecard[str(name)] = set(features) + platform_signals = {str(value) for value in controls.get("platform_signal_features", [])} + if not platform_signals.issubset(features_by_scorecard["platform"]): + raise ValueError("The platform scorecard must include every declared platform signal.") + if platform_signals.intersection(features_by_scorecard["borrower"]): + raise ValueError("The borrower scorecard must exclude declared platform signals.") + + +def _validate_credit_evaluation_recovery(config: Mapping[str, Any]) -> None: + recovery = config.get("evaluation_recovery") + if not recovery: + return + if not isinstance(recovery, Mapping): + raise TypeError("Credit-control evaluation recovery must be a mapping.") + required_values = { + "status": "numerical_calibration_recovery_only", + "require_exact_coverage_equivalence": True, + "calibration_solver": "sklearn_unpenalized_lbfgs", + } + for field, expected in required_values.items(): + if recovery.get(field) != expected: + raise ValueError(f"Credit-control recovery field {field!r} must equal {expected!r}.") + reference = recovery.get("coverage_reference") + if not isinstance(reference, Mapping): + raise KeyError("Evaluation recovery requires a coverage reference.") + _require_lowercase_sha256(reference.get("sha256", ""), label="Coverage reference") + + +def load_credit_control_config(path: Path) -> dict[str, Any]: + """Load the V4-aligned, coverage-only credit-risk control protocol.""" + config = load_v4_config(path) + controls = config.get("credit_risk_controls") + if not isinstance(controls, Mapping): + raise KeyError("Credit-risk control config is missing credit_risk_controls.") + _validate_credit_control_scope(controls) + active_features = _active_model_features(config) + _validate_monotonic_control(controls, active_features=active_features) + _validate_scorecard_controls(controls, active_features=active_features) + resume = config.get("resume_credit_control_freeze") + if resume: + if not isinstance(resume, Mapping): + raise TypeError("Credit-control import must be a mapping.") + _validate_freeze_resume(resume, label="Credit-control import") + _validate_credit_evaluation_recovery(config) + return config diff --git a/src/ijds_audit/credit_control_protocol.py b/src/ijds_audit/credit_control_protocol.py new file mode 100644 index 0000000..7f46afc --- /dev/null +++ b/src/ijds_audit/credit_control_protocol.py @@ -0,0 +1,569 @@ +"""Two-phase orchestration for the IJDS credit-risk learner controls.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from src.ijds_audit.config import load_credit_control_config +from src.ijds_audit.credit_controls import ( + ScorecardFit, + credit_prediction_metrics, + feature_variation_audit, + fit_monotonic_catboost_control, + fit_woe_scorecard_control, + score_psi_audit, + scorecard_feature_psi_audit, +) +from src.ijds_audit.endpoint_recovery import reconcile_from_json_reference +from src.ijds_audit.evaluation import endpoint_resolution_audit, temporal_coverage_audit +from src.ijds_audit.prediction import ( + LearnerScores, + fit_logistic_control, + fit_primary_scores, + fit_window_recipes, + fixed_taxonomy_edges, + prepare_data, +) +from src.ijds_audit.protocol import ( + configured_archive_outcomes, + load_outcome_universe, + load_recipes, + outcome_free_geometry, + recipe_payload, + score_frame, + verified_freeze_artifact_paths, +) +from src.utils.isolated_experiment import ( + environment_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, + save_catboost_model_atomic, +) +from src.utils.pipeline_runtime import ( + atomic_write_json, + atomic_write_parquet, + atomic_write_pickle, +) + +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +OUTCOME_COLUMNS = frozenset( + { + "loan_status", + "snapshot_default", + "snapshot_resolution", + "terminal_default", + "label_available", + "label_available_at", + "total_pymnt", + } +) + + +def _implementation(config_path: Path, repo_root: Path) -> dict[str, Any]: + config = load_credit_control_config(config_path) + return implementation_provenance( + config_path=config_path, + repo_root=repo_root, + relative_paths=[ + Path("scripts/experiments/run_ijds_credit_risk_controls.py"), + Path("src/ijds_audit/config.py"), + Path("src/ijds_audit/prediction.py"), + Path("src/ijds_audit/credit_controls.py"), + Path("src/ijds_audit/credit_control_protocol.py"), + Path("src/ijds_audit/evaluation.py"), + Path("src/ijds_audit/endpoint_recovery.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/data/outcome_observability.py"), + Path("src/features/feature_engineering.py"), + Path("src/models/binary_conformal_guardrail.py"), + Path("src/models/maturity_safe_pd.py"), + *[Path(value) for value in config.get("protocol_lineage_files", [])], + ], + ) + + +def _align_active_reference( + *, + data: Any, + learners: tuple[LearnerScores, LearnerScores], + config: dict[str, Any], + repo_root: Path, +) -> tuple[tuple[LearnerScores, LearnerScores], dict[str, Any]]: + reference_spec = config["credit_risk_controls"]["active_score_reference"] + reference_path = resolve_repo_input(reference_spec["path"], repo_root=repo_root) + descriptor = relative_artifact_descriptor(reference_path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if descriptor[field] != reference_spec[field]: + raise RuntimeError(f"Active V4 score reference mismatch for {field}.") + reference = pd.read_parquet(reference_path) + identity = pd.DataFrame( + { + "id": data.universe["id"].astype("string"), + "design_split": data.universe["design_split"].astype("string"), + "row_order": np.arange(len(data.universe), dtype=np.int64), + } + ) + aligned = identity.merge( + reference, + on=["id", "design_split"], + how="left", + validate="one_to_one", + suffixes=("", "_reference"), + ).sort_values("row_order", kind="stable") + missing_reference = bool(aligned.filter(like="pd_").isna().to_numpy().any()) + if len(aligned) != len(identity) or missing_reference: + raise RuntimeError("The active V4 score reference does not align to the design universe.") + + groups = [int(value) for value in config["conformal"]["diagnostic_group_counts"]] + calibration_mask = data.universe["design_split"].eq("probability_calibration").to_numpy() + tolerance = float(reference_spec["numerical_tolerance"]) + replacements: list[LearnerScores] = [] + audit: dict[str, Any] = {"reference": descriptor, "tolerance": tolerance, "learners": {}} + for learner in learners: + column = f"pd_{learner.name}" + expected = aligned[column].to_numpy(dtype=float) + absolute = np.abs(learner.probabilities - expected) + maximum = float(np.max(absolute)) + if maximum > tolerance: + raise RuntimeError( + f"Refit {learner.name} differs from active V4 scores by {maximum:.3e}." + ) + metrics = dict(learner.metrics) + metrics["active_reference_max_abs_difference"] = maximum + replacements.append( + replace( + learner, + probabilities=expected, + taxonomy_edges=fixed_taxonomy_edges(expected[calibration_mask], groups), + metrics=metrics, + ) + ) + audit["learners"][learner.name] = { + "max_abs_difference": maximum, + "rows": int(len(expected)), + "within_tolerance": True, + } + return (replacements[0], replacements[1]), audit + + +def _scorecard_artifacts(scorecards: tuple[ScorecardFit, ...]) -> dict[str, pd.DataFrame]: + return { + "woe_summary": pd.concat([item.summary for item in scorecards], ignore_index=True), + "woe_coefficients": pd.concat( + [item.coefficients for item in scorecards], ignore_index=True + ), + "woe_binning_table": pd.concat( + [item.binning_table for item in scorecards], ignore_index=True + ), + } + + +def freeze_credit_controls(*, config_path: Path, repo_root: Path) -> Path: + """Fit and hash all learner scores and residual recipes before OOT outcomes.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_credit_control_config(resolved_config) + if config.get("resume_credit_control_freeze"): + raise ValueError("The evaluation config cannot create a new credit-control freeze.") + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + raw_path = resolve_repo_input(config["source"]["raw_path"], repo_root=root) + data = prepare_data(config, raw_path=raw_path) + primary = fit_primary_scores(data, config) + logistic = fit_logistic_control(data, config) + (primary, logistic), reference_audit = _align_active_reference( + data=data, + learners=(primary, logistic), + config=config, + repo_root=root, + ) + monotonic = fit_monotonic_catboost_control(data, config) + platform = fit_woe_scorecard_control(data, config, specification="platform") + borrower = fit_woe_scorecard_control(data, config, specification="borrower") + scorecards = (platform, borrower) + learners = (primary, logistic, monotonic, platform.scores, borrower.scores) + expected = tuple(str(value) for value in config["credit_risk_controls"]["co_primary_models"]) + if tuple(learner.name for learner in learners) != expected: + raise RuntimeError("Fitted learner order differs from the closed protocol family.") + + windows = {learner.name: fit_window_recipes(data, learner, config) for learner in learners} + scores = score_frame(data, learners) + forbidden = sorted(OUTCOME_COLUMNS.intersection(scores.columns)) + if forbidden: + raise RuntimeError(f"Outcome columns entered the frozen score artifact: {forbidden}.") + fit_audit = pd.concat( + [window.fit_audit for learner in windows.values() for window in learner.values()], + ignore_index=True, + ) + feature_audit = feature_variation_audit(data) + score_psi = score_psi_audit( + data, + learners, + bins=int(config["credit_risk_controls"]["score_psi_bins"]), + ) + scorecard_psi = pd.concat( + [scorecard_feature_psi_audit(data, item.model) for item in scorecards], + ignore_index=True, + ) + scorecard_frames = _scorecard_artifacts(scorecards) + artifact_files = { + "scores": atomic_write_parquet(scores, paths.data_dir / "prediction/scores.parquet"), + "recipes": atomic_write_json( + paths.model_dir / "prediction/residual_recipes.json", + recipe_payload(windows), + ), + "fit_audit": atomic_write_parquet( + fit_audit, + paths.data_dir / "prediction/residual_fit_audit.parquet", + ), + "outcome_free_geometry": atomic_write_parquet( + outcome_free_geometry(scores, windows), + paths.data_dir / "prediction/outcome_free_geometry.parquet", + ), + "feature_variation": atomic_write_parquet( + feature_audit, + paths.data_dir / "diagnostics/active_feature_variation.parquet", + ), + "score_psi": atomic_write_parquet( + score_psi, + paths.data_dir / "diagnostics/score_psi.parquet", + ), + "scorecard_feature_psi": atomic_write_parquet( + scorecard_psi, + paths.data_dir / "diagnostics/scorecard_feature_psi.parquet", + ), + **{ + name: atomic_write_parquet( + frame, + paths.data_dir / f"diagnostics/{name}.parquet", + ) + for name, frame in scorecard_frames.items() + }, + } + model_files = { + "catboost": save_catboost_model_atomic( + primary.model, + paths.model_dir / "prediction/catboost_seed42.cbm", + ), + "catboost_platt": atomic_write_pickle( + paths.model_dir / "prediction/catboost_platt.pkl", + primary.calibrator, + ), + "numeric_logistic": atomic_write_pickle( + paths.model_dir / "prediction/numeric_logistic.pkl", + logistic.model, + ), + "numeric_logistic_platt": atomic_write_pickle( + paths.model_dir / "prediction/numeric_logistic_platt.pkl", + logistic.calibrator, + ), + "catboost_monotonic": save_catboost_model_atomic( + monotonic.model, + paths.model_dir / "prediction/catboost_monotonic_seed42.cbm", + ), + "catboost_monotonic_platt": atomic_write_pickle( + paths.model_dir / "prediction/catboost_monotonic_platt.pkl", + monotonic.calibrator, + ), + "woe_scorecard_platform": atomic_write_pickle( + paths.model_dir / "prediction/woe_scorecard_platform.pkl", + platform.model, + ), + "woe_scorecard_platform_platt": atomic_write_pickle( + paths.model_dir / "prediction/woe_scorecard_platform_platt.pkl", + platform.scores.calibrator, + ), + "woe_scorecard_borrower": atomic_write_pickle( + paths.model_dir / "prediction/woe_scorecard_borrower.pkl", + borrower.model, + ), + "woe_scorecard_borrower_platt": atomic_write_pickle( + paths.model_dir / "prediction/woe_scorecard_borrower_platt.pkl", + borrower.scores.calibrator, + ), + } + freeze = { + "schema_version": str(config["schema_version"]), + "status": "credit_control_scores_frozen_before_primary_oot_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "source_inventory": data.source_inventory, + "active_reference_reproduction": reference_audit, + "learner_metrics": {learner.name: learner.metrics for learner in learners}, + "co_primary_learners": list(expected), + "model_selection": "none_all_five_reported", + "window_selection": "none_all_eight_reported", + "portfolio_optimization": False, + "sampling": "none_all_eligible_rows", + "primary_oot_outcome_columns_in_frozen_scores": [], + "implementation_provenance": _implementation(resolved_config, root), + "environment": environment_provenance(root), + "outcome_free_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_files.items() + }, + "model_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in model_files.items() + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze) + + +def _prediction_metrics(scores: pd.DataFrame, outcomes: pd.DataFrame) -> pd.DataFrame: + joined = scores.merge( + outcomes[["id", "snapshot_default"]], + on="id", + how="left", + validate="one_to_one", + ) + rows: list[dict[str, Any]] = [] + roles = ( + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ) + for column in [value for value in scores.columns if value.startswith("pd_")]: + learner = column.removeprefix("pd_") + for role in roles: + frame = joined.loc[joined["design_split"].eq(role)] + labels = pd.to_numeric(frame["snapshot_default"], errors="coerce") + observed = labels.notna() + if not bool(observed.any()): + raise RuntimeError(f"No resolved outcomes for {learner}/{role}.") + values = labels.loc[observed].astype(int).to_numpy() + probability = frame.loc[observed, column].to_numpy(dtype=float) + metrics = credit_prediction_metrics(values, probability) + rows.append( + { + "learner": learner, + "role": role, + "candidate_rows": int(len(frame)), + "resolved_rows": int(observed.sum()), + "unresolved_rows": int((~observed).sum()), + **metrics, + } + ) + return pd.DataFrame(rows) + + +def _evaluation_summary( + *, + config: dict[str, Any], + coverage: pd.DataFrame, + prediction: pd.DataFrame, +) -> dict[str, Any]: + canonical = coverage.loc[ + coverage["role"].eq("primary_oot") + & coverage["taxonomy_groups"].eq(5) + & coverage["conformal_group"].eq(-1) + ] + primary_prediction = prediction.loc[prediction["role"].eq("primary_oot")].set_index("learner") + learners: dict[str, Any] = {} + for learner in config["credit_risk_controls"]["co_primary_models"]: + cells = canonical.loc[canonical["learner"].eq(learner)] + if len(cells) != 8: + raise RuntimeError(f"Expected eight canonical OOT cells for {learner}.") + row = primary_prediction.loc[learner] + learners[str(learner)] = { + "primary_oot_roc_auc": float(row["roc_auc"]), + "primary_oot_brier": float(row["brier"]), + "primary_oot_log_loss": float(row["log_loss"]), + "primary_oot_gini": float(row["gini"]), + "primary_oot_ks": float(row["ks"]), + "primary_oot_average_precision": float(row["average_precision"]), + "primary_oot_ece_10": float(row["ece_10"]), + "primary_oot_calibration_intercept": float(row["calibration_intercept"]), + "primary_oot_calibration_slope": float(row["calibration_slope"]), + "primary_oot_default_rate": float(row["default_rate"]), + "canonical_coverage_lower_min": float(cells["coverage_lower"].min()), + "canonical_coverage_lower_max": float(cells["coverage_lower"].max()), + "canonical_coverage_upper_min": float(cells["coverage_upper"].min()), + "canonical_coverage_upper_max": float(cells["coverage_upper"].max()), + "windows_with_upper_below_target": int( + (cells["coverage_upper"] < 1.0 - float(config["conformal"]["alpha"])).sum() + ), + } + platform = learners["woe_scorecard_platform_platt"] + borrower = learners["woe_scorecard_borrower_platt"] + active = learners["catboost_platt"] + monotonic = learners["catboost_monotonic_platt"] + return { + "schema_version": str(config["schema_version"]), + "status": "complete_no_model_selection_credit_risk_control_evaluation", + "run_tag": str(config["run_tag"]), + "co_primary_learners": learners, + "declared_diagnostics": { + "platform_minus_borrower_oot_auc": float( + platform["primary_oot_roc_auc"] - borrower["primary_oot_roc_auc"] + ), + "platform_minus_borrower_oot_brier": float( + platform["primary_oot_brier"] - borrower["primary_oot_brier"] + ), + "monotonic_minus_active_oot_auc": float( + monotonic["primary_oot_roc_auc"] - active["primary_oot_roc_auc"] + ), + "monotonic_minus_active_oot_brier": float( + monotonic["primary_oot_brier"] - active["primary_oot_brier"] + ), + }, + "interpretation": { + "model_or_feature_selected_from_oot": False, + "portfolio_claim_authorized": False, + "scorecard_superiority_claim_authorized": False, + "universal_transport_claim_authorized": False, + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + + +def evaluate_credit_controls(*, config_path: Path, repo_root: Path) -> Path: + """Verify the V1 score freeze and evaluate all five controls once.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_credit_control_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + resume = config.get("resume_credit_control_freeze") + if not resume: + raise ValueError("Evaluation requires resume_credit_control_freeze.") + source_model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(resume["source_run_tag"]), + ) + source_freeze_path = source_model_dir / "protocol_freeze.json" + source_descriptor = relative_artifact_descriptor(source_freeze_path, repo_root=root) + if source_descriptor["sha256"] != str(resume["source_freeze_sha256"]): + raise RuntimeError("Imported credit-control freeze SHA-256 mismatch.") + freeze = json.loads(source_freeze_path.read_text(encoding="utf-8")) + expected = { + "status": "credit_control_scores_frozen_before_primary_oot_outcome_join", + "run_tag": str(resume["source_run_tag"]), + "protocol_tag": str(resume["source_protocol_tag"]), + "protocol_commit": str(resume["source_protocol_commit"]), + } + for field, value in expected.items(): + if freeze.get(field) != value: + raise RuntimeError(f"Imported credit-control freeze mismatch for {field}.") + if freeze.get("primary_oot_outcome_columns_in_frozen_scores") != []: + raise RuntimeError("Imported credit-control freeze reports OOT outcome leakage.") + artifacts = verified_freeze_artifact_paths(freeze, repo_root=root) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + raw_path = resolve_repo_input(config["source"]["raw_path"], repo_root=root) + universe = load_outcome_universe(config, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, config) + endpoint_audit = endpoint_resolution_audit(outcomes) + scores = pd.read_parquet(artifacts["scores"]) + recipes = load_recipes(artifacts["recipes"]) + fit_audit = pd.read_parquet(artifacts["fit_audit"]) + coverage = temporal_coverage_audit(scores, outcomes, recipes, fit_audit) + coverage_equivalence: dict[str, Any] | None = None + recovery = config.get("evaluation_recovery") + if recovery: + reference_spec = recovery["coverage_reference"] + reference_path = resolve_repo_input(reference_spec["path"], repo_root=root) + reference_descriptor = relative_artifact_descriptor(reference_path, repo_root=root) + for field in ("path", "bytes", "sha256"): + if reference_descriptor[field] != reference_spec[field]: + raise RuntimeError(f"Coverage recovery reference mismatch for {field}.") + reference_coverage = pd.read_parquet(reference_path) + pd.testing.assert_frame_equal( + coverage, + reference_coverage, + check_dtype=True, + check_exact=True, + check_like=False, + ) + coverage_equivalence = { + "status": "exact_frame_equivalence_verified", + "reference": reference_descriptor, + "rows": int(len(coverage)), + "columns": int(len(coverage.columns)), + } + prediction = _prediction_metrics(scores, outcomes) + recovery_audit = None + endpoint_recovery = config.get("endpoint_reason_recovery") + if endpoint_recovery: + recovery_audit = reconcile_from_json_reference( + { + "temporal_coverage": coverage, + "prediction_metrics": prediction, + }, + reference_json=endpoint_recovery["reference_json"], + artifact_section=str(endpoint_recovery["artifact_section"]), + repo_root=root, + float_atol=float(endpoint_recovery.get("float_atol", 0.0)), + float_rtol=float(endpoint_recovery.get("float_rtol", 0.0)), + ) + output_files = { + "temporal_coverage": atomic_write_parquet( + coverage, + paths.data_dir / "evaluation/temporal_coverage.parquet", + ), + "prediction_metrics": atomic_write_parquet( + prediction, + paths.data_dir / "evaluation/prediction_metrics.parquet", + ), + "endpoint_resolution_audit": atomic_write_parquet( + endpoint_audit, + paths.data_dir / "evaluation/endpoint_resolution_audit.parquet", + ), + } + summary = _evaluation_summary(config=config, coverage=coverage, prediction=prediction) + summary["source_freeze"] = source_descriptor + summary["source_protocol"] = expected + summary["coverage_recovery"] = coverage_equivalence + summary["endpoint_reason_recovery"] = recovery_audit + summary["endpoint_resolution_audit"] = endpoint_audit.to_dict(orient="records") + summary["evaluation_artifacts"] = { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in output_files.items() + } + summary["implementation_provenance"] = _implementation(resolved_config, root) + summary["environment"] = environment_provenance(root) + summary_path = atomic_write_json( + paths.model_dir / str(config["output"]["deterministic_summary"]), + summary, + ) + receipt = { + "schema_version": str(config["schema_version"]), + "status": "credit_risk_control_evaluation_complete", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "summary": relative_artifact_descriptor(summary_path, repo_root=root), + "source_freeze": source_descriptor, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + atomic_write_json(paths.model_dir / str(config["output"]["execution_receipt"]), receipt) + return summary_path diff --git a/src/ijds_audit/credit_controls.py b/src/ijds_audit/credit_controls.py new file mode 100644 index 0000000..a0f5f29 --- /dev/null +++ b/src/ijds_audit/credit_controls.py @@ -0,0 +1,482 @@ +"""Auditable credit-risk learner controls for the V4 temporal design.""" + +from __future__ import annotations + +import copy +import json +import warnings +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd +from catboost import CatBoostClassifier +from optbinning import BinningProcess +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import average_precision_score, roc_curve + +from src.data.outcome_observability import temporal_tail_split +from src.ijds_audit.prediction import ( + LearnerScores, + PreparedData, + available_binary_labels, + binary_probability_metrics, + fixed_taxonomy_edges, +) +from src.models.maturity_safe_pd import ( + apply_platt_calibrator, + catboost_raw_margin, + fit_platt_calibrator, +) + +STABILITY_ROLES = ( + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", +) + + +def credit_prediction_metrics(labels: np.ndarray, probabilities: np.ndarray) -> dict[str, Any]: + """Return fixed discrimination and calibration diagnostics for credit PD.""" + y = np.asarray(labels, dtype=int) + probability = np.clip(np.asarray(probabilities, dtype=float), 1e-8, 1.0 - 1e-8) + if len(y) != len(probability) or set(np.unique(y)) != {0, 1}: + raise ValueError("Credit prediction metrics require aligned binary classes.") + metrics: dict[str, Any] = dict(binary_probability_metrics(y, probability)) + false_positive_rate, true_positive_rate, _ = roc_curve(y, probability) + logits = np.log(probability / (1.0 - probability)).reshape(-1, 1) + maximum_iterations = 5000 + calibration = LogisticRegression( + C=np.inf, + solver="lbfgs", + tol=1e-10, + max_iter=maximum_iterations, + random_state=42, + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", ConvergenceWarning) + calibration.fit(logits, y) + coefficients = np.asarray([calibration.intercept_[0], calibration.coef_[0, 0]], dtype=float) + iterations = int(calibration.n_iter_[0]) + if iterations >= maximum_iterations or not bool(np.isfinite(coefficients).all()): + raise RuntimeError("Calibration intercept/slope optimization returned non-finite values.") + metrics.update( + { + "gini": float(2.0 * float(metrics["roc_auc"]) - 1.0), + "ks": float(np.max(true_positive_rate - false_positive_rate)), + "average_precision": float(average_precision_score(y, probability)), + "calibration_in_the_large": float(np.mean(probability) - np.mean(y)), + "calibration_intercept": float(coefficients[0]), + "calibration_slope": float(coefficients[1]), + "calibration_iterations": iterations, + "calibration_optimizer_success": True, + } + ) + return metrics + + +@dataclass +class WOELogisticModel: + """Pickle-safe OptBinning transformer and regularized logistic model.""" + + name: str + features: tuple[str, ...] + categorical_features: tuple[str, ...] + binning_process: BinningProcess + logistic_regression: LogisticRegression + + def transform(self, frame: pd.DataFrame, *, metric: str = "woe") -> pd.DataFrame: + """Apply the frozen binning process with neutral missing and special values.""" + transformed = self.binning_process.transform( + frame.loc[:, self.features], + metric=metric, + metric_missing=0, + metric_special=0, + check_input=False, + ) + if isinstance(transformed, pd.DataFrame): + return transformed + return pd.DataFrame(transformed, index=frame.index, columns=pd.Index(self.features)) + + def decision_function(self, frame: pd.DataFrame) -> np.ndarray: + """Return the uncalibrated scorecard log-odds margin.""" + return np.asarray( + self.logistic_regression.decision_function(self.transform(frame)), + dtype=float, + ) + + def predict_proba(self, frame: pd.DataFrame) -> np.ndarray: + """Return the uncalibrated scorecard class probabilities.""" + return np.asarray( + self.logistic_regression.predict_proba(self.transform(frame)), dtype=float + ) + + +@dataclass(frozen=True) +class ScorecardFit: + """One scorecard learner and its auditable WOE/IV artifacts.""" + + scores: LearnerScores + model: WOELogisticModel + summary: pd.DataFrame + coefficients: pd.DataFrame + binning_table: pd.DataFrame + + +def _calibrated_scores( + *, + data: PreparedData, + config: Mapping[str, Any], + name: str, + model: Any, + validation: pd.DataFrame, + validation_probability: np.ndarray, + margin_predictor: Any, + random_state: int, +) -> LearnerScores: + universe = data.universe + calibration_all = universe.loc[universe["design_split"].eq("probability_calibration")] + calibration_fit = calibration_all.loc[calibration_all["label_available"]] + calibration_margin = np.asarray( + margin_predictor(data.features.loc[calibration_fit.index]), dtype=float + ) + calibrator_config = copy.deepcopy(config["probability_calibration"]) + calibrator_config["logistic_regression"]["random_state"] = int(random_state) + calibrator = fit_platt_calibrator( + calibration_margin, + available_binary_labels(calibration_fit, block=f"{name}_probability_calibration"), + calibrator_config, + ) + all_margin = np.asarray(margin_predictor(data.features), dtype=float) + all_probability = apply_platt_calibrator(calibrator, all_margin) + groups = [int(value) for value in config["conformal"]["diagnostic_group_counts"]] + return LearnerScores( + name=name, + model=model, + calibrator=calibrator, + probabilities=all_probability, + taxonomy_edges=fixed_taxonomy_edges( + all_probability[calibration_all.index], + groups, + ), + metrics={ + "validation_uncalibrated": binary_probability_metrics( + available_binary_labels(validation, block=f"{name}_validation"), + validation_probability, + ), + "probability_calibration": binary_probability_metrics( + available_binary_labels( + calibration_fit, + block=f"{name}_probability_calibration", + ), + all_probability[calibration_fit.index], + ), + }, + ) + + +def fit_monotonic_catboost_control(data: PreparedData, config: Mapping[str, Any]) -> LearnerScores: + """Fit CatBoost with only the predeclared domain-safe monotonic constraints.""" + universe = data.universe + development = universe.loc[ + universe["design_split"].eq("pd_development") & universe["label_available"] + ] + train, validation, validation_cutoff = temporal_tail_split( + development, + tail_fraction=float(config["design"]["validation_tail_fraction"]), + ) + control = config["credit_risk_controls"]["monotonic_catboost"] + constraints = {str(key): int(value) for key, value in control["constraints"].items()} + parameters = dict(config["model"]["fixed_params"]) + parameters.update( + random_seed=int(config["model"]["canonical_seed"]), + thread_count=int(config["execution"]["threads"]), + monotone_constraints=constraints, + ) + model = CatBoostClassifier(**parameters) + model.fit( + data.features.loc[train.index], + available_binary_labels(train, block="monotonic_catboost_train"), + cat_features=list(data.categorical_features), + ) + validation_probability = np.asarray( + model.predict_proba(data.features.loc[validation.index])[:, 1], dtype=float + ) + scores = _calibrated_scores( + data=data, + config=config, + name="catboost_monotonic_platt", + model=model, + validation=validation, + validation_probability=validation_probability, + margin_predictor=lambda frame: catboost_raw_margin(model, frame), + random_state=int(config["model"]["canonical_seed"]), + ) + scores.metrics["validation_cutoff"] = str(validation_cutoff.to_period("M")) + scores.metrics["monotonic_constraints"] = constraints + return scores + + +def _scorecard_binning_process( + *, + features: Sequence[str], + categorical_features: Sequence[str], + config: Mapping[str, Any], +) -> BinningProcess: + options = config["credit_risk_controls"]["optbinning"] + return BinningProcess( + variable_names=list(features), + categorical_variables=list(categorical_features), + max_n_prebins=int(options["max_n_prebins"]), + min_prebin_size=float(options["min_prebin_size"]), + min_n_bins=int(options["min_n_bins"]), + max_n_bins=int(options["max_n_bins"]), + min_bin_size=float(options["min_bin_size"]), + n_jobs=int(options["n_jobs"]), + verbose=False, + ) + + +def scorecard_binning_table(model: WOELogisticModel) -> pd.DataFrame: + """Return a Parquet-safe long table for every frozen WOE bin.""" + tables: list[pd.DataFrame] = [] + numeric_columns = ( + "Count", + "Count (%)", + "Non-event", + "Event", + "Event rate", + "WoE", + "IV", + "JS", + ) + for feature in model.features: + table = model.binning_process.get_binned_variable(feature).binning_table.build().copy() + table.insert(0, "feature", feature) + table.insert(0, "learner", model.name) + table["Bin"] = table["Bin"].map(str) + for column in numeric_columns: + if column in table: + table[column] = pd.to_numeric(table[column], errors="coerce") + tables.append(table) + return pd.concat(tables, ignore_index=True) + + +def fit_woe_scorecard_control( + data: PreparedData, + config: Mapping[str, Any], + *, + specification: str, +) -> ScorecardFit: + """Fit one closed OptBinning WOE/IV scorecard on the temporal train block.""" + controls = config["credit_risk_controls"] + scorecard = controls["scorecards"][specification] + name = str(scorecard["name"]) + features = tuple(str(value) for value in scorecard["features"]) + missing = sorted(set(features).difference(data.features.columns)) + if missing: + raise KeyError(f"Scorecard {name} is missing active features: {missing}.") + categorical = tuple(value for value in features if value in data.categorical_features) + + universe = data.universe + development = universe.loc[ + universe["design_split"].eq("pd_development") & universe["label_available"] + ] + train, validation, validation_cutoff = temporal_tail_split( + development, + tail_fraction=float(config["design"]["validation_tail_fraction"]), + ) + labels = available_binary_labels(train, block=f"{name}_train") + process = _scorecard_binning_process( + features=features, + categorical_features=categorical, + config=config, + ) + process.fit(data.features.loc[train.index, features], labels) + train_woe = process.transform( + data.features.loc[train.index, features], + metric="woe", + metric_missing=0, + metric_special=0, + check_input=False, + ) + logistic = controls["scorecard_logistic"] + estimator = LogisticRegression( + C=float(logistic["C"]), + class_weight=str(logistic["class_weight"]), + solver=str(logistic["solver"]), + max_iter=int(logistic["max_iter"]), + random_state=int(logistic["random_state"]), + ) + estimator.fit(train_woe, labels) + model = WOELogisticModel( + name=name, + features=features, + categorical_features=categorical, + binning_process=process, + logistic_regression=estimator, + ) + validation_probability = model.predict_proba(data.features.loc[validation.index])[:, 1] + scores = _calibrated_scores( + data=data, + config=config, + name=name, + model=model, + validation=validation, + validation_probability=validation_probability, + margin_predictor=model.decision_function, + random_state=int(logistic["random_state"]), + ) + scores.metrics["validation_cutoff"] = str(validation_cutoff.to_period("M")) + scores.metrics["scorecard_features"] = list(features) + summary = process.summary().copy() + summary.insert(0, "learner", name) + selected = [str(value) for value in process.get_support(names=True)] + coefficients = pd.DataFrame( + { + "learner": name, + "feature": selected, + "logistic_coefficient": estimator.coef_.reshape(-1), + } + ) + coefficients["logistic_intercept"] = float(estimator.intercept_[0]) + return ScorecardFit( + scores=scores, + model=model, + summary=summary, + coefficients=coefficients, + binning_table=scorecard_binning_table(model), + ) + + +def feature_variation_audit(data: PreparedData) -> pd.DataFrame: + """Describe missingness and support for every active feature by temporal role.""" + rows: list[dict[str, Any]] = [] + roles = ( + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ) + categorical = set(data.categorical_features) + for role in roles: + mask = data.universe["design_split"].eq(role) + frame = data.features.loc[mask] + for feature in data.features.columns: + series = frame[feature] + if feature in categorical: + missing = series.astype(str).eq("__MISSING__") + observed = series.loc[~missing] + standard_deviation = np.nan + minimum = None + maximum = None + else: + numeric = pd.to_numeric(series, errors="coerce") + missing = numeric.isna() + observed = numeric.loc[~missing] + standard_deviation = float(observed.std(ddof=0)) if len(observed) else np.nan + minimum = float(observed.min()) if len(observed) else None + maximum = float(observed.max()) if len(observed) else None + rows.append( + { + "role": role, + "feature": feature, + "feature_type": "categorical" if feature in categorical else "numeric", + "rows": int(len(series)), + "missing_rows": int(missing.sum()), + "missing_share": float(missing.mean()), + "unique_observed": int(observed.nunique(dropna=True)), + "constant_observed": bool(observed.nunique(dropna=True) <= 1), + "minimum": minimum, + "maximum": maximum, + "standard_deviation": standard_deviation, + } + ) + return pd.DataFrame(rows) + + +def _population_stability_index( + reference: pd.Series, comparison: pd.Series, *, epsilon: float = 1e-6 +) -> float: + categories = sorted(set(reference.astype(str)).union(comparison.astype(str))) + reference_share = ( + reference.astype(str).value_counts(normalize=True).reindex(categories, fill_value=0) + ) + comparison_share = ( + comparison.astype(str).value_counts(normalize=True).reindex(categories, fill_value=0) + ) + reference_safe = np.maximum(reference_share.to_numpy(dtype=float), epsilon) + comparison_safe = np.maximum(comparison_share.to_numpy(dtype=float), epsilon) + return float( + np.sum((comparison_safe - reference_safe) * np.log(comparison_safe / reference_safe)) + ) + + +def score_psi_audit( + data: PreparedData, + learners: Sequence[LearnerScores], + *, + bins: int, +) -> pd.DataFrame: + """Compute outcome-free score PSI against the PD-development population.""" + reference_mask = data.universe["design_split"].eq("pd_development").to_numpy(dtype=bool) + rows: list[dict[str, Any]] = [] + for learner in learners: + reference_scores = learner.probabilities[reference_mask] + edges = np.unique( + np.quantile(reference_scores, np.linspace(0.0, 1.0, bins + 1), method="linear") + ) + if len(edges) < 3: + raise RuntimeError(f"{learner.name} has insufficient score variation for PSI.") + edges[0] = -np.inf + edges[-1] = np.inf + reference_bins = pd.Series(pd.cut(reference_scores, bins=edges, include_lowest=True)) + for role in STABILITY_ROLES: + role_mask = data.universe["design_split"].eq(role).to_numpy(dtype=bool) + comparison_bins = pd.Series( + pd.cut(learner.probabilities[role_mask], bins=edges, include_lowest=True) + ) + rows.append( + { + "learner": learner.name, + "reference_role": "pd_development", + "comparison_role": role, + "reference_rows": int(reference_mask.sum()), + "comparison_rows": int(role_mask.sum()), + "psi": _population_stability_index(reference_bins, comparison_bins), + "bin_edges": json.dumps([float(value) for value in edges]), + } + ) + return pd.DataFrame(rows) + + +def scorecard_feature_psi_audit(data: PreparedData, model: WOELogisticModel) -> pd.DataFrame: + """Compute outcome-free PSI on each scorecard's frozen optimal-bin indices.""" + indices = model.transform(data.features, metric="indices") + reference_mask = data.universe["design_split"].eq("pd_development").to_numpy(dtype=bool) + rows: list[dict[str, Any]] = [] + for feature in model.features: + reference = indices.loc[reference_mask, feature] + for role in STABILITY_ROLES: + role_mask = data.universe["design_split"].eq(role).to_numpy(dtype=bool) + comparison = indices.loc[role_mask, feature] + rows.append( + { + "learner": model.name, + "feature": feature, + "reference_role": "pd_development", + "comparison_role": role, + "reference_rows": int(reference_mask.sum()), + "comparison_rows": int(role_mask.sum()), + "reference_bins": int(reference.nunique(dropna=False)), + "comparison_bins": int(comparison.nunique(dropna=False)), + "psi": _population_stability_index(reference, comparison), + } + ) + return pd.DataFrame(rows) diff --git a/src/ijds_audit/endpoint_recovery.py b/src/ijds_audit/endpoint_recovery.py new file mode 100644 index 0000000..38b4bd1 --- /dev/null +++ b/src/ijds_audit/endpoint_recovery.py @@ -0,0 +1,149 @@ +"""Scientific-output reconciliation for reason-only endpoint recoveries.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from pandas.api.types import is_float_dtype + +from src.utils.isolated_experiment import relative_artifact_descriptor + + +def verified_json_artifact( + descriptor: Mapping[str, Any], + *, + repo_root: Path, +) -> dict[str, Any]: + """Load a JSON artifact only after its path, size, and hash reconcile.""" + path = (repo_root / str(descriptor["path"])).resolve() + path.relative_to(repo_root) + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Endpoint recovery reference mismatch for {field}: {path}.") + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError(f"Endpoint recovery reference must be a JSON object: {path}.") + return payload + + +def reconcile_reference_frames( + current: Mapping[str, pd.DataFrame], + reference_descriptors: Mapping[str, Mapping[str, Any]], + *, + repo_root: Path, + float_atol: float = 0.0, + float_rtol: float = 0.0, +) -> dict[str, Any]: + """Reconcile every reference column in named Parquet frames. + + A corrected lineage may append diagnostic columns, such as identification + widths. Non-floating columns remain exact. Floating columns may use a + protocol-declared machine-precision tolerance; the observed maximum drift + is persisted for audit. + """ + for name, value in (("float_atol", float_atol), ("float_rtol", float_rtol)): + if not np.isfinite(value) or value < 0.0: + raise ValueError(f"{name} must be finite and non-negative.") + if set(current) != set(reference_descriptors): + raise RuntimeError( + "Endpoint recovery frame inventory changed: " + f"current={sorted(current)}, reference={sorted(reference_descriptors)}." + ) + audits: dict[str, Any] = {} + for name, frame in current.items(): + descriptor = reference_descriptors[name] + path = (repo_root / str(descriptor["path"])).resolve() + path.relative_to(repo_root) + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Endpoint recovery artifact mismatch for {name}/{field}.") + reference = pd.read_parquet(path) + missing_columns = sorted(set(reference.columns).difference(frame.columns)) + if missing_columns: + raise RuntimeError( + f"Endpoint recovery dropped reference columns from {name}: {missing_columns}." + ) + reference_columns = frame.loc[:, reference.columns] + exact = float_atol == 0.0 and float_rtol == 0.0 + pd.testing.assert_frame_equal( + reference_columns, + reference, + check_dtype=True, + check_exact=exact, + check_like=False, + atol=float_atol, + rtol=float_rtol, + ) + float_drift: dict[str, dict[str, float]] = {} + for column in reference.columns: + if not is_float_dtype(reference[column].dtype): + continue + current_values = reference_columns[column].to_numpy(dtype=float) + reference_values = reference[column].to_numpy(dtype=float) + finite = np.isfinite(current_values) & np.isfinite(reference_values) + absolute = np.abs(current_values[finite] - reference_values[finite]) + scale = np.maximum(np.abs(reference_values[finite]), np.finfo(float).tiny) + relative = absolute / scale + float_drift[str(column)] = { + "maximum_absolute": float(absolute.max(initial=0.0)), + "maximum_relative": float(relative.max(initial=0.0)), + } + audits[name] = { + "rows": int(len(reference)), + "reference_columns": int(len(reference.columns)), + "appended_columns": sorted(set(frame.columns).difference(reference.columns)), + "float_drift": float_drift, + "reference": actual, + } + return { + "status": ( + "exact_reference_column_equivalence_verified" + if float_atol == 0.0 and float_rtol == 0.0 + else "reference_column_equivalence_verified_with_float_tolerance" + ), + "equivalence": { + "non_float_columns_exact": True, + "float_atol": float(float_atol), + "float_rtol": float(float_rtol), + }, + "frames": audits, + } + + +def reconcile_from_json_reference( + current: Mapping[str, pd.DataFrame], + *, + reference_json: Mapping[str, Any], + artifact_section: str, + repo_root: Path, + float_atol: float = 0.0, + float_rtol: float = 0.0, +) -> dict[str, Any]: + """Verify one reference JSON and reconcile selected named frame artifacts.""" + payload = verified_json_artifact(reference_json, repo_root=repo_root) + descriptors = payload.get(artifact_section) + if not isinstance(descriptors, Mapping): + raise KeyError(f"Reference JSON is missing artifact section {artifact_section!r}.") + selected: dict[str, Mapping[str, Any]] = {} + for name in current: + descriptor = descriptors.get(name) + if not isinstance(descriptor, Mapping): + raise KeyError(f"Reference artifact section is missing {name!r}.") + selected[name] = descriptor + result = reconcile_reference_frames( + current, + selected, + repo_root=repo_root, + float_atol=float_atol, + float_rtol=float_rtol, + ) + result["reference_json"] = dict(reference_json) + result["artifact_section"] = str(artifact_section) + return result diff --git a/src/ijds_audit/endpoint_sensitivity.py b/src/ijds_audit/endpoint_sensitivity.py new file mode 100644 index 0000000..9ecfd4e --- /dev/null +++ b/src/ijds_audit/endpoint_sensitivity.py @@ -0,0 +1,114 @@ +"""Endpoint-availability sensitivity for frozen IJDS scores and allocations.""" + +from __future__ import annotations + +from typing import Any + +import pandas as pd + +from src.data.outcome_observability import build_outcome_label_availability +from src.ijds_audit.evaluation import build_archive_outcomes + + +def rebuild_archive_outcomes( + universe: pd.DataFrame, + *, + evaluation_cutoff: str | pd.Timestamp, + charged_off_lag_months: int, +) -> pd.DataFrame: + """Reconstruct one endpoint without changing candidate membership.""" + lag = int(charged_off_lag_months) + if lag < 0: + raise ValueError("Charged-off reporting lag must be nonnegative.") + required = {"id", "loan_status", "last_pymnt_d", "design_split", "issue_d"} + missing = sorted(required.difference(universe.columns)) + if missing: + raise KeyError(f"Endpoint sensitivity universe is missing columns: {missing}.") + labels = build_outcome_label_availability( + universe["loan_status"], + universe["last_pymnt_d"], + cutoff=evaluation_cutoff, + charged_off_lag_months=lag, + ) + lagged = universe.copy() + lagged["terminal_default"] = labels["terminal_outcome"].astype("Int8") + lagged["label_available_at"] = labels["label_available_at"] + outcomes = build_archive_outcomes(lagged, evaluation_cutoff=evaluation_cutoff) + if len(outcomes) != len(universe) or not outcomes["id"].equals(universe["id"].astype("string")): + raise RuntimeError("Endpoint reconstruction changed universe membership or order.") + return outcomes + + +def endpoint_census(outcomes: pd.DataFrame, *, lag_months: int) -> pd.DataFrame: + """Count resolved and unresolved candidates by declared design role.""" + rows: list[dict[str, Any]] = [] + for role, frame in outcomes.groupby("role", observed=True, sort=True): + resolved = int(frame["snapshot_default"].notna().sum()) + rows.append( + { + "charged_off_lag_months": int(lag_months), + "role": str(role), + "candidate_rows": int(len(frame)), + "resolved_rows": resolved, + "unresolved_rows": int(len(frame) - resolved), + "default_rows": int(frame["snapshot_default"].eq(1).sum()), + "nondefault_rows": int(frame["snapshot_default"].eq(0).sum()), + } + ) + return pd.DataFrame(rows) + + +def summarize_coverage_sensitivity(coverage: pd.DataFrame) -> pd.DataFrame: + """Summarize complete-window canonical coverage without selecting a learner.""" + required = { + "charged_off_lag_months", + "learner", + "role", + "window_id", + "coverage_lower", + "coverage_upper", + } + missing = sorted(required.difference(coverage.columns)) + if missing: + raise KeyError(f"Coverage sensitivity is missing columns: {missing}.") + return ( + coverage.groupby( + ["charged_off_lag_months", "learner", "role"], + observed=True, + sort=True, + ) + .agg( + windows=("window_id", "nunique"), + coverage_lower_min=("coverage_lower", "min"), + coverage_upper_max=("coverage_upper", "max"), + ) + .reset_index() + ) + + +def direction_census(directions: pd.DataFrame, *, lag_months: int) -> pd.DataFrame: + """Count every two-ruler direction, retaining crossing and exact-zero cells.""" + rows: list[dict[str, Any]] = [] + for metric, frame in directions.groupby("metric", observed=True, sort=True): + counts = frame["direction"].value_counts().to_dict() + for direction in ("gamma_1_lower", "gamma_1_higher", "crosses_zero", "exact_zero"): + rows.append( + { + "charged_off_lag_months": int(lag_months), + "metric": str(metric), + "direction": direction, + "cells": int(counts.get(direction, 0)), + } + ) + return pd.DataFrame(rows) + + +def exact_support_census(envelopes: pd.DataFrame, *, lag_months: int) -> pd.DataFrame: + """Count exact-support directions by scope and metric.""" + census = ( + envelopes.groupby(["scope", "metric", "direction"], observed=True, sort=True) + .size() + .reset_index() + ) + census.columns = ["scope", "metric", "direction", "cells"] + return census.assign(charged_off_lag_months=int(lag_months)) diff --git a/src/ijds_audit/evaluation.py b/src/ijds_audit/evaluation.py new file mode 100644 index 0000000..227c2b9 --- /dev/null +++ b/src/ijds_audit/evaluation.py @@ -0,0 +1,695 @@ +"""Post-freeze archive evaluation, coverage transport, and comparator envelopes.""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping, Sequence +from typing import Any, cast + +import numpy as np +import pandas as pd + +from src.evaluation.coverage_transport import binary_miscoverage_bounds +from src.evaluation.maturity_safe_portfolio import ( + aggregate_monthly_evaluation, + evaluate_prejoined_frozen_allocation, +) +from src.evaluation.policy_contrast_bounds import PolicyContrastIndex +from src.ijds_audit.geometry import summarize_binary_geometry +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, +) + +EVALUATION_ROLES = ("conformal_fit", "policy_development", "primary_oot", "censored_extension") + +_EXPECTED_ROLE = "__expected_role" +_EXPECTED_PERIOD = "__expected_period" +_OUTCOME_ROLE = "__outcome_role" +_OUTCOME_PERIOD = "__outcome_period" +_OUTCOME_JOIN = "__outcome_join" + +RESOLUTION_FULLY_PAID_BY_CUTOFF = "fully_paid_by_reconstructed_cutoff" +RESOLUTION_CHARGED_OFF_BY_CUTOFF = "charged_off_by_reconstructed_cutoff" +RESOLUTION_TERMINAL_DATE_MISSING = "terminal_availability_date_missing" +RESOLUTION_TERMINAL_AFTER_CUTOFF = "terminal_after_reconstructed_cutoff" +RESOLUTION_NONTERMINAL = "nonterminal_or_unresolved_status" + + +def _outcome_payload( + outcomes: pd.DataFrame, + *, + value_columns: Sequence[str], +) -> pd.DataFrame: + required = {"id", *value_columns} + missing = sorted(required - set(outcomes.columns)) + if missing: + raise KeyError(f"Outcome census is missing required columns: {missing}.") + if bool(outcomes["id"].isna().any()): + raise RuntimeError("The outcome census contains missing IDs.") + duplicate_ids = outcomes.loc[outcomes["id"].duplicated(keep=False), "id"] + if not duplicate_ids.empty: + sample = duplicate_ids.astype(str).drop_duplicates().head(5).tolist() + raise RuntimeError(f"The outcome census contains duplicate IDs: {sample}.") + + columns = ["id", *value_columns] + rename: dict[str, str] = {} + if "role" in outcomes.columns: + columns.append("role") + rename["role"] = _OUTCOME_ROLE + if "period" in outcomes.columns: + columns.append("period") + rename["period"] = _OUTCOME_PERIOD + return outcomes.loc[:, columns].rename(columns=rename) + + +def _with_expected_outcome_metadata(frame: pd.DataFrame) -> pd.DataFrame: + result = frame.copy() + if "role" in result.columns: + result[_EXPECTED_ROLE] = result["role"].astype("string") + elif "design_split" in result.columns: + result[_EXPECTED_ROLE] = result["design_split"].astype("string") + + if "period" in result.columns: + result[_EXPECTED_PERIOD] = result["period"].astype("string") + elif "issue_d" in result.columns: + issue_date = pd.to_datetime(result["issue_d"], errors="coerce") + if bool(issue_date.isna().any()): + raise RuntimeError("The expected outcome census contains an invalid issue date.") + result[_EXPECTED_PERIOD] = issue_date.dt.to_period("M").astype("string") + return result + + +def _validate_outcome_metadata(joined: pd.DataFrame) -> None: + comparisons = ( + ("role", _EXPECTED_ROLE, _OUTCOME_ROLE), + ("period", _EXPECTED_PERIOD, _OUTCOME_PERIOD), + ) + for label, expected_column, outcome_column in comparisons: + if expected_column not in joined.columns or outcome_column not in joined.columns: + continue + expected = joined[expected_column].astype("string") + observed = joined[outcome_column].astype("string") + mismatch = expected.isna() | observed.isna() | expected.ne(observed).fillna(True) + if bool(mismatch.any()): + sample = joined.loc[mismatch, "id"].astype(str).head(5).tolist() + raise RuntimeError(f"Outcome {label} disagrees with the frozen census: {sample}.") + + +def _drop_outcome_join_helpers(frame: pd.DataFrame) -> pd.DataFrame: + return frame.drop( + columns=[ + _EXPECTED_ROLE, + _EXPECTED_PERIOD, + _OUTCOME_ROLE, + _OUTCOME_PERIOD, + _OUTCOME_JOIN, + ], + errors="ignore", + ) + + +def _scoped_outcome_payload( + expected: pd.DataFrame, + payload: pd.DataFrame, +) -> pd.DataFrame: + expected_ids = expected["id"] + metadata_scope = np.zeros(len(payload), dtype=bool) + metadata_pairs = ( + (_EXPECTED_ROLE, _OUTCOME_ROLE), + (_EXPECTED_PERIOD, _OUTCOME_PERIOD), + ) + available_pairs = [ + (expected_column, outcome_column) + for expected_column, outcome_column in metadata_pairs + if expected_column in expected.columns and outcome_column in payload.columns + ] + if len(available_pairs) == 2: + expected_keys = pd.MultiIndex.from_frame( + expected[[column for column, _ in available_pairs]].drop_duplicates() + ) + outcome_keys = pd.MultiIndex.from_frame(payload[[column for _, column in available_pairs]]) + metadata_scope = outcome_keys.isin(expected_keys) + elif len(available_pairs) == 1: + expected_column, outcome_column = available_pairs[0] + metadata_scope = payload[outcome_column].isin(expected[expected_column]) + return payload.loc[payload["id"].isin(expected_ids) | metadata_scope].copy() + + +def _exact_outcome_join( + expected: pd.DataFrame, + outcomes: pd.DataFrame, + *, + value_columns: Sequence[str], +) -> pd.DataFrame: + census = _with_expected_outcome_metadata(expected) + if bool(census["id"].isna().any()): + raise RuntimeError("The frozen census contains missing IDs.") + if bool(census["id"].duplicated().any()): + raise RuntimeError("The frozen census contains duplicate IDs.") + payload = _scoped_outcome_payload( + census, + _outcome_payload(outcomes, value_columns=value_columns), + ) + joined = census.merge( + payload, + on="id", + how="outer", + validate="one_to_one", + indicator=_OUTCOME_JOIN, + sort=False, + ) + missing = int(joined[_OUTCOME_JOIN].eq("left_only").sum()) + extra = int(joined[_OUTCOME_JOIN].eq("right_only").sum()) + if missing or extra: + raise RuntimeError(f"Outcome ID census mismatch: missing={missing}, extra={extra}.") + _validate_outcome_metadata(joined) + return _drop_outcome_join_helpers(joined) + + +def _validate_portfolio_candidate_census( + records: pd.DataFrame, + outcomes: pd.DataFrame, +) -> None: + required = {"role", "period", "n_candidates"} + missing = sorted(required - set(records.columns)) + if missing: + raise KeyError(f"Solve records are missing candidate census columns: {missing}.") + if not {"role", "period"}.issubset(outcomes.columns): + raise KeyError("Portfolio outcomes must contain role and period columns.") + + expected = records.loc[:, ["role", "period", "n_candidates"]].copy() + expected["role"] = expected["role"].astype("string") + expected["period"] = expected["period"].astype("string") + candidate_counts = pd.to_numeric(expected["n_candidates"], errors="raise") + candidate_values = candidate_counts.to_numpy(dtype=float) + if ( + not bool(np.isfinite(candidate_values).all()) + or bool((candidate_values < 0).any()) + or not bool(np.equal(candidate_values, np.floor(candidate_values)).all()) + ): + raise RuntimeError("Solve records contain an invalid candidate census.") + expected["expected_candidates"] = candidate_counts.astype("int64") + conflicts = expected.groupby(["role", "period"], observed=True)["expected_candidates"].nunique( + dropna=False + ) + if bool(conflicts.gt(1).any()): + raise RuntimeError("Solve records disagree on the role/period candidate census.") + expected = expected[["role", "period", "expected_candidates"]].drop_duplicates() + + evaluated_roles = expected["role"].drop_duplicates() + actual = outcomes.loc[ + outcomes["role"].astype("string").isin(evaluated_roles), ["role", "period", "id"] + ].copy() + actual["role"] = actual["role"].astype("string") + actual["period"] = actual["period"].astype("string") + actual = ( + actual.groupby(["role", "period"], observed=True, sort=False) + .size() + .rename("actual_candidates") + .reset_index() + ) + comparison = expected.merge( + actual, + on=["role", "period"], + how="outer", + validate="one_to_one", + indicator="__candidate_census_join", + ) + expected_count = comparison["expected_candidates"].fillna(0).to_numpy(dtype=np.int64) + actual_count = comparison["actual_candidates"].fillna(0).to_numpy(dtype=np.int64) + missing_count = int(np.maximum(expected_count - actual_count, 0).sum()) + extra_count = int(np.maximum(actual_count - expected_count, 0).sum()) + if missing_count or extra_count: + raise RuntimeError( + f"Portfolio outcome ID census mismatch: missing={missing_count}, extra={extra_count}." + ) + + +def build_archive_outcomes( + universe: pd.DataFrame, + *, + evaluation_cutoff: str | pd.Timestamp | None = None, +) -> pd.DataFrame: + """Materialize the nullable endpoint panel kept outside policy construction. + + When ``evaluation_cutoff`` is supplied, a terminal status is retained only + when its conservative ``label_available_at`` date is no later than that + cutoff. This reconstructs an as-of endpoint from the distributed archive; + it does not pretend that the archive itself is a point-in-time snapshot. + """ + archive_terminal = universe["terminal_default"].astype("Int8") + available_at = pd.to_datetime( + universe.get("label_available_at", pd.Series(pd.NaT, index=universe.index)), + errors="coerce", + ) + terminal = archive_terminal.copy() + observed_by_cutoff = archive_terminal.notna() + resolution = pd.Series("right_censored", index=universe.index, dtype="string") + if evaluation_cutoff is not None: + cutoff = pd.Timestamp(evaluation_cutoff) + if pd.isna(cutoff): + raise ValueError("Evaluation cutoff must be a valid timestamp.") + observed_by_cutoff &= available_at.notna() & available_at.le(cutoff) + terminal = archive_terminal.where(observed_by_cutoff).astype("Int8") + terminal_missing_date = archive_terminal.notna() & available_at.isna() + terminal_after_cutoff = ( + archive_terminal.notna() & available_at.notna() & available_at.gt(cutoff) + ) + resolution[:] = RESOLUTION_NONTERMINAL + resolution.loc[terminal.eq(0).fillna(False)] = RESOLUTION_FULLY_PAID_BY_CUTOFF + resolution.loc[terminal.eq(1).fillna(False)] = RESOLUTION_CHARGED_OFF_BY_CUTOFF + resolution.loc[terminal_missing_date] = RESOLUTION_TERMINAL_DATE_MISSING + resolution.loc[terminal_after_cutoff] = RESOLUTION_TERMINAL_AFTER_CUTOFF + else: + resolution.loc[terminal.eq(0).fillna(False)] = "fully_paid" + resolution.loc[terminal.eq(1).fillna(False)] = "charged_off" + return pd.DataFrame( + { + "id": universe["id"].astype("string"), + "snapshot_default": terminal, + "snapshot_resolution": resolution, + "outcome_available_at": available_at, + "role": universe["design_split"].astype("string"), + "period": pd.to_datetime(universe["issue_d"]).dt.to_period("M").astype(str), + } + ) + + +def endpoint_resolution_audit( + outcomes: pd.DataFrame, + *, + roles: Collection[str] | None = None, +) -> pd.DataFrame: + """Count resolved and unresolved candidates by explicit endpoint reason.""" + required = {"id", "role", "snapshot_default", "snapshot_resolution"} + missing = sorted(required.difference(outcomes.columns)) + if missing: + raise KeyError(f"Endpoint audit is missing required columns: {missing}.") + selected = outcomes.copy() + if roles is not None: + selected = selected.loc[selected["role"].astype(str).isin(map(str, roles))].copy() + if selected.empty: + raise ValueError("Endpoint audit received no candidate rows.") + selected["resolved"] = selected["snapshot_default"].notna() + audit = ( + selected.groupby(["role", "snapshot_resolution"], observed=True, sort=True) + .agg(candidate_rows=("id", "size"), resolved_rows=("resolved", "sum")) + .reset_index() + ) + audit["resolved_rows"] = audit["resolved_rows"].astype("int64") + audit["unresolved_rows"] = audit["candidate_rows"] - audit["resolved_rows"] + if int(audit["candidate_rows"].sum()) != len(selected): + raise RuntimeError("Endpoint reason taxonomy does not partition the candidate census.") + return audit + + +def evaluate_frozen_portfolios( + records: pd.DataFrame, + allocations: pd.DataFrame, + outcomes: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Perform one validated outcome join and evaluate every frozen allocation.""" + keys = ["window_id", "role", "period", "policy_label", "comparator_rule"] + record_index = records.set_index(keys) + if not record_index.index.is_unique: + raise ValueError("Solve records contain duplicate evaluation keys.") + outcome_payload = _outcome_payload( + outcomes, + value_columns=("snapshot_default", "snapshot_resolution"), + ) + join_source = _with_expected_outcome_metadata(allocations) + joined = join_source.merge( + outcome_payload, + on="id", + how="left", + validate="many_to_one", + indicator=_OUTCOME_JOIN, + sort=False, + ) + missing_funded = int(joined[_OUTCOME_JOIN].eq("left_only").sum()) + if missing_funded: + raise RuntimeError(f"Funded outcome ID census mismatch: missing={missing_funded}.") + _validate_outcome_metadata(joined) + joined = _drop_outcome_join_helpers(joined) + _validate_portfolio_candidate_census(records, outcomes) + candidate_unresolved = outcomes.groupby(["role", "period"], observed=True)[ + "snapshot_default" + ].apply(lambda values: int(values.isna().sum())) + if len(joined) != len(allocations) or bool(joined["snapshot_resolution"].isna().any()): + raise RuntimeError("The shared outcome join did not preserve every funded row.") + evaluated: list[dict[str, Any]] = [] + joined_groups: list[pd.DataFrame] = [] + grouped = joined.groupby(keys, observed=True, sort=True, dropna=False) + if grouped.ngroups != len(record_index): + raise RuntimeError("Solve records and allocation groups have different cardinality.") + for raw_keys, allocation in grouped: + key_values = raw_keys if isinstance(raw_keys, tuple) else (raw_keys,) + base = record_index.loc[key_values] + if isinstance(base, pd.DataFrame): + raise RuntimeError(f"Solve-record key is not unique: {key_values}.") + record = base.to_dict() + record.update(dict(zip(keys, key_values, strict=True))) + unresolved_key = (str(record["role"]), str(record["period"])) + if unresolved_key not in candidate_unresolved.index: + raise RuntimeError(f"Candidate outcome count is unavailable for {unresolved_key}.") + result, funded = evaluate_prejoined_frozen_allocation( + record, + allocation, + config=config, + n_unresolved_candidates=int(candidate_unresolved.loc[unresolved_key]), + ) + evaluated.append(result) + joined_groups.append(funded) + return pd.DataFrame(evaluated), pd.concat(joined_groups, ignore_index=True) + + +def aggregate_portfolios(evaluated: pd.DataFrame) -> pd.DataFrame: + """Aggregate monthly cells without treating months or scopes as replications.""" + keys = ["window_id", "role", "policy_label", "comparator_rule", "paired_policy_id"] + rows: list[dict[str, Any]] = [] + for raw_keys, frame in evaluated.groupby(keys, observed=True, sort=True, dropna=False): + key_values = raw_keys if isinstance(raw_keys, tuple) else (raw_keys,) + row = aggregate_monthly_evaluation(frame) + row.update(dict(zip(keys, key_values, strict=True))) + cap = pd.to_numeric(frame["frontier_cap"], errors="coerce") + row["frontier_cap"] = float(cap.dropna().iloc[0]) if bool(cap.notna().any()) else np.nan + rows.append(row) + return pd.DataFrame(rows) + + +def indexed_portfolio_contrasts( + allocations: pd.DataFrame, + *, + loan_facts: pd.DataFrame | None, + window_id: str, + policy_ids: tuple[str, ...], + lgd: float, +) -> pd.DataFrame: + """Compute one window's contrast grid from one validated sparse index.""" + primary = allocations.loc[allocations["role"].eq("primary_oot")] + observed_windows = set(primary["window_id"].astype(str)) + if observed_windows != {str(window_id)}: + raise RuntimeError( + f"Indexed contrast window mismatch: expected={window_id!r}, " + f"observed={sorted(observed_windows)}." + ) + index = PolicyContrastIndex(primary, role="primary_oot", loan_facts=loan_facts) + frontier = primary.loc[ + primary["comparator_rule"].eq("point_cap_frontier"), + ["policy_label", "frontier_cap"], + ].drop_duplicates() + duplicate_frontier = frontier["policy_label"].duplicated(keep=False) + if bool(duplicate_frontier.any()): + raise RuntimeError("A frontier policy label maps to conflicting cap values.") + frontier = frontier.sort_values("frontier_cap") + + rows: list[dict[str, Any]] = [] + for policy_id in policy_ids: + guard_label = f"guardrail_{policy_id}" + named = ( + ("c0_same_numeric_cap", f"c0_same_numeric_cap_{policy_id}"), + ("c1_development_mean", f"c1_development_mean_{policy_id}"), + ("c2_contemporaneous", f"c2_contemporaneous_{policy_id}"), + ) + for rule, comparator_label in named: + cap = primary.loc[primary["policy_label"].eq(comparator_label), "frontier_cap"] + if cap.empty: + raise RuntimeError(f"Named comparator {comparator_label!r} is missing.") + cap_values = pd.to_numeric(cap, errors="raise").drop_duplicates() + if rule != "c2_contemporaneous" and len(cap_values) != 1: + raise RuntimeError( + f"Named comparator {comparator_label!r} maps to conflicting cap values." + ) + rows.append( + { + "window_id": str(window_id), + "paired_policy_id": policy_id, + "comparator_rule": rule, + # C2 is matched monthly; the retained scalar is legacy metadata. + "frontier_cap": float(pd.to_numeric(cap, errors="raise").iloc[0]), + **index.sharp_bounds( + policy_a=guard_label, + policy_b=comparator_label, + lgd=float(lgd), + ), + } + ) + for item in frontier.itertuples(index=False): + comparator_label = str(item.policy_label) + rows.append( + { + "window_id": str(window_id), + "paired_policy_id": policy_id, + "comparator_rule": "point_cap_frontier", + "frontier_cap": float(item.frontier_cap), + **index.sharp_bounds( + policy_a=guard_label, + policy_b=comparator_label, + lgd=float(lgd), + ), + } + ) + return pd.DataFrame(rows) + + +def paired_portfolio_contrasts( + joined_allocations: pd.DataFrame, + *, + policy_ids: tuple[str, ...], + lgd: float, +) -> pd.DataFrame: + """Compute guardrail-minus-point bounds while indexing each window once.""" + frames = [ + indexed_portfolio_contrasts( + window, + loan_facts=None, + window_id=str(window_id), + policy_ids=policy_ids, + lgd=lgd, + ) + for window_id, window in joined_allocations.groupby("window_id", observed=True, sort=True) + ] + if not frames: + raise ValueError("No allocation windows are available for policy contrasts.") + return pd.concat(frames, ignore_index=True) + + +def comparator_envelopes( + contrasts: pd.DataFrame, + support: pd.DataFrame, + *, + broad_lower: float, + broad_upper: float, +) -> pd.DataFrame: + """Summarize one interval per policy, metric, window, and named comparator scope.""" + metrics = { + "standardized_payoff": ( + "realized_payoff_difference_lower", + "realized_payoff_difference_upper", + ), + "terminal_default": ( + "weighted_default_difference_lower", + "weighted_default_difference_upper", + ), + "funded_miscoverage": ( + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ), + } + rows: list[dict[str, Any]] = [] + support_index = support.set_index(["window_id", "paired_policy_id"]) + if not support_index.index.is_unique: + raise ValueError("Comparator support contains duplicate policy-window keys.") + for raw_keys, frame in contrasts.groupby( + ["window_id", "paired_policy_id"], observed=True, sort=True + ): + key_values = raw_keys if isinstance(raw_keys, tuple) else (raw_keys,) + if len(key_values) != 2: + raise RuntimeError("Comparator-envelope key has unexpected cardinality.") + window_id, policy_id = key_values + limits = support_index.loc[(window_id, policy_id)] + scopes = { + "named_c0_c1_c2": frame.loc[ + frame["comparator_rule"].isin( + ["c0_same_numeric_cap", "c1_development_mean", "c2_contemporaneous"] + ) + ], + "development_admissible_exact_frontier": frame.loc[ + frame["comparator_rule"].eq("point_cap_frontier") + & frame["frontier_cap"].between( + float(limits["support_lower"]), + float(limits["support_upper"]), + inclusive="both", + ) + ], + "broad_stress_exact_frontier": frame.loc[ + frame["comparator_rule"].eq("point_cap_frontier") + & frame["frontier_cap"].between( + float(broad_lower), float(broad_upper), inclusive="both" + ) + ], + } + for scope, scoped in scopes.items(): + if scoped.empty: + raise RuntimeError( + f"Comparator scope {scope} is empty for {window_id}/{policy_id}." + ) + for metric, (lower, upper) in metrics.items(): + lower_value = float(scoped[lower].min()) + upper_value = float(scoped[upper].max()) + rows.append( + { + "window_id": str(window_id), + "paired_policy_id": str(policy_id), + "scope": scope, + "metric": metric, + "lower": lower_value, + "upper": upper_value, + "direction": ( + "guardrail_higher" + if lower_value > 0.0 + else "guardrail_lower" + if upper_value < 0.0 + else "crosses_zero" + ), + "evaluated_comparators": int(len(scoped)), + "nested_scope_independent_replications": False, + } + ) + return pd.DataFrame(rows) + + +def temporal_coverage_audit( + scores: pd.DataFrame, + outcomes: pd.DataFrame, + recipes: Mapping[str, Mapping[str, Mapping[int, BinaryOutcomeConformalRecipe]]], + fit_audit: pd.DataFrame, + *, + roles: Sequence[str] = EVALUATION_ROLES, + taxonomy_group_counts: Collection[int] | None = None, + strata: Collection[int] | None = None, +) -> pd.DataFrame: + """Evaluate all learner/window/taxonomy/role/stratum coverage cells.""" + requested_roles = frozenset(str(role) for role in roles) + score_role = "role" if "role" in scores.columns else "design_split" + selected_scores = scores.loc[scores[score_role].astype(str).isin(requested_roles)].copy() + joined = _exact_outcome_join( + selected_scores, + outcomes, + value_columns=("snapshot_default",), + ) + fit_lookup: dict[tuple[str, str, int], pd.DataFrame] = {} + for raw_key, frame in fit_audit.groupby( + ["learner", "window_id", "taxonomy_groups"], + observed=True, + sort=False, + ): + learner, window_id, taxonomy_groups = cast(tuple[Any, Any, Any], raw_key) + fit_lookup[(str(learner), str(window_id), int(taxonomy_groups))] = frame + rows: list[dict[str, Any]] = [] + selected_taxonomies = ( + None + if taxonomy_group_counts is None + else frozenset(int(value) for value in taxonomy_group_counts) + ) + for learner, learner_windows in recipes.items(): + probability = joined[f"pd_{learner}"].to_numpy(dtype=float) + for window_id, group_recipes in learner_windows.items(): + for taxonomy_groups, recipe in sorted(group_recipes.items()): + if selected_taxonomies is not None and taxonomy_groups not in selected_taxonomies: + continue + assigned, lower, upper = apply_binary_outcome_recipe(probability, recipe) + selected_strata = ( + (-1, *range(int(taxonomy_groups))) + if strata is None + else tuple(int(value) for value in strata) + ) + invalid_strata = [ + value for value in selected_strata if value < -1 or value >= taxonomy_groups + ] + if invalid_strata: + raise ValueError( + f"Invalid strata for {taxonomy_groups} groups: {invalid_strata}." + ) + for role in roles: + role_mask = joined["design_split"].eq(role).to_numpy(dtype=bool) + for stratum in selected_strata: + mask = role_mask & ((assigned == stratum) if stratum >= 0 else True) + if not bool(mask.any()): + raise RuntimeError( + f"Empty coverage cell: {learner}/{window_id}/{role}/{stratum}." + ) + outcome = pd.to_numeric( + joined.loc[mask, "snapshot_default"], errors="coerce" + ).to_numpy(dtype=float) + observed = np.isfinite(outcome) + miss_low, miss_high = binary_miscoverage_bounds( + outcome, + lower[mask], + upper[mask], + ) + n = int(mask.sum()) + geometry = summarize_binary_geometry(lower[mask], upper[mask]) + fitted_base = fit_lookup.get( + (str(learner), str(window_id), int(taxonomy_groups)) + ) + if fitted_base is None: + raise RuntimeError( + f"Missing fit audit: {learner}/{window_id}/{taxonomy_groups}." + ) + fitted = ( + fitted_base.loc[fitted_base["conformal_group"].eq(stratum)] + if stratum >= 0 + else fitted_base + ) + if fitted.empty: + raise RuntimeError( + f"Missing fit audit: {learner}/{window_id}/{taxonomy_groups}/{stratum}." + ) + fit_min = float(fitted["pd_point"].min()) + fit_max = float(fitted["pd_point"].max()) + if stratum >= 0: + residual_quantile = float(recipe.residual_quantiles[stratum]) + else: + residual = np.sort( + np.abs( + fitted["terminal_default"].to_numpy(dtype=float) + - fitted["pd_point"].to_numpy(dtype=float) + ) + ) + raw_rank = int(np.ceil((len(residual) + 1) * (1.0 - recipe.alpha))) + residual_quantile = ( + 1.0 if raw_rank > len(residual) else float(residual[raw_rank - 1]) + ) + rows.append( + { + "learner": learner, + "window_id": window_id, + "taxonomy_groups": int(taxonomy_groups), + "role": role, + "conformal_group": int(stratum), + "candidate_rows": n, + "resolved_rows": int(observed.sum()), + "unresolved_rows": int((~observed).sum()), + "coverage_resolved": float(1.0 - miss_low[observed].mean()), + "coverage_lower": float(1.0 - miss_high.mean()), + "coverage_upper": float(1.0 - miss_low.mean()), + "score_min": float(np.min(probability[mask])), + "score_max": float(np.max(probability[mask])), + "fit_rows": int(len(fitted)), + "fit_prevalence": float(fitted["terminal_default"].mean()), + "fit_residual_quantile": residual_quantile, + "fit_score_min": fit_min, + "fit_score_max": fit_max, + "scores_below_fit_range": int( + np.sum(probability[mask] < fit_min - 1e-12) + ), + "scores_above_fit_range": int( + np.sum(probability[mask] > fit_max + 1e-12) + ), + **geometry, + } + ) + return pd.DataFrame(rows) diff --git a/src/ijds_audit/fit_label_sensitivity.py b/src/ijds_audit/fit_label_sensitivity.py new file mode 100644 index 0000000..c3ccf6f --- /dev/null +++ b/src/ijds_audit/fit_label_sensitivity.py @@ -0,0 +1,122 @@ +"""Fitting-label completion scenarios for the active temporal design.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import pandas as pd + +FIT_LABEL_SCENARIOS = ( + "observed_only", + "all_unavailable_nondefault", + "all_unavailable_default", + "hindsight_terminal", +) +FIT_SPLITS = ("pd_development", "probability_calibration", "conformal_fit") + + +def apply_fit_label_scenario( + universe: pd.DataFrame, + *, + scenario: str, + fit_splits: Sequence[str] = FIT_SPLITS, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Complete only unavailable fitting labels under one declared scenario.""" + if scenario not in FIT_LABEL_SCENARIOS: + raise ValueError(f"Unknown fitting-label scenario: {scenario!r}.") + required = {"design_split", "terminal_default", "label_available"} + missing = sorted(required.difference(universe.columns)) + if missing: + raise KeyError(f"Fitting-label sensitivity is missing columns: {missing}.") + + completed = universe.copy() + fit_mask = completed["design_split"].astype(str).isin(tuple(map(str, fit_splits))) + unavailable = fit_mask & ~completed["label_available"].astype(bool) + if bool(completed.loc[unavailable, "terminal_default"].isna().any()): + raise RuntimeError("Unavailable fitting rows do not have terminal archive outcomes.") + + if scenario != "observed_only": + completed.loc[unavailable, "label_available"] = True + if scenario == "all_unavailable_nondefault": + completed.loc[unavailable, "terminal_default"] = 0 + elif scenario == "all_unavailable_default": + completed.loc[unavailable, "terminal_default"] = 1 + elif scenario != "hindsight_terminal": + raise AssertionError("Scenario validation is incomplete.") + completed["terminal_default"] = completed["terminal_default"].astype("Int8") + completed["label_available"] = completed["label_available"].astype(bool) + + outside_fit = ~fit_mask + if not completed.loc[outside_fit, ["terminal_default", "label_available"]].equals( + universe.loc[outside_fit, ["terminal_default", "label_available"]] + ): + raise RuntimeError("A fitting-label scenario changed an evaluation row.") + + rows: list[dict[str, object]] = [] + for split in fit_splits: + split_mask = completed["design_split"].astype(str).eq(str(split)) + source_available = universe.loc[split_mask, "label_available"].astype(bool) + active_available = completed.loc[split_mask, "label_available"].astype(bool) + source_labels = universe.loc[split_mask, "terminal_default"].astype("Int8") + active_labels = completed.loc[split_mask, "terminal_default"].astype("Int8") + source_prevalence = ( + float(source_labels[source_available].mean()) + if bool(source_available.any()) + else np.nan + ) + active_prevalence = ( + float(active_labels[active_available].mean()) + if bool(active_available.any()) + else np.nan + ) + rows.append( + { + "scenario": scenario, + "design_split": str(split), + "rows": int(split_mask.sum()), + "source_available_rows": int(source_available.sum()), + "source_unavailable_rows": int((~source_available).sum()), + "active_available_rows": int(active_available.sum()), + "completed_rows": int((active_available & ~source_available).sum()), + "source_available_prevalence": source_prevalence, + "active_prevalence": active_prevalence, + } + ) + return completed, pd.DataFrame(rows) + + +def summarize_fit_label_coverage( + coverage: pd.DataFrame, + *, + scenarios: Sequence[str] = FIT_LABEL_SCENARIOS, + window_ids: Sequence[str], + nominal_coverage: float, +) -> pd.DataFrame: + """Summarize the complete overall-window coverage grid by scenario.""" + overall = coverage.loc[coverage["conformal_group"].eq(-1)].copy() + expected = {(str(scenario), str(window)) for scenario in scenarios for window in window_ids} + observed = set(zip(overall["fit_label_scenario"], overall["window_id"], strict=True)) + if observed != expected or len(overall) != len(expected): + raise RuntimeError("Fitting-label coverage grid is incomplete.") + rows: list[dict[str, object]] = [] + for scenario, frame in overall.groupby("fit_label_scenario", observed=True, sort=True): + lower = pd.to_numeric(frame["coverage_lower"], errors="raise") + upper = pd.to_numeric(frame["coverage_upper"], errors="raise") + rows.append( + { + "fit_label_scenario": str(scenario), + "windows": int(len(frame)), + "coverage_lower_min": float(lower.min()), + "coverage_upper_max": float(upper.max()), + "windows_upper_below_nominal": int(upper.lt(nominal_coverage).sum()), + "all_windows_upper_below_nominal": bool(upper.lt(nominal_coverage).all()), + "mean_width_min": float(pd.to_numeric(frame["mean_width"], errors="raise").min()), + "mean_width_max": float(pd.to_numeric(frame["mean_width"], errors="raise").max()), + } + ) + result = pd.DataFrame(rows) + numeric = result.select_dtypes(include=[np.number]).to_numpy(dtype=float) + if not bool(np.isfinite(numeric).all()): + raise RuntimeError("Fitting-label coverage summary contains non-finite values.") + return result diff --git a/src/ijds_audit/geometry.py b/src/ijds_audit/geometry.py new file mode 100644 index 0000000..1ee8991 --- /dev/null +++ b/src/ijds_audit/geometry.py @@ -0,0 +1,100 @@ +"""Binary prediction-set geometry for clipped residual intervals.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np + +EMPTY = 0 +ZERO_ONLY = 1 +ONE_ONLY = 2 +BOTH = 3 +SET_LABELS = {EMPTY: "empty", ZERO_ONLY: "{0}", ONE_ONLY: "{1}", BOTH: "{0,1}"} + + +@dataclass(frozen=True) +class ConstantScorePhase: + """Population geometry under a constant score and Bernoulli prevalence.""" + + residual_quantile: float + discrete_set: str + coverage: float + + +def binary_set_codes( + lower: Sequence[float] | np.ndarray, + upper: Sequence[float] | np.ndarray, + *, + tolerance: float = 1e-12, +) -> np.ndarray: + """Encode each interval's intersection with the binary outcome space.""" + low = np.asarray(lower, dtype=float) + high = np.asarray(upper, dtype=float) + if low.shape != high.shape or low.ndim != 1: + raise ValueError("Binary interval endpoints must be aligned one-dimensional arrays.") + if not bool(np.isfinite(low).all() and np.isfinite(high).all()): + raise ValueError("Binary interval endpoints must be finite.") + if bool(np.any(low > high + tolerance)): + raise ValueError("An interval has lower endpoint above its upper endpoint.") + contains_zero = (low <= tolerance) & (high >= -tolerance) + contains_one = (low <= 1.0 + tolerance) & (high >= 1.0 - tolerance) + return contains_zero.astype(np.int8) + 2 * contains_one.astype(np.int8) + + +def summarize_binary_geometry( + lower: Sequence[float] | np.ndarray, + upper: Sequence[float] | np.ndarray, + *, + width_quantiles: Sequence[float] = (0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0), +) -> dict[str, Any]: + """Return continuous width and discrete-set diagnostics without outcomes.""" + low = np.asarray(lower, dtype=float) + high = np.asarray(upper, dtype=float) + if len(low) == 0: + raise ValueError("Binary geometry requires at least one interval.") + codes = binary_set_codes(low, high) + width = high - low + result: dict[str, Any] = { + "rows": int(len(low)), + "mean_width": float(np.mean(width)), + "lower_positive_share": float(np.mean(low > 1e-12)), + "upper_saturated_share": float(np.mean(high >= 1.0 - 1e-12)), + } + for code in SET_LABELS: + key = {EMPTY: "empty", ZERO_ONLY: "zero_only", ONE_ONLY: "one_only", BOTH: "both"}[code] + count = int(np.sum(codes == code)) + result[f"set_{key}_count"] = count + result[f"set_{key}_share"] = float(count / len(codes)) if len(codes) else np.nan + for quantile in width_quantiles: + if not 0.0 <= float(quantile) <= 1.0: + raise ValueError("Width quantiles must lie in [0, 1].") + name = f"width_q{int(round(100 * float(quantile))):02d}" + result[name] = float(np.quantile(width, float(quantile))) + return result + + +def constant_score_population_phase( + *, score: float, prevalence: float, alpha: float +) -> ConstantScorePhase: + """Evaluate the exact population phase for p<1/2 and Bernoulli outcomes. + + At a constant score ``p``, residuals are ``p`` for Y=0 and ``1-p`` for + Y=1. The lower population quantile is selected when prevalence is at most + alpha; otherwise the upper residual is required. + """ + p = float(score) + pi = float(prevalence) + a = float(alpha) + if not 0.0 <= p < 0.5: + raise ValueError("The stated proposition requires 0 <= score < 1/2.") + if not 0.0 <= pi <= 1.0 or not 0.0 < a < 1.0: + raise ValueError("Prevalence and alpha are outside their probability domains.") + quantile = p if pi <= a else 1.0 - p + lower = max(0.0, p - quantile) + upper = min(1.0, p + quantile) + code = int(binary_set_codes([lower], [upper])[0]) + coverage = (1.0 - pi) * float(lower <= 0.0 <= upper) + pi * float(lower <= 1.0 <= upper) + return ConstantScorePhase(quantile, SET_LABELS[code], float(coverage)) diff --git a/src/ijds_audit/grid_contracts.py b/src/ijds_audit/grid_contracts.py new file mode 100644 index 0000000..cd65cf6 --- /dev/null +++ b/src/ijds_audit/grid_contracts.py @@ -0,0 +1,135 @@ +"""Small dataframe contracts for paper-facing evidence grids.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from itertools import product +from typing import Any + +import numpy as np +import pandas as pd +from pandas.api.types import is_float_dtype +from pandas.testing import assert_frame_equal + + +def require_exact_grid( + frame: pd.DataFrame, + *, + domains: Mapping[str, Sequence[Any]], + label: str, +) -> None: + """Require exactly one row for every declared Cartesian key.""" + keys = tuple(domains) + missing_columns = sorted(set(keys).difference(frame.columns)) + if missing_columns: + raise KeyError(f"{label} is missing grid keys: {missing_columns}.") + if bool(frame.duplicated(list(keys)).any()): + raise RuntimeError(f"{label} contains duplicate grid keys.") + actual = set(frame.loc[:, list(keys)].itertuples(index=False, name=None)) + expected = set(product(*(domains[key] for key in keys))) + if actual != expected: + missing = sorted(expected.difference(actual), key=repr)[:5] + extra = sorted(actual.difference(expected), key=repr)[:5] + raise RuntimeError(f"{label} grid changed; missing={missing}, extra={extra}.") + + +def require_finite(frame: pd.DataFrame, columns: Sequence[str], *, label: str) -> None: + """Reject missing, nonnumeric, NaN, or infinite paper-facing values.""" + missing = sorted(set(columns).difference(frame.columns)) + if missing: + raise KeyError(f"{label} is missing numeric columns: {missing}.") + values = frame.loc[:, list(columns)].apply(pd.to_numeric, errors="coerce").to_numpy() + if not bool(np.isfinite(values).all()): + raise RuntimeError(f"{label} contains nonfinite numeric evidence.") + + +def require_unique_row( + frame: pd.DataFrame, + *, + key: Mapping[str, Any], + label: str, +) -> pd.Series: + """Return the single row matching named keys, independent of row order.""" + missing = sorted(set(key).difference(frame.columns)) + if missing: + raise KeyError(f"{label} is missing lookup keys: {missing}.") + mask = pd.Series(True, index=frame.index) + for column, value in key.items(): + mask &= frame[column].eq(value) + rows = frame.loc[mask] + if len(rows) != 1: + raise RuntimeError(f"{label} requires one row for {dict(key)}, found {len(rows)}.") + return rows.iloc[0] + + +def require_unique_value(frame: pd.DataFrame, column: str, *, label: str) -> Any: + """Return one nonmissing value only when it is constant over the frame.""" + if column not in frame: + raise KeyError(f"{label} is missing column {column!r}.") + values = frame[column].drop_duplicates() + if len(values) != 1 or bool(values.isna().any()): + raise RuntimeError(f"{label} has no unique nonmissing {column!r} value.") + return values.iloc[0] + + +def require_exact_frame( + actual: pd.DataFrame, + expected: pd.DataFrame, + *, + keys: Sequence[str], + label: str, + float_atol: float = 0.0, + float_rtol: float = 0.0, + allowed_expected_extra_columns: Sequence[str] = (), +) -> None: + """Require exact frames, optionally allowing declared float roundoff only.""" + if ( + not np.isfinite(float_atol) + or not np.isfinite(float_rtol) + or float_atol < 0.0 + or float_rtol < 0.0 + or float_atol > 1.0e-12 + or float_rtol > 1.0e-12 + ): + raise ValueError("Frame float tolerances must be finite and at most 1e-12.") + actual_columns = set(actual.columns) + expected_columns = set(expected.columns) + actual_only = actual_columns.difference(expected_columns) + expected_only = expected_columns.difference(actual_columns) + allowed_extras = set(allowed_expected_extra_columns) + if actual_only or expected_only.difference(allowed_extras): + raise RuntimeError(f"{label} columns differ from the active reference.") + columns = [column for column in expected.columns if column in actual_columns] + actual_sorted = ( + actual.loc[:, columns].sort_values(list(keys), kind="stable").reset_index(drop=True) + ) + expected_sorted = ( + expected.loc[:, columns].sort_values(list(keys), kind="stable").reset_index(drop=True) + ) + exact_columns = [ + column + for column in columns + if column in keys or not is_float_dtype(expected_sorted[column].dtype) + ] + float_columns = [column for column in columns if column not in exact_columns] + try: + if exact_columns: + assert_frame_equal( + actual_sorted.loc[:, exact_columns], + expected_sorted.loc[:, exact_columns], + check_exact=True, + check_dtype=True, + check_like=False, + ) + if float_columns: + assert_frame_equal( + actual_sorted.loc[:, float_columns], + expected_sorted.loc[:, float_columns], + check_exact=float_atol == 0.0 and float_rtol == 0.0, + check_dtype=True, + check_like=False, + atol=float_atol, + rtol=float_rtol, + ) + except AssertionError as error: + raise RuntimeError(f"{label} does not reconcile exactly to active evidence.") from error diff --git a/src/ijds_audit/lag_sensitivity.py b/src/ijds_audit/lag_sensitivity.py new file mode 100644 index 0000000..68520b5 --- /dev/null +++ b/src/ijds_audit/lag_sensitivity.py @@ -0,0 +1,101 @@ +"""Outcome-label lag sensitivity for frozen scores and taxonomies.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from src.data.outcome_observability import build_outcome_label_availability +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, + fit_binary_outcome_recipe, +) + + +def build_label_lag_phase_sensitivity( + universe: pd.DataFrame, + probabilities: np.ndarray, + frozen_recipes: Mapping[str, Mapping[int, BinaryOutcomeConformalRecipe]], + config: Mapping[str, Any], + *, + lag_months: Sequence[int], +) -> pd.DataFrame: + """Refit residual recipes only, preserving frozen scores and taxonomy edges.""" + if len(universe) != len(probabilities): + raise ValueError("Universe and frozen probabilities must align row-for-row.") + canonical_groups = int(config["conformal"]["canonical_groups"]) + phase_stratum = int(config["lag_sensitivity"]["phase_stratum"]) + if not 0 <= phase_stratum < canonical_groups: + raise ValueError("Phase stratum is outside the canonical taxonomy.") + conformal = universe["design_split"].eq("conformal_fit") + cutoff = str(config["source"]["information_cutoff"]) + alpha = float(config["conformal"]["alpha"]) + threshold = float(config["lag_sensitivity"]["minimum_monthly_label_retention"]) + rows: list[dict[str, Any]] = [] + for lag in lag_months: + labels = build_outcome_label_availability( + universe["loan_status"], + universe["last_pymnt_d"], + cutoff=cutoff, + charged_off_lag_months=int(lag), + ) + available = labels["label_available"].astype(bool) + conformal_frame = universe.loc[conformal, ["issue_d"]].copy() + conformal_frame["label_available"] = available.loc[conformal].to_numpy() + monthly_retention = ( + conformal_frame.assign(month=conformal_frame["issue_d"].dt.to_period("M")) + .groupby("month", observed=True)["label_available"] + .mean() + ) + minimum_monthly_retention = float(monthly_retention.min()) + for specification in config["residual_specification"]["windows"]: + window_id = str(specification["id"]) + window = universe["issue_d"].between( + pd.Timestamp(specification["start"]), + pd.Timestamp(specification["end"]), + ) + eligible = conformal & window + retained = eligible & available + frozen = frozen_recipes[window_id][canonical_groups] + score = probabilities[retained.to_numpy(dtype=bool)] + outcome = labels.loc[retained, "terminal_outcome"].astype("int8").to_numpy(dtype=int) + recipe = fit_binary_outcome_recipe( + score, + outcome, + alpha=alpha, + n_groups=canonical_groups, + bin_edges=frozen.bin_edges, + taxonomy_provenance=frozen.taxonomy_provenance, + taxonomy_method=frozen.taxonomy_method, + method=frozen.method, + ) + assigned, lower, upper = apply_binary_outcome_recipe(score, recipe) + phase = assigned == phase_stratum + rows.append( + { + "charged_off_lag_months": int(lag), + "window_id": window_id, + "window_rows": int(eligible.sum()), + "retained_rows": int(retained.sum()), + "window_retention": float(retained.sum() / eligible.sum()), + "minimum_monthly_retention": minimum_monthly_retention, + "passes_locked_retention": bool(minimum_monthly_retention > threshold), + "phase_stratum": phase_stratum, + "phase_rows": int(phase.sum()), + "phase_prevalence": float(outcome[phase].mean()), + "phase_residual_quantile": float(recipe.residual_quantiles[phase_stratum]), + "phase_interval_width_mean": float((upper[phase] - lower[phase]).mean()), + "phase_set_both_share": float( + np.mean((lower[phase] <= 0.0) & (upper[phase] >= 1.0)) + ), + } + ) + return ( + pd.DataFrame(rows) + .sort_values(["charged_off_lag_months", "window_id"], kind="stable") + .reset_index(drop=True) + ) diff --git a/src/ijds_audit/missingness_sensitivity.py b/src/ijds_audit/missingness_sensitivity.py new file mode 100644 index 0000000..22485db --- /dev/null +++ b/src/ijds_audit/missingness_sensitivity.py @@ -0,0 +1,525 @@ +"""Protocol-locked sensitivity across three declared missing-value encodings.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pandas as pd +import yaml + +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.credit_controls import credit_prediction_metrics +from src.ijds_audit.evaluation import endpoint_resolution_audit, temporal_coverage_audit +from src.ijds_audit.prediction import ( + PreparedData, + fit_primary_scores, + fit_window_recipes, + prepare_data, +) +from src.ijds_audit.protocol import ( + configured_archive_outcomes, + load_outcome_universe, + load_recipes, + recipe_payload, + verified_freeze_artifact_paths, +) +from src.utils.isolated_experiment import ( + environment_provenance, + git_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, + save_catboost_model_atomic, +) +from src.utils.pipeline_runtime import atomic_write_json, atomic_write_parquet, atomic_write_pickle + +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") +SPECIFICATION_IDS = ( + "catboost_platt", + "catboost_missing_indicators_platt", + "catboost_native_missing_platt", +) + + +def load_missingness_config(path: Path, *, repo_root: Path) -> dict[str, Any]: + """Load and validate the closed missingness-sensitivity contract.""" + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Missingness protocol must be a YAML mapping.") + required = { + "schema_version", + "protocol_status", + "protocol_tag", + "run_tag", + "base_config", + "protocol_document", + "source_freeze", + "specifications", + "evaluation", + "output", + } + missing = sorted(required.difference(payload)) + if missing: + raise KeyError(f"Missingness protocol is missing sections: {missing}.") + if payload["protocol_status"] != "locked_outcome_free_then_evaluate_missingness_sensitivity": + raise ValueError("Unexpected missingness protocol status.") + specifications = payload["specifications"] + if not isinstance(specifications, list): + raise TypeError("Missingness specifications must be a list.") + if tuple(str(item["id"]) for item in specifications) != SPECIFICATION_IDS: + raise ValueError("The complete missingness specification family changed.") + schema_version = str(payload["schema_version"]) + if schema_version != "2026-07-15.3": + raise ValueError("Unexpected missingness protocol schema version.") + expected_encodings = ( + "active_sentinel_convention", + "active_mappings_plus_explicit_missing_indicators", + "native_numeric_nan_with_nullable_binary_semantics", + ) + if tuple(str(item["encoding"]) for item in specifications) != expected_encodings: + raise ValueError("The missingness encoding family changed.") + if specifications[0] != { + "id": "catboost_platt", + "encoding": "active_sentinel_convention", + "imported": True, + }: + raise ValueError("The imported missingness baseline contract changed.") + if specifications[1] != { + "id": "catboost_missing_indicators_platt", + "encoding": "active_mappings_plus_explicit_missing_indicators", + "imported": False, + "added_numeric_features": [ + "delinq_recency_missing", + "bankruptcy_count_missing", + ], + }: + raise ValueError("The explicit-indicator feature contract changed.") + if specifications[2] != { + "id": "catboost_native_missing_platt", + "encoding": "native_numeric_nan_with_nullable_binary_semantics", + "imported": False, + "dropped_numeric_features": ["delinq_recency", "has_bankruptcy"], + "added_numeric_features": ["delinq_recency_native", "has_bankruptcy_native"], + }: + raise ValueError("The native-missingness feature contract changed.") + evaluation = payload["evaluation"] + expected_evaluation = { + "role": "primary_oot", + "expected_candidates": 376890, + "expected_windows": 8, + "taxonomy_groups": 5, + "aggregate_stratum": -1, + "nominal_coverage": 0.90, + "no_model_selection": True, + "no_window_selection": True, + "no_portfolio_optimization": True, + } + if evaluation != expected_evaluation: + raise ValueError("The missingness evaluation contract changed.") + if payload["output"].get("immutability") != "hard_no_overwrite_choose_fresh_run_tag": + raise ValueError("Missingness outputs must be immutable.") + source = payload["source_freeze"] + if not isinstance(source, Mapping) or str(source.get("sha256", "")) != ( + "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" + ): + raise ValueError("The active V4-v1 source freeze changed.") + base_path = resolve_repo_input(str(payload["base_config"]), repo_root=repo_root) + load_v4_config(base_path) + return payload + + +def _base_config(config: Mapping[str, Any], *, repo_root: Path) -> dict[str, Any]: + return load_v4_config(resolve_repo_input(str(config["base_config"]), repo_root=repo_root)) + + +def build_missingness_variant(data: PreparedData, *, variant: str) -> PreparedData: + """Construct one declared feature matrix without changing the row universe.""" + features = data.features.copy() + numeric = list(data.numeric_features) + if variant == "explicit_indicators": + features["delinq_recency_missing"] = ( + pd.to_numeric(data.universe["mths_since_last_delinq"], errors="coerce") + .isna() + .astype("int8") + ) + features["bankruptcy_count_missing"] = ( + pd.to_numeric(data.universe["pub_rec_bankruptcies"], errors="coerce") + .isna() + .astype("int8") + ) + numeric.extend(["delinq_recency_missing", "bankruptcy_count_missing"]) + elif variant == "native_missing": + features = features.drop(columns=["delinq_recency", "has_bankruptcy"]) + numeric = [name for name in numeric if name not in {"delinq_recency", "has_bankruptcy"}] + features["delinq_recency_native"] = pd.to_numeric( + data.universe["mths_since_last_delinq"], errors="coerce" + ) + bankruptcy_count = pd.to_numeric(data.universe["pub_rec_bankruptcies"], errors="coerce") + features["has_bankruptcy_native"] = ( + bankruptcy_count.gt(0).astype("float64").where(bankruptcy_count.notna()) + ) + numeric.extend(["delinq_recency_native", "has_bankruptcy_native"]) + else: + raise ValueError(f"Unknown missingness variant: {variant!r}.") + if len(numeric) != len(set(numeric)): + raise RuntimeError("Missingness variant contains duplicate numeric features.") + if not features.index.equals(data.features.index): + raise RuntimeError("Missingness variant changed the row index.") + return replace(data, features=features, numeric_features=tuple(numeric)) + + +def _verified_source_freeze( + config: Mapping[str, Any], *, repo_root: Path +) -> tuple[dict[str, Any], dict[str, Path]]: + source = config["source_freeze"] + path = resolve_repo_input(str(source["path"]), repo_root=repo_root) + descriptor = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if descriptor[field] != source[field]: + raise RuntimeError(f"Missingness source freeze mismatch for {field}.") + freeze = json.loads(path.read_text(encoding="utf-8")) + expected = { + "status": "outcome_free_allocations_frozen_before_archive_outcome_join", + "run_tag": str(source["run_tag"]), + "protocol_tag": str(source["protocol_tag"]), + "protocol_commit": str(source["protocol_commit"]), + } + if any(freeze.get(field) != value for field, value in expected.items()): + raise RuntimeError("Missingness source freeze identity changed.") + if freeze.get("outcome_columns_passed_to_policy_or_comparator") != []: + raise RuntimeError("Missingness source freeze reports outcome leakage.") + return freeze, verified_freeze_artifact_paths(freeze, repo_root=repo_root) + + +def _missingness_census(data: PreparedData) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for role, frame in data.universe.groupby("design_split", observed=True, sort=True): + for feature in ("mths_since_last_delinq", "pub_rec_bankruptcies"): + missing = pd.to_numeric(frame[feature], errors="coerce").isna() + rows.append( + { + "role": str(role), + "feature": feature, + "rows": int(len(frame)), + "missing_rows": int(missing.sum()), + "missing_share": float(missing.mean()), + } + ) + return pd.DataFrame(rows) + + +def _implementation( + config_path: Path, + config: Mapping[str, Any], + *, + repo_root: Path, +) -> dict[str, Any]: + return implementation_provenance( + config_path=config_path, + repo_root=repo_root, + relative_paths=[ + Path("scripts/experiments/run_ijds_missingness_sensitivity.py"), + Path("src/ijds_audit/missingness_sensitivity.py"), + Path("src/ijds_audit/evaluation.py"), + Path("src/ijds_audit/prediction.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/data/outcome_observability.py"), + Path("src/features/feature_engineering.py"), + Path("docs/research/ijds_missingness_sensitivity_protocol_2026-07-15.md"), + *[Path(value) for value in config.get("implementation_lineage_files", [])], + ], + ) + + +def freeze_missingness_sensitivity(*, config_path: Path, repo_root: Path) -> Path: + """Freeze baseline and two alternative score/recipe specifications.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_missingness_config(resolved_config, repo_root=root) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + source_freeze, source_artifacts = _verified_source_freeze(config, repo_root=root) + parent = _base_config(config, repo_root=root) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + raw_path = resolve_repo_input(str(parent["source"]["raw_path"]), repo_root=root) + data = prepare_data(parent, raw_path=raw_path) + + source_scores = pd.read_parquet(source_artifacts["scores"]) + identity = pd.DataFrame( + { + "id": data.universe["id"].astype("string"), + "issue_d": data.universe["issue_d"], + "design_split": data.universe["design_split"].astype("string"), + } + ) + baseline = identity.merge( + source_scores[["id", "design_split", "pd_catboost_platt"]], + on=["id", "design_split"], + how="left", + validate="one_to_one", + ) + if len(baseline) != len(identity) or bool(baseline["pd_catboost_platt"].isna().any()): + raise RuntimeError("The frozen baseline score does not align to the active universe.") + + indicator_data = build_missingness_variant(data, variant="explicit_indicators") + native_data = build_missingness_variant(data, variant="native_missing") + indicator = replace( + fit_primary_scores(indicator_data, parent), + name="catboost_missing_indicators_platt", + ) + native = replace( + fit_primary_scores(native_data, parent), + name="catboost_native_missing_platt", + ) + alternative_windows = { + indicator.name: fit_window_recipes(indicator_data, indicator, parent), + native.name: fit_window_recipes(native_data, native, parent), + } + baseline_recipes = json.loads(source_artifacts["recipes"].read_text(encoding="utf-8")) + combined_recipes = { + "catboost_platt": baseline_recipes["catboost_platt"], + **recipe_payload(alternative_windows), + } + source_fit = pd.read_parquet(source_artifacts["fit_audit"]) + fit_audit = pd.concat( + [ + source_fit.loc[source_fit["learner"].eq("catboost_platt")], + *[ + window.fit_audit + for learner_windows in alternative_windows.values() + for window in learner_windows.values() + ], + ], + ignore_index=True, + ) + scores = baseline.assign( + pd_catboost_missing_indicators_platt=indicator.probabilities, + pd_catboost_native_missing_platt=native.probabilities, + ) + census = _missingness_census(data) + artifacts = { + "scores": atomic_write_parquet(scores, paths.data_dir / "prediction/scores.parquet"), + "recipes": atomic_write_json( + paths.model_dir / "prediction/residual_recipes.json", combined_recipes + ), + "fit_audit": atomic_write_parquet( + fit_audit, paths.data_dir / "prediction/residual_fit_audit.parquet" + ), + "missingness_census": atomic_write_parquet( + census, paths.data_dir / "data/missingness_census.parquet" + ), + } + model_artifacts = { + "catboost_missing_indicators": relative_artifact_descriptor( + save_catboost_model_atomic( + indicator.model, paths.model_dir / "prediction/catboost_missing_indicators.cbm" + ), + repo_root=root, + ), + "catboost_missing_indicators_platt": relative_artifact_descriptor( + atomic_write_pickle( + paths.model_dir / "prediction/catboost_missing_indicators_platt.pkl", + indicator.calibrator, + ), + repo_root=root, + ), + "catboost_native_missing": relative_artifact_descriptor( + save_catboost_model_atomic( + native.model, paths.model_dir / "prediction/catboost_native_missing.cbm" + ), + repo_root=root, + ), + "catboost_native_missing_platt": relative_artifact_descriptor( + atomic_write_pickle( + paths.model_dir / "prediction/catboost_native_missing_platt.pkl", + native.calibrator, + ), + repo_root=root, + ), + } + freeze = { + "schema_version": str(config["schema_version"]), + "status": "missingness_scores_frozen_before_primary_oot_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "specifications": list(config["specifications"]), + "source_freeze": dict(config["source_freeze"]), + "source_outcome_free_status": str(source_freeze["status"]), + "primary_oot_outcome_columns_in_frozen_scores": [], + "outcome_free_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifacts.items() + }, + "model_artifacts": model_artifacts, + "implementation_provenance": _implementation(resolved_config, config, repo_root=root), + "environment": environment_provenance(root), + "git": git_provenance(root), + "selection": {"model": None, "encoding": None, "window": None}, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze) + + +def _prediction_metrics(scores: pd.DataFrame, outcomes: pd.DataFrame) -> pd.DataFrame: + primary_scores = scores.loc[scores["design_split"].eq("primary_oot")].copy() + primary_outcomes = outcomes.loc[outcomes["role"].eq("primary_oot"), ["id", "snapshot_default"]] + joined = primary_scores.merge(primary_outcomes, on="id", how="outer", validate="one_to_one") + if len(joined) != 376890 or bool(joined["design_split"].isna().any()): + raise RuntimeError("Missingness prediction outcome census changed.") + observed = joined["snapshot_default"].notna() + labels = joined.loc[observed, "snapshot_default"].astype(int).to_numpy() + rows: list[dict[str, Any]] = [] + for learner in SPECIFICATION_IDS: + metrics = credit_prediction_metrics( + labels, + joined.loc[observed, f"pd_{learner}"].to_numpy(dtype=float), + ) + rows.append( + { + "learner": learner, + "candidate_rows": int(len(joined)), + "resolved_rows": int(observed.sum()), + "unresolved_rows": int((~observed).sum()), + **metrics, + } + ) + return pd.DataFrame(rows) + + +def evaluate_missingness_sensitivity(*, config_path: Path, repo_root: Path) -> Path: + """Join the corrected endpoint once and report the complete specification family.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_missingness_config(resolved_config, repo_root=root) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + parent = _base_config(config, repo_root=root) + model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(config["run_tag"]), + ) + data_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=str(config["run_tag"]), + ) + freeze_path = model_dir / "protocol_freeze.json" + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + expected = { + "status": "missingness_scores_frozen_before_primary_oot_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + } + if any(freeze.get(field) != value for field, value in expected.items()): + raise RuntimeError("Missingness freeze identity changed before evaluation.") + if freeze.get("primary_oot_outcome_columns_in_frozen_scores") != []: + raise RuntimeError("Missingness freeze reports OOT outcome leakage.") + if (data_dir / "evaluation").exists(): + raise FileExistsError("Missingness evaluation already exists.") + artifacts = verified_freeze_artifact_paths(freeze, repo_root=root) + raw_path = resolve_repo_input(str(parent["source"]["raw_path"]), repo_root=root) + universe = load_outcome_universe(parent, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, parent) + scores = pd.read_parquet(artifacts["scores"]) + recipes = load_recipes(artifacts["recipes"]) + fit_audit = pd.read_parquet(artifacts["fit_audit"]) + coverage = temporal_coverage_audit( + scores, + outcomes, + recipes, + fit_audit, + roles=("primary_oot",), + taxonomy_group_counts=(5,), + strata=(-1,), + ) + prediction = _prediction_metrics(scores, outcomes) + endpoint_audit = endpoint_resolution_audit(outcomes, roles=("primary_oot",)) + if len(coverage) != 24 or coverage["learner"].nunique() != 3: + raise RuntimeError("Missingness coverage grid is incomplete.") + summaries: list[dict[str, Any]] = [] + nominal = float(config["evaluation"]["nominal_coverage"]) + for learner, frame in coverage.groupby("learner", observed=True, sort=True): + if len(frame) != int(config["evaluation"]["expected_windows"]): + raise RuntimeError(f"Missingness coverage is incomplete for {learner}.") + summaries.append( + { + "learner": str(learner), + "coverage_lower_min": float(frame["coverage_lower"].min()), + "coverage_upper_max": float(frame["coverage_upper"].max()), + "windows_with_upper_below_nominal": int(frame["coverage_upper"].lt(nominal).sum()), + "all_windows_upper_below_nominal": bool(frame["coverage_upper"].lt(nominal).all()), + } + ) + output_files = { + "temporal_coverage": atomic_write_parquet( + coverage, data_dir / "evaluation/temporal_coverage.parquet" + ), + "prediction_metrics": atomic_write_parquet( + prediction, data_dir / "evaluation/prediction_metrics.parquet" + ), + "endpoint_resolution_audit": atomic_write_parquet( + endpoint_audit, data_dir / "evaluation/endpoint_resolution_audit.parquet" + ), + } + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_no_selection_missingness_sensitivity", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "specifications": list(config["specifications"]), + "coverage": summaries, + "prediction_metrics": prediction.to_dict(orient="records"), + "endpoint_resolution_audit": endpoint_audit.to_dict(orient="records"), + "interpretation": { + "model_or_encoding_selected": False, + "portfolio_claim_authorized": False, + "missing_at_random_claim_authorized": False, + "robustness_scope": ( + "three_declared_feature_semantics_preserving_missingness_encodings_only" + ), + }, + "source_freeze": relative_artifact_descriptor(freeze_path, repo_root=root), + "evaluation_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in output_files.items() + }, + "implementation_provenance": _implementation(resolved_config, config, repo_root=root), + "environment": environment_provenance(root), + "git": git_provenance(root), + "protected_stages_run": [], + "protected_artifacts_written": [], + } + summary_path = atomic_write_json( + model_dir / str(config["output"]["deterministic_summary"]), summary + ) + atomic_write_json( + model_dir / str(config["output"]["execution_receipt"]), + { + "status": summary["status"], + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "summary": relative_artifact_descriptor(summary_path, repo_root=root), + "protected_stages_run": [], + "protected_artifacts_written": [], + }, + ) + return summary_path diff --git a/src/ijds_audit/policy_support.py b/src/ijds_audit/policy_support.py new file mode 100644 index 0000000..e033d4c --- /dev/null +++ b/src/ijds_audit/policy_support.py @@ -0,0 +1,193 @@ +"""Outcome-free policy-support and point-LP stability diagnostics.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable, Sequence +from typing import Any + +import highspy +import numpy as np +import pandas as pd + +from src.ijds_audit.portfolio import PointPortfolioSession, PointPortfolioSolution + +NAMED_SOURCE = { + "c0_same_numeric_cap": "named_c0", + "c1_development_mean": "named_c1", + "c2_contemporaneous": "named_c2", +} + + +def classify_cap( + cap: float, + *, + minimum_feasible_score: float, + unconstrained_objective_score: float, + tolerance: float, +) -> str: + """Classify a score cap without observing any realized outcome.""" + value = float(cap) + lower = float(minimum_feasible_score) + upper = float(unconstrained_objective_score) + tol = float(tolerance) + if upper < lower - tol: + raise ValueError("The unconstrained objective score is below the feasible minimum.") + if value < lower - tol: + return "infeasible" + if abs(value - lower) <= tol: + return "minimum_boundary" + if value < upper - tol: + return "decision_active" + if abs(value - upper) <= tol: + return "objective_boundary" + return "objective_slack" + + +def point_basis_diagnostics( + session: PointPortfolioSession, + solution: PointPortfolioSolution, + *, + dual_tolerance: float, + primal_tolerance: float, +) -> dict[str, Any]: + """Inspect one optimal HiGHS basis for alternate-optimum warning signs.""" + raw = session.solver.getSolution() + basis = session.solver.getBasis() + lp = session.solver.getLp() + values = np.asarray(raw.col_value, dtype=float) + reduced = np.asarray(raw.col_dual, dtype=float) + costs = np.asarray(lp.col_cost_, dtype=float) + statuses = np.asarray(basis.col_status, dtype=object) + basic = statuses == highspy.HighsBasisStatus.kBasic + lower = statuses == highspy.HighsBasisStatus.kLower + upper = statuses == highspy.HighsBasisStatus.kUpper + nonbasic = ~basic + if not bool(nonbasic.any()): + raise RuntimeError("Point LP basis unexpectedly has no nonbasic columns.") + absolute_reduced = np.abs(reduced[nonbasic]) + scaled_reduced = absolute_reduced / np.maximum(1.0, np.abs(costs[nonbasic])) + near_zero = absolute_reduced <= float(dual_tolerance) + basic_column_degeneracy = basic & ( + (values <= float(primal_tolerance)) | (values >= 1.0 - float(primal_tolerance)) + ) + + row_statuses = np.asarray(basis.row_status, dtype=object) + row_values = np.asarray(raw.row_value, dtype=float) + row_lower = np.asarray(lp.row_lower_, dtype=float) + row_upper = np.asarray(lp.row_upper_, dtype=float) + basic_rows = row_statuses == highspy.HighsBasisStatus.kBasic + inequality_rows = np.abs(row_upper - row_lower) > float(primal_tolerance) + lower_distance = np.where(np.isfinite(row_lower), np.abs(row_values - row_lower), np.inf) + upper_distance = np.where(np.isfinite(row_upper), np.abs(row_upper - row_values), np.inf) + basic_row_degeneracy = ( + basic_rows + & inequality_rows + & (np.minimum(lower_distance, upper_distance) <= float(primal_tolerance)) + ) + + lower_violation = float(np.maximum(reduced[lower], 0.0).max(initial=0.0)) + upper_violation = float(np.maximum(-reduced[upper], 0.0).max(initial=0.0)) + reconciled_objective = float(session.amount @ (values * session.objective)) + return { + "basis_valid": bool(basis.valid), + "basic_columns": int(basic.sum()), + "lower_nonbasic_columns": int(lower.sum()), + "upper_nonbasic_columns": int(upper.sum()), + "minimum_absolute_nonbasic_reduced_cost": float(absolute_reduced.min()), + "minimum_scaled_nonbasic_reduced_cost": float(scaled_reduced.min()), + "near_zero_nonbasic_reduced_costs": int(near_zero.sum()), + "primal_degenerate_basic_columns": int(basic_column_degeneracy.sum()), + "primal_degenerate_basic_rows": int(basic_row_degeneracy.sum()), + "basis_primal_degenerate": bool( + basic_column_degeneracy.any() or basic_row_degeneracy.any() + ), + "maximum_dual_sign_violation": max(lower_violation, upper_violation), + "objective_reconciliation_error": float(reconciled_objective - solution.objective_value), + } + + +def _append_caps( + target: dict[str, list[tuple[float, str]]], + rows: Iterable[tuple[str, float, str]], +) -> None: + for period, cap, source in rows: + target[str(period)].append((float(cap), str(source))) + + +def build_cap_census( + solve_records: pd.DataFrame, + comparator_support: pd.DataFrame, + frontier: pd.DataFrame, + *, + periods: Sequence[str], + broad_support: tuple[float, float], + tolerance: float, +) -> pd.DataFrame: + """Return the tolerance-deduplicated cap-month union with source flags.""" + period_set = {str(period) for period in periods} + if not period_set: + raise ValueError("Point-cap census requires at least one period.") + collected: dict[str, list[tuple[float, str]]] = defaultdict(list) + named = solve_records.loc[solve_records["comparator_rule"].isin(NAMED_SOURCE)].copy() + _append_caps( + collected, + ( + (str(row.period), float(row.frontier_cap), NAMED_SOURCE[str(row.comparator_rule)]) + for row in named.itertuples(index=False) + ), + ) + support_values = { + "development_support_lower": comparator_support["support_lower"].to_numpy(dtype=float), + "development_support_upper": comparator_support["support_upper"].to_numpy(dtype=float), + } + for period in sorted(period_set): + for source, values in support_values.items(): + _append_caps(collected, ((period, float(value), source) for value in values)) + _append_caps( + collected, + ( + (period, float(broad_support[0]), "broad_support_lower"), + (period, float(broad_support[1]), "broad_support_upper"), + ), + ) + enumerated = frontier.loc[frontier["is_enumerated_support_breakpoint"]] + _append_caps( + collected, + ( + (str(row.period), float(row.frontier_cap), "period_basis_breakpoint") + for row in enumerated.itertuples(index=False) + ), + ) + if set(collected) != period_set: + raise RuntimeError("Cap census periods do not match the primary monthly menus.") + + output: list[dict[str, Any]] = [] + sources = sorted( + {source for period_values in collected.values() for _, source in period_values} + ) + for period in sorted(collected): + ordered = sorted(collected[period]) + clusters: list[list[tuple[float, str]]] = [] + for item in ordered: + if not clusters or item[0] - clusters[-1][-1][0] > float(tolerance): + clusters.append([item]) + else: + clusters[-1].append(item) + for cluster in clusters: + caps = [item[0] for item in cluster] + observed_sources = {item[1] for item in cluster} + output.append( + { + "period": period, + "point_cap": float(sum(caps) / len(caps)), + "cluster_cap_min": float(min(caps)), + "cluster_cap_max": float(max(caps)), + "cap_sources": "|".join(sorted(observed_sources)), + **{f"is_{source}": source in observed_sources for source in sources}, + } + ) + result = pd.DataFrame(output).sort_values(["period", "point_cap"]).reset_index(drop=True) + if bool((result["cluster_cap_max"] - result["cluster_cap_min"] > tolerance).any()): + raise RuntimeError("Tolerance-deduplicated cap cluster exceeds its declared width.") + return result diff --git a/src/ijds_audit/portfolio.py b/src/ijds_audit/portfolio.py new file mode 100644 index 0000000..21f782d --- /dev/null +++ b/src/ijds_audit/portfolio.py @@ -0,0 +1,307 @@ +"""Exact-budget point-score LPs and comparator-frontier identities.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import highspy +import numpy as np +import pandas as pd +from scipy.sparse import csc_matrix + + +@dataclass(frozen=True) +class PointPortfolioSolution: + """A reconciled solution of the exact-budget point-score LP.""" + + allocation_fraction: np.ndarray + exposure: np.ndarray + objective_value: float + weighted_point_score: float + total_allocated: float + basis_cap_lower: float + basis_cap_upper: float + simplex_iterations: int + + +class PointPortfolioSession: + """A warm-started HiGHS model for repeated caps on one monthly menu.""" + + def __init__( + self, + frame: pd.DataFrame, + *, + point_score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + budget: float, + purpose_cap: float, + time_limit: int = 300, + threads: int = 1, + ) -> None: + n = len(frame) + if n == 0 or float(budget) <= 0.0: + raise ValueError("The point portfolio requires candidates and a positive budget.") + self.point = _validated_vector(point_score, name="point_score", n=n) + self.objective = _validated_vector(objective_rate, name="objective_rate", n=n) + if bool(np.any((self.point < 0.0) | (self.point > 1.0))): + raise ValueError("point_score must lie in [0, 1].") + if not 0.0 < float(purpose_cap) <= 1.0: + raise ValueError("purpose_cap must lie in (0, 1].") + self.budget = float(budget) + lp, self.risk_row, self.amount = _point_lp( + frame, + self.point, + self.objective, + budget=self.budget, + risk_cap=0.5, + purpose_cap=float(purpose_cap), + ) + self.solver = highspy.Highs() + if hasattr(self.solver, "resetGlobalScheduler"): + self.solver.resetGlobalScheduler(True) + self.solver.setOptionValue("output_flag", False) + self.solver.setOptionValue("log_to_console", False) + self.solver.setOptionValue("solver", "simplex") + self.solver.setOptionValue("presolve", "on") + self.solver.setOptionValue("time_limit", float(time_limit)) + self.solver.setOptionValue("threads", max(1, int(threads))) + if self.solver.passModel(lp) != highspy.HighsStatus.kOk: + raise RuntimeError("HiGHS rejected the exact-budget point LP.") + + def solve(self, risk_cap: float) -> PointPortfolioSolution: + """Change only the risk RHS and reoptimize from the current basis.""" + cap = float(risk_cap) + if not 0.0 <= cap <= 1.0: + raise ValueError("risk_cap must lie in [0, 1].") + status = self.solver.changeRowBounds( + int(self.risk_row), + -highspy.kHighsInf, + cap * self.budget, + ) + if status != highspy.HighsStatus.kOk: + raise RuntimeError("HiGHS rejected the point-risk RHS update.") + if self.solver.run() == highspy.HighsStatus.kError: + raise RuntimeError("HiGHS failed while solving the exact-budget point LP.") + model_status = self.solver.modelStatusToString(self.solver.getModelStatus()) + if "Optimal" not in str(model_status): + raise RuntimeError(f"Point LP is not optimal: {model_status}.") + fraction = np.clip(np.asarray(self.solver.getSolution().col_value, dtype=float), 0.0, 1.0) + exposure = self.amount * fraction + total = float(exposure.sum()) + if not np.isclose(total, self.budget, rtol=0.0, atol=1e-4): + raise RuntimeError(f"Point LP did not fill its budget: {total}.") + objective_value = float(exposure @ self.objective) + weighted_point = float(exposure @ self.point / total) + ranging_status, ranging = self.solver.getRanging() + if ranging_status != highspy.HighsStatus.kOk: + raise RuntimeError("HiGHS did not return basis ranging information.") + lower_rhs = float(ranging.row_bound_dn.value_[self.risk_row]) + upper_rhs = float(ranging.row_bound_up.value_[self.risk_row]) + info = self.solver.getInfo() + return PointPortfolioSolution( + allocation_fraction=fraction, + exposure=exposure, + objective_value=objective_value, + weighted_point_score=weighted_point, + total_allocated=total, + basis_cap_lower=max(0.0, lower_rhs / self.budget), + basis_cap_upper=min(1.0, upper_rhs / self.budget), + simplex_iterations=int(getattr(info, "simplex_iteration_count", 0) or 0), + ) + + def basis_breakpoints( + self, + *, + lower_cap: float, + upper_cap: float, + tolerance: float = 1e-10, + max_bases: int = 10_000, + ) -> tuple[float, ...]: + """Enumerate basis-range endpoints over a closed cap interval.""" + lower = float(lower_cap) + upper = float(upper_cap) + if not 0.0 <= lower < upper <= 1.0: + raise ValueError("Frontier cap support must be a nonempty subset of [0, 1].") + caps = {lower, upper} + probe = lower + for _ in range(int(max_bases)): + solution = self.solve(probe) + basis_lower = float(np.clip(solution.basis_cap_lower, lower, upper)) + basis_upper = float(np.clip(solution.basis_cap_upper, lower, upper)) + caps.update((basis_lower, basis_upper)) + if basis_upper >= upper - tolerance: + break + if ( + basis_upper <= probe + tolerance + and solution.weighted_point_score < probe - 0.5 * tolerance + ): + caps.add(upper) + break + next_probe = np.nextafter(basis_upper, np.inf) + minimum_step = max(float(tolerance), 1e-12) + if next_probe <= basis_upper + minimum_step: + next_probe = basis_upper + minimum_step + probe = min(next_probe, upper) + else: + raise RuntimeError(f"Frontier enumeration exceeded {max_bases} bases.") + ordered = sorted(cap for cap in caps if lower - tolerance <= cap <= upper + tolerance) + deduplicated: list[float] = [] + for cap in ordered: + if not deduplicated or cap - deduplicated[-1] > tolerance: + deduplicated.append(cap) + elif abs(cap - upper) <= tolerance: + deduplicated[-1] = upper + deduplicated[0] = lower + deduplicated[-1] = upper + return tuple(deduplicated) + + +def _validated_vector(values: Sequence[float] | np.ndarray, *, name: str, n: int) -> np.ndarray: + array = np.asarray(values, dtype=float) + if array.shape != (n,) or not bool(np.isfinite(array).all()): + raise ValueError(f"{name} must be a finite vector with {n} rows.") + return array + + +def _point_lp( + frame: pd.DataFrame, + point_score: np.ndarray, + objective_rate: np.ndarray, + *, + budget: float, + risk_cap: float, + purpose_cap: float, +) -> tuple[highspy.HighsLp, int, np.ndarray]: + n = len(frame) + amount = _validated_vector(frame["loan_amnt"], name="loan_amnt", n=n) + if bool(np.any(amount <= 0.0)): + raise ValueError("loan_amnt must be positive.") + rows = [amount] + lower = [float(budget)] + upper = [float(budget)] + risk_row = len(rows) + rows.append(amount * point_score) + lower.append(-highspy.kHighsInf) + upper.append(float(risk_cap) * float(budget)) + purposes = frame["purpose"].astype("string").fillna("unknown") + for purpose in sorted(purposes.unique()): + rows.append(amount * purposes.eq(purpose).to_numpy(dtype=float)) + lower.append(-highspy.kHighsInf) + upper.append(float(purpose_cap) * float(budget)) + matrix = csc_matrix(np.vstack(rows)) + lp = highspy.HighsLp() + lp.num_col_ = n + lp.num_row_ = len(rows) + lp.col_cost_ = (amount * objective_rate).tolist() + lp.col_lower_ = np.zeros(n).tolist() + lp.col_upper_ = np.ones(n).tolist() + lp.row_lower_ = lower + lp.row_upper_ = upper + lp.sense_ = highspy.ObjSense.kMaximize + lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise + lp.a_matrix_.num_col_ = n + lp.a_matrix_.num_row_ = len(rows) + lp.a_matrix_.start_ = matrix.indptr.astype(np.int32).tolist() + lp.a_matrix_.index_ = matrix.indices.astype(np.int32).tolist() + lp.a_matrix_.value_ = matrix.data.astype(float).tolist() + return lp, risk_row, amount + + +def solve_point_portfolio( + frame: pd.DataFrame, + *, + point_score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + budget: float, + risk_cap: float, + purpose_cap: float, + time_limit: int = 300, + threads: int = 1, +) -> PointPortfolioSolution: + """Solve the full-budget point LP and expose the current basis cap range.""" + session = PointPortfolioSession( + frame, + point_score=point_score, + objective_rate=objective_rate, + budget=float(budget), + purpose_cap=float(purpose_cap), + time_limit=time_limit, + threads=threads, + ) + return session.solve(float(risk_cap)) + + +def enumerate_basis_breakpoints( + frame: pd.DataFrame, + *, + point_score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + budget: float, + purpose_cap: float, + lower_cap: float, + upper_cap: float, + tolerance: float = 1e-10, + max_bases: int = 10_000, + time_limit: int = 300, + threads: int = 1, +) -> tuple[float, ...]: + """Enumerate HiGHS basis-range endpoints over a closed cap interval.""" + session = PointPortfolioSession( + frame, + point_score=point_score, + objective_rate=objective_rate, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ) + return session.basis_breakpoints( + lower_cap=lower_cap, + upper_cap=upper_cap, + tolerance=tolerance, + max_bases=max_bases, + ) + + +def c2_cap( + exposure: Sequence[float] | np.ndarray, point_score: Sequence[float] | np.ndarray +) -> float: + """Return the funded point-score moment defining comparator C2.""" + funded = np.asarray(exposure, dtype=float) + point = np.asarray(point_score, dtype=float) + if funded.shape != point.shape or funded.ndim != 1: + raise ValueError("C2 exposure and point-score vectors must align.") + total = float(funded.sum()) + if total <= 0.0: + raise ValueError("C2 requires positive funded exposure.") + return float(funded @ point / total) + + +def verify_c2_dominance( + *, + guardrail_exposure: Sequence[float] | np.ndarray, + point_solution: PointPortfolioSolution, + point_score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + tolerance: float = 1e-5, +) -> dict[str, float]: + """Reconcile C2 feasibility and point-objective weak dominance.""" + guard = np.asarray(guardrail_exposure, dtype=float) + point = np.asarray(point_score, dtype=float) + objective = np.asarray(objective_rate, dtype=float) + cap = c2_cap(guard, point) + point_residual = float(point_solution.weighted_point_score - cap) + guard_objective = float(guard @ objective) + difference = float(point_solution.objective_value - guard_objective) + if point_residual > 1e-10: + raise RuntimeError(f"C2 point comparator exceeds its cap by {point_residual:.3e}.") + if difference < -float(tolerance): + raise RuntimeError(f"C2 objective dominance failed by {difference:.6f} dollars.") + return { + "c2_cap": cap, + "c2_point_cap_residual": point_residual, + "guardrail_objective": guard_objective, + "point_objective": float(point_solution.objective_value), + "point_minus_guardrail_objective": difference, + } diff --git a/src/ijds_audit/prediction.py b/src/ijds_audit/prediction.py new file mode 100644 index 0000000..a8c638b --- /dev/null +++ b/src/ijds_audit/prediction.py @@ -0,0 +1,486 @@ +"""Shared prediction stack and complete residual-window construction.""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pandas as pd +from catboost import CatBoostClassifier +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler + +from src.data.outcome_observability import ( + audit_outcome_label_availability, + build_outcome_label_availability, + load_design_universe, + temporal_tail_split, + terminal_outcome_from_status, + validate_minimum_label_retention, +) +from src.evaluation.maturity_safe_portfolio import build_decision_panel +from src.features.feature_engineering import run_feature_pipeline +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, + fit_binary_outcome_recipe, +) +from src.models.maturity_safe_pd import ( + apply_platt_calibrator, + catboost_raw_margin, + classification_metrics, + fit_platt_calibrator, + validate_model_feature_contract, +) + +LABEL_FIT_SPLITS = ("pd_development", "probability_calibration", "conformal_fit") +DECISION_SPLITS = ("policy_development", "primary_oot", "censored_extension") + + +class ProtocolFeasibilityError(RuntimeError): + """A locked scientific requirement failed without authorizing adaptation.""" + + def __init__(self, message: str, *, details: Mapping[str, Any]) -> None: + super().__init__(message) + self.protocol_details = dict(details) + + +@dataclass(frozen=True) +class PreparedData: + """One status-independent universe and its model matrix.""" + + universe: pd.DataFrame + features: pd.DataFrame + numeric_features: tuple[str, ...] + categorical_features: tuple[str, ...] + source_inventory: dict[str, Any] + availability_audit: pd.DataFrame + monthly_residual_availability: pd.DataFrame + + +@dataclass(frozen=True) +class LearnerScores: + """Outcome-free scores and a learner-specific fixed 2011 taxonomy.""" + + name: str + model: Any + calibrator: LogisticRegression + probabilities: np.ndarray + taxonomy_edges: dict[int, tuple[float, ...]] + metrics: dict[str, Any] + + +@dataclass(frozen=True) +class WindowRecipe: + """One residual window and all closed taxonomy diagnostics.""" + + window_id: str + start: pd.Timestamp + end: pd.Timestamp + recipes: dict[int, BinaryOutcomeConformalRecipe] + fit_audit: pd.DataFrame + + +def available_binary_labels(frame: pd.DataFrame, *, block: str) -> np.ndarray: + """Return complete binary labels or fail with the named protocol block.""" + if not bool(frame["label_available"].all()): + raise RuntimeError(f"{block} contains unavailable labels.") + labels = frame["terminal_default"] + if bool(labels.isna().any()): + raise RuntimeError(f"{block} contains unresolved terminal outcomes.") + values = labels.astype(int).to_numpy(dtype=int) + if set(np.unique(values)) != {0, 1}: + raise RuntimeError(f"{block} must contain both outcome classes.") + return values + + +def _engineer_features( + universe: pd.DataFrame, config: Mapping[str, Any] +) -> tuple[pd.DataFrame, tuple[str, ...], tuple[str, ...]]: + numeric, categorical = validate_model_feature_contract(config["model"]) + source = universe.drop( + columns=[ + "loan_status", + "snapshot_default", + "snapshot_resolution", + "terminal_default", + "label_available", + "label_available_at", + "total_pymnt", + ], + errors="ignore", + ) + engineered = run_feature_pipeline(source) + output = pd.DataFrame(index=universe.index) + for feature in numeric: + values = engineered.get(feature, pd.Series(np.nan, index=universe.index)) + output[feature] = pd.to_numeric(values, errors="coerce").replace([np.inf, -np.inf], np.nan) + for feature in categorical: + values = engineered.get(feature, pd.Series("__MISSING__", index=universe.index)) + output[feature] = values.astype("string").fillna("__MISSING__").astype(str) + return output, tuple(numeric), tuple(categorical) + + +def _monthly_availability(universe: pd.DataFrame, config: Mapping[str, Any]) -> pd.DataFrame: + conformal = universe.loc[universe["design_split"].eq("conformal_fit")].copy() + conformal["issue_month"] = conformal["issue_d"].dt.to_period("M").astype(str) + audit = audit_outcome_label_availability( + conformal, + cutoff=str(config["source"]["information_cutoff"]), + charged_off_lag_months=int(config["source"]["charged_off_reporting_lag_months"]), + block_column="issue_month", + ) + validate_minimum_label_retention( + audit, + minimum_retention=float( + config["residual_specification"]["minimum_monthly_label_retention"] + ), + block_column="issue_month", + ) + return audit + + +def prepare_data(config: Mapping[str, Any], *, raw_path: Path) -> PreparedData: + """Load the status-independent archive and prepare all fixed features.""" + universe, source_inventory = load_design_universe( + config, + raw_path=raw_path, + label_required_splits=LABEL_FIT_SPLITS, + ) + labels = build_outcome_label_availability( + universe["loan_status"], + universe["last_pymnt_d"], + cutoff=str(config["source"]["information_cutoff"]), + charged_off_lag_months=int(config["source"]["charged_off_reporting_lag_months"]), + ) + universe["terminal_default"] = terminal_outcome_from_status(universe["loan_status"]) + universe["label_available"] = labels["label_available"].astype(bool) + universe["label_available_at"] = labels["label_available_at"] + availability = audit_outcome_label_availability( + universe.loc[universe["design_split"].isin(LABEL_FIT_SPLITS)], + cutoff=str(config["source"]["information_cutoff"]), + charged_off_lag_months=int(config["source"]["charged_off_reporting_lag_months"]), + ) + validate_minimum_label_retention( + availability, + minimum_retention=float(config["source"]["minimum_label_retention"]), + ) + monthly = _monthly_availability(universe, config) + features, numeric, categorical = _engineer_features(universe, config) + return PreparedData( + universe=universe, + features=features, + numeric_features=numeric, + categorical_features=categorical, + source_inventory=source_inventory, + availability_audit=availability, + monthly_residual_availability=monthly, + ) + + +def fixed_taxonomy_edges( + probabilities: np.ndarray, groups: Sequence[int] +) -> dict[int, tuple[float, ...]]: + """Freeze strictly increasing empirical score-quantile taxonomies.""" + output: dict[int, tuple[float, ...]] = {} + for group_count in groups: + edges = np.quantile( + probabilities, + np.linspace(0.0, 1.0, int(group_count) + 1), + method="linear", + ) + if bool(np.any(np.diff(edges) <= 0.0)): + raise RuntimeError(f"The fixed {group_count}-group taxonomy has repeated edges.") + output[int(group_count)] = tuple(float(value) for value in edges) + return output + + +def binary_probability_metrics(labels: np.ndarray, probabilities: np.ndarray) -> dict[str, float]: + """Report discrimination, calibration, and ten-bin ECE metrics.""" + metrics = classification_metrics(labels, probabilities) + bins = pd.cut(probabilities, bins=np.linspace(0.0, 1.0, 11), include_lowest=True) + grouped = ( + pd.DataFrame({"p": probabilities, "y": labels, "bin": bins}) + .groupby("bin", observed=True) + .agg(p=("p", "mean"), y=("y", "mean"), n=("y", "size")) + ) + metrics["ece_10"] = float( + np.sum(grouped["n"].to_numpy() * np.abs(grouped["p"] - grouped["y"])) / len(labels) + ) + return metrics + + +def fit_primary_scores(data: PreparedData, config: Mapping[str, Any]) -> LearnerScores: + """Fit the inherited CatBoost/Platt stack once and score the full universe.""" + universe = data.universe + development = universe.loc[ + universe["design_split"].eq("pd_development") & universe["label_available"] + ] + train, validation, validation_cutoff = temporal_tail_split( + development, + tail_fraction=float(config["design"]["validation_tail_fraction"]), + ) + parameters = dict(config["model"]["fixed_params"]) + parameters.update( + random_seed=int(config["model"]["canonical_seed"]), + thread_count=int(config["execution"]["threads"]), + ) + model = CatBoostClassifier(**parameters) + model.fit( + data.features.loc[train.index], + available_binary_labels(train, block="pd_development_train"), + cat_features=list(data.categorical_features), + ) + validation_labels = available_binary_labels(validation, block="pd_development_validation") + validation_probability = np.asarray( + model.predict_proba(data.features.loc[validation.index])[:, 1], dtype=float + ) + calibration_all = universe.loc[universe["design_split"].eq("probability_calibration")] + calibration_fit = calibration_all.loc[calibration_all["label_available"]] + calibration_margin = catboost_raw_margin(model, data.features.loc[calibration_fit.index]) + calibrator_config = copy.deepcopy(config["probability_calibration"]) + calibrator_config["logistic_regression"]["random_state"] = int( + config["model"]["canonical_seed"] + ) + calibrator = fit_platt_calibrator( + calibration_margin, + available_binary_labels(calibration_fit, block="probability_calibration"), + calibrator_config, + ) + all_probability = apply_platt_calibrator( + calibrator, + catboost_raw_margin(model, data.features), + ) + calibration_probability = all_probability[calibration_all.index] + groups = [int(value) for value in config["conformal"]["diagnostic_group_counts"]] + return LearnerScores( + name="catboost_platt", + model=model, + calibrator=calibrator, + probabilities=all_probability, + taxonomy_edges=fixed_taxonomy_edges(calibration_probability, groups), + metrics={ + "validation_cutoff": str(validation_cutoff.to_period("M")), + "validation": binary_probability_metrics(validation_labels, validation_probability), + "probability_calibration": binary_probability_metrics( + available_binary_labels(calibration_fit, block="probability_calibration"), + all_probability[calibration_fit.index], + ), + }, + ) + + +def fit_logistic_control(data: PreparedData, config: Mapping[str, Any]) -> LearnerScores: + """Fit the independently calibrated numeric logistic coverage control.""" + universe = data.universe + development = universe.loc[ + universe["design_split"].eq("pd_development") & universe["label_available"] + ] + train, validation, validation_cutoff = temporal_tail_split( + development, + tail_fraction=float(config["design"]["validation_tail_fraction"]), + ) + control = config["learner_control"]["logistic_regression"] + numeric = list(data.numeric_features) + pipeline = Pipeline( + [ + ( + "preprocess", + ColumnTransformer( + [ + ( + "numeric", + Pipeline( + [ + ("imputer", SimpleImputer(strategy="median")), + ("scaler", StandardScaler()), + ] + ), + numeric, + ) + ], + remainder="drop", + ), + ), + ( + "model", + LogisticRegression( + C=float(control["C"]), + class_weight=str(control["class_weight"]), + solver=str(control["solver"]), + max_iter=int(control["max_iter"]), + random_state=int(control["random_state"]), + ), + ), + ] + ) + pipeline.fit( + data.features.loc[train.index, numeric], + available_binary_labels(train, block="logit_train"), + ) + validation_margin = np.asarray( + pipeline.decision_function(data.features.loc[validation.index, numeric]), dtype=float + ) + validation_probability = np.asarray( + pipeline.predict_proba(data.features.loc[validation.index, numeric])[:, 1], dtype=float + ) + calibration_all = universe.loc[universe["design_split"].eq("probability_calibration")] + calibration_fit = calibration_all.loc[calibration_all["label_available"]] + calibration_margin = np.asarray( + pipeline.decision_function(data.features.loc[calibration_fit.index, numeric]), dtype=float + ) + calibrator_config = copy.deepcopy(config["probability_calibration"]) + calibrator_config["logistic_regression"]["random_state"] = int(control["random_state"]) + calibrator = fit_platt_calibrator( + calibration_margin, + available_binary_labels(calibration_fit, block="logit_probability_calibration"), + calibrator_config, + ) + all_margin = np.asarray(pipeline.decision_function(data.features[numeric]), dtype=float) + all_probability = apply_platt_calibrator(calibrator, all_margin) + calibration_probability = all_probability[calibration_all.index] + groups = [int(value) for value in config["conformal"]["diagnostic_group_counts"]] + return LearnerScores( + name="numeric_logistic_platt", + model=pipeline, + calibrator=calibrator, + probabilities=all_probability, + taxonomy_edges=fixed_taxonomy_edges(calibration_probability, groups), + metrics={ + "validation_cutoff": str(validation_cutoff.to_period("M")), + "validation_uncalibrated": binary_probability_metrics( + available_binary_labels(validation, block="logit_validation"), + validation_probability, + ), + "validation_margin_mean": float(np.mean(validation_margin)), + "probability_calibration": binary_probability_metrics( + available_binary_labels(calibration_fit, block="logit_probability_calibration"), + all_probability[calibration_fit.index], + ), + }, + ) + + +def fit_window_recipes( + data: PreparedData, + scores: LearnerScores, + config: Mapping[str, Any], +) -> dict[str, WindowRecipe]: + """Fit every declared residual window under every closed taxonomy size.""" + universe = data.universe + output: dict[str, WindowRecipe] = {} + group_counts = [int(value) for value in config["conformal"]["diagnostic_group_counts"]] + alpha = float(config["conformal"]["alpha"]) + calibration_start = pd.Timestamp(config["design"]["probability_calibration_start"]) + calibration_end = pd.Timestamp(config["design"]["probability_calibration_end"]) + taxonomy_provenance = ( + f"{scores.name}_{calibration_start:%Y%m}_{calibration_end:%Y%m}" + "_all_status_independent_scores" + ) + for specification in config["residual_specification"]["windows"]: + identifier = str(specification["id"]) + start = pd.Timestamp(specification["start"]) + end = pd.Timestamp(specification["end"]) + if pd.isna(start) or pd.isna(end): + raise ValueError(f"Residual window {identifier} has an invalid boundary.") + start = cast(pd.Timestamp, start) + end = cast(pd.Timestamp, end) + mask = ( + universe["design_split"].eq("conformal_fit") + & universe["issue_d"].between(start, end) + & universe["label_available"] + ) + frame = universe.loc[mask] + labels = available_binary_labels(frame, block=f"{scores.name}_{identifier}") + probability = scores.probabilities[frame.index] + recipes: dict[int, BinaryOutcomeConformalRecipe] = {} + fit_rows: list[pd.DataFrame] = [] + for groups in group_counts: + recipe = fit_binary_outcome_recipe( + probability, + labels, + alpha=alpha, + n_groups=groups, + bin_edges=scores.taxonomy_edges[groups], + taxonomy_provenance=taxonomy_provenance, + taxonomy_method="fixed_empirical_linear_score_quantiles", + method="fixed_taxonomy_split_mondrian_absolute_residual", + ) + minimum = int(config["conformal"]["minimum_rows_per_group"]) + if ( + groups == int(config["conformal"]["canonical_groups"]) + and min(recipe.group_counts) < minimum + ): + counts = tuple(int(value) for value in recipe.group_counts) + raise ProtocolFeasibilityError( + f"{identifier} has canonical residual group counts {counts}; " + f"the locked minimum is {minimum}.", + details={ + "stage": "canonical_residual_group_size", + "learner": scores.name, + "window_id": identifier, + "taxonomy_groups": groups, + "group_counts": list(counts), + "minimum_rows_per_group": minimum, + }, + ) + assigned, lower, upper = apply_binary_outcome_recipe(probability, recipe) + recipes[groups] = recipe + fit_rows.append( + pd.DataFrame( + { + "id": frame["id"].astype("string").to_numpy(), + "issue_d": frame["issue_d"].to_numpy(), + "learner": scores.name, + "window_id": identifier, + "taxonomy_groups": groups, + "conformal_group": assigned, + "pd_point": probability, + "conformal_lower": lower, + "conformal_upper": upper, + "terminal_default": labels, + "covered": (labels >= lower) & (labels <= upper), + } + ) + ) + output[identifier] = WindowRecipe( + window_id=identifier, + start=start, + end=end, + recipes=recipes, + fit_audit=pd.concat(fit_rows, ignore_index=True), + ) + return output + + +def decision_panel_for_window( + data: PreparedData, + scores: LearnerScores, + window: WindowRecipe, + *, + groups: int = 5, +) -> pd.DataFrame: + """Build one outcome-free panel for a frozen learner/window recipe.""" + universe = data.universe + source = universe.loc[universe["design_split"].isin(DECISION_SPLITS)] + probability = scores.probabilities[source.index] + assigned, lower, upper = apply_binary_outcome_recipe(probability, window.recipes[int(groups)]) + panel = build_decision_panel( + source, + pd_point=probability, + conformal_lower=lower, + conformal_upper=upper, + conformal_groups=assigned, + ) + panel["design_split"] = source["design_split"].astype(str).to_numpy() + panel["learner"] = scores.name + panel["window_id"] = window.window_id + panel["taxonomy_groups"] = int(groups) + return panel diff --git a/src/ijds_audit/protocol.py b/src/ijds_audit/protocol.py new file mode 100644 index 0000000..83e8ca0 --- /dev/null +++ b/src/ijds_audit/protocol.py @@ -0,0 +1,671 @@ +"""Two-phase orchestration for the active V4 retrospective audit.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from src.data.outcome_observability import ( + build_outcome_label_availability, + load_design_universe, + terminal_outcome_from_status, +) +from src.ijds_audit.allocations import build_outcome_free_portfolios, policy_family +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.endpoint_recovery import reconcile_from_json_reference +from src.ijds_audit.evaluation import ( + aggregate_portfolios, + build_archive_outcomes, + comparator_envelopes, + endpoint_resolution_audit, + evaluate_frozen_portfolios, + paired_portfolio_contrasts, + temporal_coverage_audit, +) +from src.ijds_audit.geometry import summarize_binary_geometry +from src.ijds_audit.prediction import ( + LABEL_FIT_SPLITS, + LearnerScores, + WindowRecipe, + decision_panel_for_window, + fit_logistic_control, + fit_primary_scores, + fit_window_recipes, + prepare_data, +) +from src.ijds_audit.simulation import run_factorial_simulation +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, +) +from src.utils.isolated_experiment import ( + environment_provenance, + implementation_provenance, + prepare_output_paths, + relative_artifact_descriptor, + require_clean_tagged_head, + resolve_isolated_run_dir, + resolve_repo_input, + save_catboost_model_atomic, +) +from src.utils.pipeline_runtime import ( + atomic_write_json, + atomic_write_parquet, + atomic_write_pickle, +) + +ALLOWED_DATA_ROOT = Path("data/processed/experiments/ijds_audit") +ALLOWED_MODEL_ROOT = Path("models/experiments/ijds_audit") + + +def recipe_payload( + learners: Mapping[str, Mapping[str, WindowRecipe]], +) -> dict[str, dict[str, dict[str, Any]]]: + """Serialize learner/window recipes without estimator objects.""" + return { + learner: { + window_id: {str(groups): asdict(recipe) for groups, recipe in window.recipes.items()} + for window_id, window in windows.items() + } + for learner, windows in learners.items() + } + + +def _recipe_from_payload(payload: Mapping[str, Any]) -> BinaryOutcomeConformalRecipe: + values = dict(payload) + for field in ( + "bin_edges", + "residual_quantiles", + "group_counts", + "finite_sample_ranks", + "raw_finite_sample_ranks", + ): + values[field] = tuple(values[field]) + return BinaryOutcomeConformalRecipe(**values) + + +def load_recipes( + path: Path, +) -> dict[str, dict[str, dict[int, BinaryOutcomeConformalRecipe]]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("V4 recipe artifact must be a JSON mapping.") + return { + str(learner): { + str(window_id): { + int(groups): _recipe_from_payload(recipe) + for groups, recipe in group_recipes.items() + } + for window_id, group_recipes in windows.items() + } + for learner, windows in payload.items() + } + + +def score_frame(data: Any, learners: Sequence[LearnerScores]) -> pd.DataFrame: + """Build the common outcome-free score artifact for any learner family.""" + frame = pd.DataFrame( + { + "id": data.universe["id"].astype("string"), + "issue_d": data.universe["issue_d"], + "design_split": data.universe["design_split"].astype("string"), + } + ) + for learner in learners: + frame[f"pd_{learner.name}"] = learner.probabilities + return frame + + +def outcome_free_geometry( + scores: pd.DataFrame, + recipes: Mapping[str, Mapping[str, WindowRecipe]], +) -> pd.DataFrame: + """Summarize interval geometry before joining evaluation outcomes.""" + rows: list[dict[str, Any]] = [] + for learner, windows in recipes.items(): + probability = scores[f"pd_{learner}"].to_numpy(dtype=float) + for window_id, window in windows.items(): + for groups, recipe in sorted(window.recipes.items()): + assigned, lower, upper = apply_binary_outcome_recipe(probability, recipe) + for role in ( + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", + ): + role_mask = scores["design_split"].eq(role).to_numpy(dtype=bool) + for stratum in (-1, *range(groups)): + mask = role_mask & ((assigned == stratum) if stratum >= 0 else True) + if not bool(mask.any()): + raise RuntimeError( + f"Empty geometry cell: {learner}/{window_id}/{groups}/{role}/{stratum}." + ) + rows.append( + { + "learner": learner, + "window_id": window_id, + "taxonomy_groups": groups, + "role": role, + "conformal_group": stratum, + "score_min": float(np.min(probability[mask])), + "score_max": float(np.max(probability[mask])), + **summarize_binary_geometry(lower[mask], upper[mask]), + } + ) + return pd.DataFrame(rows) + + +def verified_freeze_artifact_paths( + freeze: Mapping[str, Any], *, repo_root: Path +) -> dict[str, Path]: + paths: dict[str, Path] = {} + for name, descriptor in freeze["outcome_free_artifacts"].items(): + path = (repo_root / str(descriptor["path"])).resolve() + path.relative_to(repo_root) + if not path.is_file(): + raise FileNotFoundError(path) + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Frozen artifact mismatch for {name}: {field}.") + paths[str(name)] = path + return paths + + +def _implementation(config_path: Path, repo_root: Path) -> dict[str, Any]: + return implementation_provenance( + config_path=config_path, + repo_root=repo_root, + relative_paths=[ + Path("scripts/experiments/run_ijds_binary_geometry_frontier_v4.py"), + Path("src/ijds_audit/config.py"), + Path("src/ijds_audit/geometry.py"), + Path("src/ijds_audit/prediction.py"), + Path("src/ijds_audit/portfolio.py"), + Path("src/ijds_audit/allocations.py"), + Path("src/ijds_audit/evaluation.py"), + Path("src/ijds_audit/endpoint_recovery.py"), + Path("src/ijds_audit/simulation.py"), + Path("src/ijds_audit/protocol.py"), + Path("src/data/outcome_observability.py"), + Path("src/models/binary_conformal_guardrail.py"), + Path("src/models/maturity_safe_pd.py"), + Path("src/evaluation/maturity_safe_portfolio.py"), + Path("src/evaluation/policy_contrast_bounds.py"), + Path("src/evaluation/standardized_credit_payoff.py"), + Path("src/optimization/portfolio_model.py"), + *[ + Path(value) + for value in load_v4_config(config_path).get("protocol_lineage_files", []) + ], + ], + ) + + +def freeze_outcome_free( + *, + config_path: Path, + repo_root: Path, +) -> Path: + """Fit, solve, persist, and hash all objects before the archive outcome join.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_v4_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + resume = config.get("resume_outcome_free") + if resume: + source_model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(resume["source_run_tag"]), + ) + source_freeze_path = source_model_dir / "protocol_freeze.json" + if not source_freeze_path.is_file(): + raise FileNotFoundError(source_freeze_path) + source_descriptor = relative_artifact_descriptor(source_freeze_path, repo_root=root) + if source_descriptor["sha256"] != str(resume["source_freeze_sha256"]): + raise RuntimeError("Imported outcome-free freeze SHA-256 mismatch.") + source_freeze = json.loads(source_freeze_path.read_text(encoding="utf-8")) + source_expected = { + "status": "outcome_free_allocations_frozen_before_archive_outcome_join", + "run_tag": str(resume["source_run_tag"]), + "protocol_tag": str(resume["source_protocol_tag"]), + "protocol_commit": str(resume["source_protocol_commit"]), + } + for field, value in source_expected.items(): + if source_freeze.get(field) != value: + raise RuntimeError(f"Imported outcome-free freeze mismatch for {field}.") + if source_freeze.get("outcome_columns_passed_to_policy_or_comparator") != []: + raise RuntimeError("Imported freeze reports outcome leakage.") + verified_freeze_artifact_paths(source_freeze, repo_root=root) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + imported = { + "schema_version": str(config["schema_version"]), + "status": "verified_outcome_free_freeze_imported_before_archive_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "source_inventory": source_freeze["source_inventory"], + "learner_metrics": source_freeze["learner_metrics"], + "outcome_columns_passed_to_policy_or_comparator": [], + "policy_selection": "none_all_nine_co_primary", + "window_selection": "none_all_eight_co_primary", + "implementation_provenance": _implementation(resolved_config, root), + "environment": environment_provenance(root), + "outcome_free_artifacts": source_freeze["outcome_free_artifacts"], + "model_artifacts": source_freeze["model_artifacts"], + "outcome_free_lineage": { + "source_protocol_freeze": source_descriptor, + **source_expected, + "reuse_scope": "verified_outcome_free_v4_v1_artifacts_only", + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", imported) + paths = prepare_output_paths( + config, + repo_root=root, + allowed_data_root=ALLOWED_DATA_ROOT, + allowed_model_root=ALLOWED_MODEL_ROOT, + ) + raw_path = resolve_repo_input(config["source"]["raw_path"], repo_root=root) + data = prepare_data(config, raw_path=raw_path) + primary = fit_primary_scores(data, config) + logistic = fit_logistic_control(data, config) + learner_scores = (primary, logistic) + windows = { + primary.name: fit_window_recipes(data, primary, config), + logistic.name: fit_window_recipes(data, logistic, config), + } + primary_panels = { + window_id: decision_panel_for_window(data, primary, recipe) + for window_id, recipe in windows[primary.name].items() + } + portfolio = build_outcome_free_portfolios(primary_panels, config) + scores = score_frame(data, learner_scores) + geometry = outcome_free_geometry(scores, windows) + fit_audits = pd.concat( + [window.fit_audit for learner in windows.values() for window in learner.values()], + ignore_index=True, + ) + + artifact_files = { + "scores": atomic_write_parquet(scores, paths.data_dir / "prediction/scores.parquet"), + "recipes": atomic_write_json( + paths.model_dir / "prediction/residual_recipes.json", recipe_payload(windows) + ), + "fit_audit": atomic_write_parquet( + fit_audits, paths.data_dir / "prediction/residual_fit_audit.parquet" + ), + "outcome_free_geometry": atomic_write_parquet( + geometry, paths.data_dir / "prediction/outcome_free_geometry.parquet" + ), + "availability_audit": atomic_write_parquet( + data.availability_audit, paths.data_dir / "data/label_availability_audit.parquet" + ), + "monthly_residual_availability": atomic_write_parquet( + data.monthly_residual_availability, + paths.data_dir / "data/monthly_residual_availability.parquet", + ), + "solve_records": atomic_write_parquet( + portfolio.records, paths.data_dir / "portfolio/outcome_free_solve_records.parquet" + ), + "allocations": atomic_write_parquet( + portfolio.allocations, + paths.data_dir / "portfolio/outcome_free_funded_allocations.parquet", + ), + "comparator_support": atomic_write_parquet( + portfolio.comparator_support, + paths.data_dir / "portfolio/development_comparator_support.parquet", + ), + "frontier_breakpoints": atomic_write_parquet( + portfolio.frontier_breakpoints, + paths.data_dir / "portfolio/exact_frontier_breakpoints.parquet", + ), + } + primary_model = save_catboost_model_atomic( + primary.model, paths.model_dir / "prediction/catboost_seed42.cbm" + ) + primary_calibrator = atomic_write_pickle( + paths.model_dir / "prediction/catboost_platt.pkl", primary.calibrator + ) + logistic_model = atomic_write_pickle( + paths.model_dir / "prediction/numeric_logistic.pkl", logistic.model + ) + logistic_calibrator = atomic_write_pickle( + paths.model_dir / "prediction/numeric_logistic_platt.pkl", logistic.calibrator + ) + model_artifacts = { + "catboost": relative_artifact_descriptor(primary_model, repo_root=root), + "catboost_platt": relative_artifact_descriptor(primary_calibrator, repo_root=root), + "numeric_logistic": relative_artifact_descriptor(logistic_model, repo_root=root), + "numeric_logistic_platt": relative_artifact_descriptor(logistic_calibrator, repo_root=root), + } + freeze = { + "schema_version": str(config["schema_version"]), + "status": "outcome_free_allocations_frozen_before_archive_outcome_join", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "source_inventory": data.source_inventory, + "learner_metrics": {learner.name: learner.metrics for learner in learner_scores}, + "outcome_columns_passed_to_policy_or_comparator": [], + "policy_selection": "none_all_nine_co_primary", + "window_selection": "none_all_eight_co_primary", + "implementation_provenance": _implementation(resolved_config, root), + "environment": environment_provenance(root), + "outcome_free_artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in artifact_files.items() + }, + "model_artifacts": model_artifacts, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + return atomic_write_json(paths.model_dir / "protocol_freeze.json", freeze) + + +def load_outcome_universe(config: Mapping[str, Any], *, raw_path: Path) -> pd.DataFrame: + universe, _ = load_design_universe( + config, + raw_path=raw_path, + label_required_splits=LABEL_FIT_SPLITS, + ) + labels = build_outcome_label_availability( + universe["loan_status"], + universe["last_pymnt_d"], + cutoff=str(config["source"]["information_cutoff"]), + charged_off_lag_months=int(config["source"]["charged_off_reporting_lag_months"]), + ) + universe["terminal_default"] = terminal_outcome_from_status(universe["loan_status"]) + universe["label_available"] = labels["label_available"].astype(bool) + universe["label_available_at"] = labels["label_available_at"] + return universe + + +def configured_archive_outcomes( + universe: pd.DataFrame, + config: Mapping[str, Any], +) -> pd.DataFrame: + """Apply the declared endpoint without exposing outcomes to selection.""" + contract = config["target"].get("evaluation_outcome_contract") + cutoff = None if contract is None else str(contract["cutoff"]) + return build_archive_outcomes(universe, evaluation_cutoff=cutoff) + + +def expand_frontier_for_window( + shared: pd.DataFrame, + scores: pd.DataFrame, + recipe: BinaryOutcomeConformalRecipe, + *, + window_id: str, +) -> pd.DataFrame: + primary = scores.loc[scores["design_split"].eq("primary_oot")] + probability = primary["pd_catboost_platt"].to_numpy(dtype=float) + assigned, lower, upper = apply_binary_outcome_recipe(probability, recipe) + endpoints = pd.DataFrame( + { + "id": primary["id"].astype("string"), + "conformal_lower": lower, + "conformal_upper": upper, + "conformal_group": assigned, + } + ) + placeholders = [ + "conformal_lower", + "conformal_upper", + "conformal_group", + "learner", + "taxonomy_groups", + ] + expanded = shared.drop(columns=placeholders, errors="ignore").merge( + endpoints, on="id", how="left", validate="many_to_one" + ) + if bool(expanded["conformal_lower"].isna().any()): + raise RuntimeError(f"Shared frontier could not be aligned to window {window_id}.") + expanded["window_id"] = str(window_id) + expanded["learner"] = "catboost_platt" + expanded["taxonomy_groups"] = 5 + return expanded + + +def evaluate_frozen( + *, + config_path: Path, + repo_root: Path, +) -> Path: + """Verify the freeze, join archive outcomes once, and build V4 evidence.""" + root = repo_root.resolve() + resolved_config = resolve_repo_input(config_path, repo_root=root) + config = load_v4_config(resolved_config) + protocol_commit = require_clean_tagged_head(root, str(config["protocol_tag"])) + data_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["data_root"]), + allowed_relative_root=ALLOWED_DATA_ROOT, + run_tag=str(config["run_tag"]), + ) + model_dir = resolve_isolated_run_dir( + repo_root=root, + configured_root=str(config["output"]["model_root"]), + allowed_relative_root=ALLOWED_MODEL_ROOT, + run_tag=str(config["run_tag"]), + ) + freeze_path = model_dir / "protocol_freeze.json" + if not freeze_path.is_file(): + raise FileNotFoundError("Run the outcome-free freeze phase first.") + summary_path = model_dir / str(config["output"]["deterministic_summary"]) + if summary_path.exists() or (data_dir / "evaluation").exists(): + raise FileExistsError("V4 evaluation already exists; experiment outputs are immutable.") + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + expected_status = ( + "verified_outcome_free_freeze_imported_before_archive_outcome_join" + if config.get("resume_outcome_free") + else "outcome_free_allocations_frozen_before_archive_outcome_join" + ) + expected = { + "status": expected_status, + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + } + for field, value in expected.items(): + if freeze.get(field) != value: + raise RuntimeError(f"Protocol freeze mismatch for {field}.") + if freeze.get("outcome_columns_passed_to_policy_or_comparator") != []: + raise RuntimeError("Protocol freeze reports outcome leakage.") + artifacts = verified_freeze_artifact_paths(freeze, repo_root=root) + raw_path = resolve_repo_input(config["source"]["raw_path"], repo_root=root) + universe = load_outcome_universe(config, raw_path=raw_path) + outcomes = configured_archive_outcomes(universe, config) + endpoint_audit = endpoint_resolution_audit(outcomes) + scores = pd.read_parquet(artifacts["scores"]) + recipes = load_recipes(artifacts["recipes"]) + fit_audit = pd.read_parquet(artifacts["fit_audit"]) + records = pd.read_parquet(artifacts["solve_records"]) + allocations = pd.read_parquet(artifacts["allocations"]) + support = pd.read_parquet(artifacts["comparator_support"]) + + coverage = temporal_coverage_audit(scores, outcomes, recipes, fit_audit) + frontier_mask = records["comparator_rule"].eq("point_cap_frontier") + shared_records = records.loc[frontier_mask] + named_records = records.loc[~frontier_mask] + shared_allocations = allocations.loc[allocations["comparator_rule"].eq("point_cap_frontier")] + named_allocations = allocations.loc[~allocations["comparator_rule"].eq("point_cap_frontier")] + if not bool(shared_records["window_id"].eq("__shared_point_frontier__").all()): + raise RuntimeError("Point frontier is not stored under its shared freeze identity.") + evaluated, joined = evaluate_frozen_portfolios( + named_records, named_allocations, outcomes, config=config + ) + aggregates = aggregate_portfolios(evaluated) + policy_ids = tuple(candidate.candidate_id for candidate in policy_family(config)) + shared_joined = shared_allocations.merge( + outcomes[["id", "snapshot_default", "snapshot_resolution"]], + on="id", + how="left", + validate="many_to_one", + ) + if bool(shared_joined["snapshot_resolution"].isna().any()): + raise RuntimeError("Shared frontier outcome join is incomplete.") + contrast_frames: list[pd.DataFrame] = [] + primary_recipes = recipes["catboost_platt"] + for window_id, group_recipes in primary_recipes.items(): + expanded_frontier = expand_frontier_for_window( + shared_joined, + scores, + group_recipes[5], + window_id=window_id, + ) + window_allocations = pd.concat( + [joined.loc[joined["window_id"].eq(window_id)], expanded_frontier], + ignore_index=True, + ) + contrast_frames.append( + paired_portfolio_contrasts( + window_allocations, + policy_ids=policy_ids, + lgd=float(config["payoff"]["lgd"]), + ) + ) + contrasts = pd.concat(contrast_frames, ignore_index=True) + frontier = config["comparators"]["exact_point_cap_frontier"] + envelopes = comparator_envelopes( + contrasts, + support, + broad_lower=float(frontier["start"]), + broad_upper=float(frontier["stop"]), + ) + simulation, simulation_summary = run_factorial_simulation(config) + + recovery = config.get("endpoint_reason_recovery") + recovery_audit = None + if recovery: + recovery_audit = reconcile_from_json_reference( + { + "temporal_coverage": coverage, + "monthly_evaluation": evaluated, + "aggregate_evaluation": aggregates, + "paired_contrasts": contrasts, + "comparator_envelopes": envelopes, + "simulation_repetitions": simulation, + "simulation_summary": simulation_summary, + }, + reference_json=recovery["reference_json"], + artifact_section=str(recovery["artifact_section"]), + repo_root=root, + float_atol=float(recovery.get("float_atol", 0.0)), + float_rtol=float(recovery.get("float_rtol", 0.0)), + ) + + evaluation_files = { + "temporal_coverage": atomic_write_parquet( + coverage, data_dir / "evaluation/temporal_coverage.parquet" + ), + "monthly_evaluation": atomic_write_parquet( + evaluated, data_dir / "evaluation/monthly_evaluation.parquet" + ), + "funded_allocations_with_outcomes": atomic_write_parquet( + joined, data_dir / "evaluation/funded_allocations_with_outcomes.parquet" + ), + "shared_frontier_allocations_with_outcomes": atomic_write_parquet( + shared_joined, + data_dir / "evaluation/shared_frontier_allocations_with_outcomes.parquet", + ), + "aggregate_evaluation": atomic_write_parquet( + aggregates, data_dir / "evaluation/aggregate_evaluation.parquet" + ), + "paired_contrasts": atomic_write_parquet( + contrasts, data_dir / "evaluation/paired_sharp_contrasts.parquet" + ), + "comparator_envelopes": atomic_write_parquet( + envelopes, data_dir / "evaluation/comparator_envelopes.parquet" + ), + "simulation_repetitions": atomic_write_parquet( + simulation, data_dir / "simulation/factorial_repetitions.parquet" + ), + "simulation_summary": atomic_write_parquet( + simulation_summary, data_dir / "simulation/factorial_summary.parquet" + ), + "endpoint_resolution_audit": atomic_write_parquet( + endpoint_audit, data_dir / "evaluation/endpoint_resolution_audit.parquet" + ), + } + canonical_coverage = coverage.loc[ + coverage["learner"].eq("catboost_platt") + & coverage["taxonomy_groups"].eq(5) + & coverage["role"].eq("primary_oot") + & coverage["conformal_group"].eq(-1) + ] + envelope_counts: list[dict[str, Any]] = [] + for raw_keys, frame in envelopes.groupby( + ["scope", "metric", "direction"], observed=True, sort=True + ): + key_values = raw_keys if isinstance(raw_keys, tuple) else (raw_keys,) + if len(key_values) != 3: + raise RuntimeError("Envelope summary key has unexpected cardinality.") + scope, metric, direction = key_values + envelope_counts.append( + { + "scope": str(scope), + "metric": str(metric), + "direction": str(direction), + "cells": int(len(frame)), + } + ) + summary = { + "schema_version": str(config["schema_version"]), + "status": "complete_retrospective_binary_geometry_frontier_audit", + "run_tag": str(config["run_tag"]), + "protocol_tag": str(config["protocol_tag"]), + "protocol_commit": protocol_commit, + "claim_boundary": { + "previously_inspected_archive": True, + "confirmatory": False, + "prospective": False, + "causal": False, + "selected_set_validity": False, + "policy_winner": False, + "nested_scopes_are_independent_replications": False, + }, + "canonical_primary_oot_coverage": canonical_coverage.to_dict(orient="records"), + "comparator_envelope_direction_counts": envelope_counts, + "c2_objective_dominance_minimum": float( + records.loc[ + records["comparator_rule"].eq("c2_contemporaneous"), + "point_minus_guardrail_objective", + ].min() + ), + "simulation_scope": "synthetic_mechanism_interpretation_only", + "endpoint_resolution_audit": endpoint_audit.to_dict(orient="records"), + "endpoint_reason_recovery": recovery_audit, + "outcome_free_freeze": relative_artifact_descriptor(freeze_path, repo_root=root), + "artifacts": { + name: relative_artifact_descriptor(path, repo_root=root) + for name, path in evaluation_files.items() + }, + "protected_stages_run": [], + "protected_artifacts_written": [], + } + written_summary = atomic_write_json(summary_path, summary) + atomic_write_json( + model_dir / str(config["output"]["execution_receipt"]), + { + "summary": relative_artifact_descriptor(written_summary, repo_root=root), + "protocol_commit": protocol_commit, + "environment": environment_provenance(root), + }, + ) + return written_summary diff --git a/src/ijds_audit/publication_generation.py b/src/ijds_audit/publication_generation.py new file mode 100644 index 0000000..d57702f --- /dev/null +++ b/src/ijds_audit/publication_generation.py @@ -0,0 +1,175 @@ +"""Transactional publication outputs and implementation provenance.""" + +from __future__ import annotations + +import os +import shutil +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from src.utils.artifact_descriptor import relative_artifact_descriptor, sha256_file + +PUBLICATION_IMPLEMENTATION_PATHS: dict[str, str] = { + "active_source_registry": "configs/ijds_active_evidence_sources.yaml", + "claim_ledger_contract": "configs/ijds_claim_ledger.yaml", + "publication_targets_contract": "configs/crpto_publication_targets.yaml", + "evidence_builder": "scripts/build_ijds_binary_geometry_frontier_v4_evidence.py", + "publication_integrity_checker": "scripts/check_publication_integrity.py", + "paper_pdf_auditor": "scripts/inspect_ijds_pdfs.py", + "publication_generation_helper": "src/ijds_audit/publication_generation.py", + "v4_config_loader": "src/ijds_audit/config.py", + "grid_contracts": "src/ijds_audit/grid_contracts.py", + "endpoint_availability_sensitivity/loader": "src/ijds_audit/sensitivity_evidence.py", + "portfolio_structure_sensitivity/loader": "src/ijds_audit/structural_evidence.py", + "robustness_sensitivities/loader": "src/ijds_audit/robustness_evidence.py", + "claim_ledger_loader": "src/ijds_audit/claim_ledger.py", + "source_registry_loader": "src/ijds_audit/publication_sources.py", + "artifact_descriptor_helper": "src/utils/artifact_descriptor.py", + "pipeline_runtime_helper": "src/utils/pipeline_runtime.py", +} + + +def publication_implementation_descriptors(repo_root: Path) -> dict[str, dict[str, Any]]: + """Hash the complete code and contract surface that accepts publication evidence.""" + return { + name: relative_artifact_descriptor(repo_root / relative_path, repo_root=repo_root) + for name, relative_path in PUBLICATION_IMPLEMENTATION_PATHS.items() + } + + +def staged_output_path( + transaction_root: Path, + target: Path, + *, + repo_root: Path, +) -> Path: + """Return a staging path that mirrors a repository-contained final target.""" + relative = target.resolve().relative_to(repo_root.resolve()) + staged = transaction_root.resolve() / "outputs" / relative + staged.parent.mkdir(parents=True, exist_ok=True) + return staged + + +def staged_artifact_descriptor( + staged: Path, + target: Path, + *, + repo_root: Path, +) -> dict[str, Any]: + """Describe staged bytes under their canonical post-promotion path.""" + if not staged.is_file(): + raise FileNotFoundError(f"Staged publication artifact is missing: {staged}") + relative_target = target.resolve().relative_to(repo_root.resolve()).as_posix() + return { + "path": relative_target, + "bytes": int(staged.stat().st_size), + "sha256": sha256_file(staged), + } + + +def _validate_promotion_inputs( + artifacts: Mapping[Path, Path], + *, + staged_manifest: Path, + manifest_target: Path, + repo_root: Path, + transaction_root: Path, +) -> list[tuple[Path, Path]]: + if not artifacts: + raise ValueError("A publication generation must contain artifacts.") + root = repo_root.resolve() + transaction = transaction_root.resolve() + transaction.relative_to(root) + manifest_target_resolved = manifest_target.resolve() + manifest_target_resolved.relative_to(root) + staged_manifest_resolved = staged_manifest.resolve() + staged_manifest_resolved.relative_to(transaction) + if not staged_manifest_resolved.is_file(): + raise FileNotFoundError(f"Staged publication manifest is missing: {staged_manifest}") + + normalized: list[tuple[Path, Path]] = [] + staged_paths: set[Path] = set() + for target, staged in artifacts.items(): + target_resolved = target.resolve() + staged_resolved = staged.resolve() + target_resolved.relative_to(root) + staged_resolved.relative_to(transaction) + if target_resolved == manifest_target_resolved: + raise ValueError("The publication manifest must be promoted separately and last.") + if not staged_resolved.is_file(): + raise FileNotFoundError(f"Staged publication artifact is missing: {staged}") + if staged_resolved in staged_paths: + raise ValueError(f"A staged artifact is mapped more than once: {staged}") + staged_paths.add(staged_resolved) + normalized.append((target_resolved, staged_resolved)) + return sorted(normalized, key=lambda item: item[0].as_posix()) + + +def promote_publication_generation( + artifacts: Mapping[Path, Path], + *, + staged_manifest: Path, + manifest_target: Path, + repo_root: Path, + transaction_root: Path, +) -> tuple[Path, ...]: + """Promote one validated generation and roll back every target on failure. + + ``artifacts`` maps canonical targets to staged files. Existing targets are + copied to a rollback area before any replacement. The manifest is always + replaced after every other artifact, so it never advertises an incomplete + generation. + """ + ordered = _validate_promotion_inputs( + artifacts, + staged_manifest=staged_manifest, + manifest_target=manifest_target, + repo_root=repo_root, + transaction_root=transaction_root, + ) + manifest = manifest_target.resolve() + staged_manifest_resolved = staged_manifest.resolve() + targets = [target for target, _ in ordered] + [manifest] + rollback_root = transaction_root.resolve() / "rollback" + backups: dict[Path, Path | None] = {} + + for target in targets: + if target.exists() and not target.is_file(): + raise IsADirectoryError(f"Publication target is not a file: {target}") + if target.is_file(): + relative = target.relative_to(repo_root.resolve()) + backup_path = rollback_root / relative + backup_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, backup_path) + backups[target] = backup_path + else: + backups[target] = None + + promoted: list[Path] = [] + try: + for target, staged in ordered: + target.parent.mkdir(parents=True, exist_ok=True) + os.replace(staged, target) + promoted.append(target) + manifest.parent.mkdir(parents=True, exist_ok=True) + os.replace(staged_manifest_resolved, manifest) + promoted.append(manifest) + except BaseException as error: + rollback_failures: list[str] = [] + for target in reversed(promoted): + rollback_backup = backups[target] + try: + if rollback_backup is None: + target.unlink(missing_ok=True) + else: + os.replace(rollback_backup, target) + except OSError as rollback_error: + rollback_failures.append(f"{target}: {rollback_error}") + if rollback_failures: + details = "; ".join(rollback_failures) + raise RuntimeError(f"Publication rollback was incomplete: {details}") from error + raise + + shutil.rmtree(rollback_root, ignore_errors=True) + return tuple(promoted) diff --git a/src/ijds_audit/publication_sources.py b/src/ijds_audit/publication_sources.py new file mode 100644 index 0000000..d8c734d --- /dev/null +++ b/src/ijds_audit/publication_sources.py @@ -0,0 +1,456 @@ +"""Load and verify the single active paper-evidence source registry.""" + +from __future__ import annotations + +import hashlib +import posixpath +import re +import subprocess +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +import yaml + +from src.utils.artifact_descriptor import relative_artifact_descriptor + +_REGISTRY_SECTIONS = ("lineages", "diagnostics", "sensitivities", "replay_dependencies") +_IDENTITY_MARKERS = frozenset( + { + "run_tag", + "protocol_tag", + "protocol_commit", + "scientific_uv_lock_sha256", + "status", + "paper_role", + "dvc_tracked", + "dvc_roots", + "freeze_sha256", + } +) +_LEGACY_DVC_PHASES = frozenset({"outcome_free", "evaluation"}) +_DVC_ROOTS = ("data/processed", "models") +_PROTOCOL_COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}") +_SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +_DVC_DIRECTORY_MD5_PATTERN = re.compile(r"[0-9a-f]{32}\.dir") + + +@dataclass(frozen=True) +class _RegistryUnit: + location: tuple[str, ...] + run_tag: str + protocol_tag: str | None + protocol_commit: str | None + scientific_uv_lock_sha256: str | None + paper_role: str | None + declared_dvc_tracked: bool | None + dvc_roots: tuple[str, ...] | None + + +def load_source_registry( + path: Path, + *, + repo_root: Path | None = None, +) -> dict[str, Any]: + """Load and structurally validate the active source registry. + + Passing ``repo_root`` additionally validates the contents of every declared + DVC pointer. The optional argument preserves the structural-only API used by + lightweight DVC target discovery. + """ + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Active evidence source registry must be a mapping.") + if payload.get("status") != "active_ijds_paper_evidence_source_registry": + raise ValueError("Unexpected active evidence source registry status.") + + units = _validated_registry_units(payload) + tracked_units = _dvc_tracked_units(units) + pointers = payload.get("dvc_pointers") + if not isinstance(pointers, list) or not all( + isinstance(item, str) and bool(item) for item in pointers + ): + raise TypeError("Active evidence source registry dvc_pointers must be a string list.") + + expected = { + f"{prefix}/experiments/ijds_audit/{unit.run_tag}.dvc" + for unit in tracked_units + for prefix in (unit.dvc_roots or _DVC_ROOTS) + } + actual = set(pointers) + if actual != expected or len(pointers) != len(expected): + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + details = f" Missing: {missing}; unexpected: {unexpected}." + raise ValueError( + "Active DVC pointers do not match the DVC-tracked registry units." + details + ) + + if repo_root is not None: + _verify_dvc_pointers(pointers, repo_root=repo_root) + _verify_protocol_replay_contracts(units, repo_root=repo_root) + return payload + + +def active_lineage_run_tags(payload: Mapping[str, Any]) -> tuple[str, ...]: + """Return DVC-tracked run tags in causal/config declaration order.""" + units = _validated_registry_units(payload) + return tuple(unit.run_tag for unit in _dvc_tracked_units(units)) + + +def load_verified_source_registry( + path: Path, + *, + repo_root: Path, +) -> tuple[dict[str, Any], dict[str, Path]]: + """Return registry metadata and hash-verified source paths.""" + payload = load_source_registry(path, repo_root=repo_root) + sources = payload.get("sources") + if not isinstance(sources, Mapping) or not sources: + raise ValueError("Active evidence source registry is empty.") + verified: dict[str, Path] = {} + seen_paths: set[str] = set() + for name, raw_descriptor in sources.items(): + if not isinstance(raw_descriptor, Mapping): + raise TypeError(f"Evidence source descriptor {name!r} must be a mapping.") + descriptor = dict(raw_descriptor) + descriptor_path = descriptor.get("path") + if not isinstance(descriptor_path, str) or not descriptor_path: + raise TypeError(f"Evidence source descriptor {name!r} omits path.") + source_path = (repo_root / descriptor_path).resolve() + source_path.relative_to(repo_root.resolve()) + actual = relative_artifact_descriptor(source_path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor.get(field): + raise RuntimeError(f"Evidence source {name!r} mismatched on {field}.") + if actual["path"] in seen_paths: + raise ValueError(f"Duplicate active evidence source path: {actual['path']}") + seen_paths.add(str(actual["path"])) + verified[str(name)] = source_path + return payload, verified + + +def _validated_registry_units(payload: Mapping[str, Any]) -> tuple[_RegistryUnit, ...]: + lineages = payload.get("lineages") + if not isinstance(lineages, Mapping): + raise TypeError("Active evidence source registry omits lineages.") + + units = list(_walk_registry_group(lineages, location=("lineages",))) + for section in _REGISTRY_SECTIONS[1:]: + if section not in payload: + continue + section_payload = payload[section] + if not isinstance(section_payload, Mapping): + raise TypeError(f"Active evidence source registry {section} must be a mapping.") + units.extend(_walk_registry_group(section_payload, location=(section,))) + if not units: + raise ValueError("Active evidence source registry declares no identities.") + + uses_explicit_contract = any( + unit.paper_role is not None or unit.declared_dvc_tracked is not None for unit in units + ) + if uses_explicit_contract: + incomplete = [ + _format_location(unit.location) + for unit in units + if unit.paper_role is None or unit.declared_dvc_tracked is None + ] + if incomplete: + raise TypeError( + "Explicit registry identities require both paper_role and dvc_tracked: " + f"{incomplete}." + ) + + seen_run_tags: dict[str, tuple[str, ...]] = {} + seen_protocol_tags: dict[str, tuple[str, ...]] = {} + for unit in units: + previous = seen_run_tags.get(unit.run_tag) + if previous is not None: + raise ValueError( + "Active evidence registry run tags must be globally unique: " + f"{unit.run_tag!r} appears at {_format_location(previous)} and " + f"{_format_location(unit.location)}." + ) + seen_run_tags[unit.run_tag] = unit.location + if unit.protocol_tag is None: + continue + previous = seen_protocol_tags.get(unit.protocol_tag) + if previous is not None: + raise ValueError( + "Active evidence registry protocol tags must be globally unique: " + f"{unit.protocol_tag!r} appears at {_format_location(previous)} and " + f"{_format_location(unit.location)}." + ) + seen_protocol_tags[unit.protocol_tag] = unit.location + return tuple(units) + + +def _walk_registry_group( + group: Mapping[str, Any], + *, + location: tuple[str, ...], +) -> Iterator[_RegistryUnit]: + if _looks_like_identity(group): + yield _parse_registry_unit(group, location=location) + return + if not group: + raise TypeError(f"Registry identity group {_format_location(location)} is empty.") + + for raw_name, child in group.items(): + if not isinstance(raw_name, str) or not raw_name: + raise TypeError( + f"Registry identity group {_format_location(location)} has an invalid name." + ) + child_location = (*location, raw_name) + if not isinstance(child, Mapping): + raise TypeError( + f"Registry identity {_format_location(child_location)} must be a mapping." + ) + yield from _walk_registry_group(child, location=child_location) + + +def _looks_like_identity(payload: Mapping[str, Any]) -> bool: + return not payload or any(field in payload for field in _IDENTITY_MARKERS) + + +def _parse_registry_unit( + identity: Mapping[str, Any], + *, + location: tuple[str, ...], +) -> _RegistryUnit: + run_tag = _required_text(identity, "run_tag", location=location) + if run_tag in {".", ".."} or "/" in run_tag or "\\" in run_tag: + raise ValueError( + f"Registry identity {_format_location(location)}.run_tag must name one directory." + ) + + has_protocol_tag = "protocol_tag" in identity + has_protocol_commit = "protocol_commit" in identity + if has_protocol_tag != has_protocol_commit: + missing_field = "protocol_commit" if has_protocol_tag else "protocol_tag" + raise TypeError( + f"Missing registry identity: {_format_location((*location, missing_field))}." + ) + + protocol_tag: str | None = None + protocol_commit: str | None = None + scientific_uv_lock_sha256: str | None = None + if has_protocol_tag: + protocol_tag = _required_text(identity, "protocol_tag", location=location) + protocol_commit = _required_text(identity, "protocol_commit", location=location) + if _PROTOCOL_COMMIT_PATTERN.fullmatch(protocol_commit) is None: + raise ValueError( + f"Registry identity {_format_location(location)}.protocol_commit " + "must be a 40-character lowercase hexadecimal commit." + ) + scientific_uv_lock_sha256 = _required_text( + identity, + "scientific_uv_lock_sha256", + location=location, + ) + if _SHA256_PATTERN.fullmatch(scientific_uv_lock_sha256) is None: + raise ValueError( + f"Registry identity {_format_location(location)}.scientific_uv_lock_sha256 " + "must be a 64-character lowercase hexadecimal digest." + ) + else: + _required_text(identity, "status", location=location) + if "scientific_uv_lock_sha256" in identity: + raise ValueError( + f"Registry identity {_format_location(location)} cannot declare a scientific " + "lock without a protocol commit." + ) + + if "status" in identity: + _required_text(identity, "status", location=location) + paper_role = None + if "paper_role" in identity: + paper_role = _required_text(identity, "paper_role", location=location) + + declared_dvc_tracked: bool | None = None + if "dvc_tracked" in identity: + raw_dvc_tracked = identity["dvc_tracked"] + if not isinstance(raw_dvc_tracked, bool): + raise TypeError( + f"Registry identity {_format_location(location)}.dvc_tracked must be boolean." + ) + declared_dvc_tracked = raw_dvc_tracked + dvc_roots: tuple[str, ...] | None = None + if "dvc_roots" in identity: + raw_roots = identity["dvc_roots"] + if declared_dvc_tracked is not True: + raise ValueError( + f"Registry identity {_format_location(location)}.dvc_roots requires " + "dvc_tracked=true." + ) + if ( + not isinstance(raw_roots, list) + or not raw_roots + or not all(isinstance(value, str) and value in _DVC_ROOTS for value in raw_roots) + or len(raw_roots) != len(set(raw_roots)) + ): + raise ValueError( + f"Registry identity {_format_location(location)}.dvc_roots must be a " + f"nonempty unique subset of {list(_DVC_ROOTS)}." + ) + dvc_roots = tuple(raw_roots) + return _RegistryUnit( + location=location, + run_tag=run_tag, + protocol_tag=protocol_tag, + protocol_commit=protocol_commit, + scientific_uv_lock_sha256=scientific_uv_lock_sha256, + paper_role=paper_role, + declared_dvc_tracked=declared_dvc_tracked, + dvc_roots=dvc_roots, + ) + + +def _required_text( + payload: Mapping[str, Any], + field: str, + *, + location: tuple[str, ...], +) -> str: + value = payload.get(field) + if not isinstance(value, str) or not value or value != value.strip(): + raise TypeError(f"Missing registry identity: {_format_location((*location, field))}.") + return value + + +def _dvc_tracked_units(units: tuple[_RegistryUnit, ...]) -> tuple[_RegistryUnit, ...]: + uses_explicit_tracking = any(unit.declared_dvc_tracked is not None for unit in units) + if uses_explicit_tracking: + return tuple(unit for unit in units if unit.declared_dvc_tracked is True) + return tuple(unit for unit in units if _legacy_dvc_tracked(unit)) + + +def _legacy_dvc_tracked(unit: _RegistryUnit) -> bool: + middle = set(unit.location[1:-1]) + return ( + unit.location[0] == "lineages" + and unit.location[-1] in _LEGACY_DVC_PHASES + and not middle.intersection({"diagnostics", "sensitivities"}) + ) + + +def _verify_dvc_pointers(pointers: list[str], *, repo_root: Path) -> None: + resolved_root = repo_root.resolve() + for pointer in pointers: + pointer_path = (resolved_root / pointer).resolve() + try: + pointer_path.relative_to(resolved_root) + except ValueError as exc: + raise ValueError(f"Active DVC pointer escapes the repository: {pointer}") from exc + if pointer_path.suffix != ".dvc" or not pointer_path.is_file(): + raise FileNotFoundError(f"Invalid active DVC pointer: {pointer_path}") + _verify_dvc_pointer(pointer_path, display_path=pointer) + + +def _verify_protocol_replay_contracts(units: tuple[_RegistryUnit, ...], *, repo_root: Path) -> None: + """Verify every protocol tag and the environment lock stored at that commit.""" + if not (repo_root / ".git").exists(): + return + for unit in units: + if ( + unit.protocol_tag is None + or unit.protocol_commit is None + or unit.scientific_uv_lock_sha256 is None + ): + continue + tag_result = subprocess.run( + ["git", "rev-list", "-n", "1", unit.protocol_tag], + cwd=repo_root, + check=False, + capture_output=True, + text=True, + ) + resolved = tag_result.stdout.strip() + if tag_result.returncode != 0 or resolved != unit.protocol_commit: + raise RuntimeError( + f"Registry protocol tag {unit.protocol_tag!r} does not resolve to " + f"declared commit {unit.protocol_commit}." + ) + lock_result = subprocess.run( + ["git", "show", f"{unit.protocol_commit}:uv.lock"], + cwd=repo_root, + check=False, + capture_output=True, + ) + if lock_result.returncode != 0: + raise RuntimeError( + f"Registry protocol commit {unit.protocol_commit} does not contain uv.lock." + ) + actual_lock_sha256 = hashlib.sha256(lock_result.stdout).hexdigest() + if actual_lock_sha256 != unit.scientific_uv_lock_sha256: + raise RuntimeError( + f"Registry protocol tag {unit.protocol_tag!r} declares uv.lock " + f"{unit.scientific_uv_lock_sha256}, but its commit contains " + f"{actual_lock_sha256}." + ) + + +def _verify_dvc_pointer(path: Path, *, display_path: str) -> None: + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"Malformed active DVC pointer YAML: {display_path}") from exc + if not isinstance(payload, Mapping): + raise TypeError(f"Active DVC pointer {display_path} must be a mapping.") + + outs = payload.get("outs") + if not isinstance(outs, list) or len(outs) != 1: + raise ValueError(f"Active DVC pointer {display_path} must declare exactly one out.") + out = outs[0] + if not isinstance(out, Mapping): + raise TypeError(f"Active DVC pointer {display_path} out must be a mapping.") + + raw_out_path = out.get("path") + if not isinstance(raw_out_path, str) or not raw_out_path: + raise TypeError(f"Active DVC pointer {display_path} out path must be a string.") + normalized_out_path = _normalize_dvc_out_path(raw_out_path, display_path=display_path) + if normalized_out_path != path.stem: + raise ValueError( + f"Active DVC pointer {display_path} out path {normalized_out_path!r} " + f"does not match run directory {path.stem!r}." + ) + + md5 = out.get("md5") + if not isinstance(md5, str) or _DVC_DIRECTORY_MD5_PATTERN.fullmatch(md5) is None: + raise ValueError( + f"Active DVC pointer {display_path} md5 must be a lowercase DVC directory hash." + ) + if "hash" in out and out["hash"] != "md5": + raise ValueError(f"Active DVC pointer {display_path} hash must be 'md5'.") + _validate_nonnegative_integer(out, "size", display_path=display_path) + _validate_nonnegative_integer(out, "nfiles", display_path=display_path) + + +def _normalize_dvc_out_path(value: str, *, display_path: str) -> str: + normalized = posixpath.normpath(value.replace("\\", "/")) + normalized_path = PurePosixPath(normalized) + if ( + normalized in {"", ".", ".."} + or normalized_path.is_absolute() + or normalized.startswith("../") + ): + raise ValueError(f"Active DVC pointer {display_path} out path must be relative.") + return normalized_path.as_posix() + + +def _validate_nonnegative_integer( + payload: Mapping[str, Any], + field: str, + *, + display_path: str, +) -> None: + value = payload.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError( + f"Active DVC pointer {display_path} {field} must be a non-negative integer." + ) + + +def _format_location(location: tuple[str, ...]) -> str: + return ".".join(location) diff --git a/src/ijds_audit/raw_data_audit.py b/src/ijds_audit/raw_data_audit.py new file mode 100644 index 0000000..3873aad --- /dev/null +++ b/src/ijds_audit/raw_data_audit.py @@ -0,0 +1,477 @@ +"""Full-archive schema and temporal-availability audit for the IJDS design.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import duckdb +import pandas as pd + +POST_OUTCOME_COLUMNS = frozenset( + { + "loan_status", + "out_prncp", + "out_prncp_inv", + "total_pymnt", + "total_pymnt_inv", + "total_rec_prncp", + "total_rec_int", + "total_rec_late_fee", + "recoveries", + "collection_recovery_fee", + "last_pymnt_d", + "last_pymnt_amnt", + "next_pymnt_d", + "last_credit_pull_d", + "last_fico_range_high", + "last_fico_range_low", + "pymnt_plan", + "hardship_flag", + "hardship_type", + "hardship_reason", + "hardship_status", + "deferral_term", + "hardship_amount", + "hardship_start_date", + "hardship_end_date", + "payment_plan_start_date", + "hardship_length", + "hardship_dpd", + "hardship_loan_status", + "orig_projected_additional_accrued_interest", + "hardship_payoff_balance_amount", + "hardship_last_payment_amount", + "debt_settlement_flag", + } +) + +IDENTIFIER_OR_FREE_TEXT_COLUMNS = frozenset( + {"column000", "id", "url", "title", "emp_title", "zip_code"} +) +GEOGRAPHY_PROXY_COLUMNS = frozenset({"addr_state"}) +PROTOCOL_METADATA_COLUMNS = frozenset({"issue_d", "term"}) +CONTEMPORANEOUS_FUNDING_COLUMNS = frozenset({"funded_amnt", "funded_amnt_inv"}) + + +@dataclass(frozen=True) +class RawDataAudit: + """Small, inspectable outputs from one full raw-CSV scan.""" + + archive_inventory: pd.DataFrame + status_inventory: pd.DataFrame + inventory: pd.DataFrame + feature_coverage: pd.DataFrame + feature_contract: pd.DataFrame + amount_alignment: pd.DataFrame + cutoff_label_availability: pd.DataFrame + + +def _identifier(column: str) -> str: + return '"' + str(column).replace('"', '""') + '"' + + +def _date(value: Any) -> str: + timestamp = pd.Timestamp(value) + if pd.isna(timestamp): + raise ValueError(f"Invalid protocol date: {value!r}") + return cast(pd.Timestamp, timestamp).strftime("%Y-%m-%d") + + +def _period_start(value: Any) -> str: + period = pd.Period(str(value), freq="M") + if pd.isna(period): + raise ValueError(f"Invalid protocol month: {value!r}") + return cast(pd.Period, period).start_time.strftime("%Y-%m-%d") + + +def _period_end(value: Any) -> str: + period = pd.Period(str(value), freq="M") + if pd.isna(period): + raise ValueError(f"Invalid protocol month: {value!r}") + return cast(pd.Period, period).end_time.normalize().strftime("%Y-%m-%d") + + +def cohort_case_sql(design: Mapping[str, Any], source: Mapping[str, Any]) -> str: + """Return the exhaustive 36-month cohort partition used by the audit.""" + return f"""CASE + WHEN issue_date <= DATE '{_date(design["development_end"])}' + THEN 'pd_development' + WHEN issue_date BETWEEN DATE '{_date(design["probability_calibration_start"])}' + AND DATE '{_date(design["probability_calibration_end"])}' + THEN 'probability_calibration' + WHEN issue_date BETWEEN DATE '{_date(design["conformal_fit_start"])}' + AND DATE '{_date(design["conformal_fit_end"])}' + THEN 'conformal_fit' + WHEN issue_date BETWEEN DATE '{_date(design["policy_development_start"])}' + AND DATE '{_date(design["policy_development_end"])}' + THEN 'policy_development' + WHEN issue_date > DATE '{_date(design["policy_development_end"])}' + AND issue_date <= DATE '{_date(source["information_cutoff"])}' + THEN 'maturity_gap' + WHEN issue_date BETWEEN DATE '{_period_start(design["primary_oot_start_month"])}' + AND DATE '{_period_end(design["primary_oot_end_month"])}' + THEN 'primary_oot' + WHEN issue_date BETWEEN DATE '{_period_start(design["censored_extension_start_month"])}' + AND DATE '{_period_end(design["censored_extension_end_month"])}' + THEN 'censored_extension' + WHEN issue_date > DATE '{_period_end(design["censored_extension_end_month"])}' + THEN 'post_extension' + ELSE 'outside_declared_timeline' + END""" + + +def classify_raw_column(column: str, *, active_required: Sequence[str]) -> tuple[str, str]: + """Assign a conservative provenance role without looking at outcomes.""" + name = str(column) + if name in POST_OUTCOME_COLUMNS or name.startswith("hardship_"): + return "post_outcome_or_servicing", "never eligible as a prediction feature" + if name in IDENTIFIER_OR_FREE_TEXT_COLUMNS: + return "identifier_or_free_text", "excluded from the scientific feature contract" + if name in GEOGRAPHY_PROXY_COLUMNS: + return "geography_proxy", "available but excluded from the active model" + if name in PROTOCOL_METADATA_COLUMNS: + return "protocol_metadata", "used for chronology or term eligibility" + if name in CONTEMPORANEOUS_FUNDING_COLUMNS: + return ( + "contemporaneous_funding", + "audited separately; not needed by the active portfolio because requested and funded amounts nearly coincide", + ) + if name.startswith("sec_app_") or name in { + "application_type", + "annual_inc_joint", + "dti_joint", + "verification_status_joint", + "revol_bal_joint", + }: + return "joint_application_origination", "origination-time but introduced late" + if name in active_required: + return "active_protocol_input", "loaded by the active maturity-safe protocol" + return "candidate_origination", "requires temporal coverage and leakage review" + + +def _raw_relation(path: Path) -> str: + escaped = str(path.resolve()).replace("'", "''") + return ( + "read_csv('" + + escaped + + "', auto_detect=true, sample_size=200000, all_varchar=true, ignore_errors=false)" + ) + + +def _header(connection: duckdb.DuckDBPyConnection, raw_path: Path) -> list[str]: + rows = connection.execute(f"DESCRIBE SELECT * FROM {_raw_relation(raw_path)}").fetchall() + return [str(row[0]) for row in rows] + + +def _archive_inventory( + connection: duckdb.DuckDBPyConnection, + raw_path: Path, + *, + servicing_cutoff: str, +) -> tuple[pd.DataFrame, pd.DataFrame]: + cutoff = pd.Timestamp(servicing_cutoff).date().isoformat() + parsed = f""" + SELECT *, + try_strptime(issue_d, '%b-%Y') AS issue_date, + try_strptime(last_pymnt_d, '%b-%Y') AS last_payment_date, + try_strptime(last_credit_pull_d, '%b-%Y') AS last_credit_pull_date, + try_cast(regexp_extract(term, '([0-9]+)', 1) AS INTEGER) AS term_months + FROM {_raw_relation(raw_path)} + """ + inventory = connection.execute( + f""" + WITH parsed AS ({parsed}) + SELECT count(*) AS raw_rows, + count(DISTINCT nullif(trim(id), '')) AS distinct_nonblank_ids, + sum(CASE WHEN id IS NULL OR trim(id) = '' THEN 1 ELSE 0 END) AS blank_id_rows, + sum(CASE WHEN issue_date IS NOT NULL THEN 1 ELSE 0 END) AS valid_loan_rows, + sum(CASE WHEN issue_date IS NULL THEN 1 ELSE 0 END) AS invalid_issue_date_rows, + min(issue_date) AS first_issue_date, + max(issue_date) AS last_issue_date, + max(last_payment_date) AS last_payment_date_max, + max(last_credit_pull_date) AS last_credit_pull_date_max, + sum(CASE WHEN last_payment_date > DATE '{cutoff}' THEN 1 ELSE 0 END) + AS last_payment_rows_after_cutoff, + sum(CASE WHEN last_credit_pull_date > DATE '{cutoff}' THEN 1 ELSE 0 END) + AS last_credit_pull_rows_after_cutoff, + sum(CASE WHEN term_months = 36 THEN 1 ELSE 0 END) AS term36_rows, + sum(CASE WHEN term_months = 60 THEN 1 ELSE 0 END) AS term60_rows, + sum(CASE WHEN term_months NOT IN (36, 60) OR term_months IS NULL THEN 1 ELSE 0 END) + AS other_or_invalid_term_rows + FROM parsed + """ + ).fetchdf() + for column in ( + "first_issue_date", + "last_issue_date", + "last_payment_date_max", + "last_credit_pull_date_max", + ): + inventory[column] = inventory[column].map( + lambda value: _date(value) if pd.notna(value) else None + ) + statuses = connection.execute( + f""" + WITH parsed AS ({parsed}) + SELECT coalesce(nullif(trim(loan_status), ''), '__MISSING__') AS loan_status, + count(*) AS rows, + sum(CASE WHEN term_months = 36 THEN 1 ELSE 0 END) AS term36_rows, + sum(CASE WHEN term_months = 60 THEN 1 ELSE 0 END) AS term60_rows + FROM parsed + GROUP BY loan_status + ORDER BY rows DESC, loan_status + """ + ).fetchdf() + return inventory, statuses + + +def _profile_coverage( + connection: duckdb.DuckDBPyConnection, + raw_path: Path, + *, + columns: Sequence[str], + cohort_sql: str, +) -> tuple[pd.DataFrame, pd.DataFrame]: + profile_columns = [ + column for column in columns if column not in {"column000", "id", "issue_d", "term"} + ] + expressions = ["count(*) AS rows"] + for column in profile_columns: + quoted = _identifier(column) + present = f"({quoted} IS NOT NULL AND trim({quoted}) <> '')" + expressions.extend( + [ + f"sum(CASE WHEN {present} THEN 1 ELSE 0 END) AS {_identifier(column + '__n')}", + f"min(issue_date) FILTER (WHERE {present}) AS {_identifier(column + '__first')}", + ] + ) + query = f""" + WITH parsed AS ( + SELECT *, + try_strptime(issue_d, '%b-%Y') AS issue_date, + try_cast(regexp_extract(term, '([0-9]+)', 1) AS INTEGER) AS term_months + FROM {_raw_relation(raw_path)} + ), term36 AS ( + SELECT *, {cohort_sql} AS cohort + FROM parsed + WHERE issue_date IS NOT NULL AND term_months = 36 + ) + SELECT cohort, {", ".join(expressions)} + FROM term36 + GROUP BY cohort + ORDER BY cohort + """ + wide = connection.execute(query).fetchdf() + coverage_rows: list[dict[str, Any]] = [] + for record in wide.to_dict(orient="records"): + cohort = str(record["cohort"]) + rows = int(record["rows"]) + for column in profile_columns: + present_rows = int(record[f"{column}__n"]) + first_seen = record[f"{column}__first"] + coverage_rows.append( + { + "cohort": cohort, + "feature": column, + "rows": rows, + "present_rows": present_rows, + "coverage": present_rows / rows if rows else 0.0, + "first_seen": ( + cast(pd.Timestamp, pd.Timestamp(first_seen)).strftime("%Y-%m-%d") + if pd.notna(first_seen) + else None + ), + } + ) + coverage = pd.DataFrame(coverage_rows).sort_values(["feature", "cohort"]).reset_index(drop=True) + inventory = wide[["cohort", "rows"]].copy() + return inventory, coverage + + +def _amount_alignment( + connection: duckdb.DuckDBPyConnection, raw_path: Path, *, cohort_sql: str +) -> pd.DataFrame: + query = f""" + WITH parsed AS ( + SELECT try_strptime(issue_d, '%b-%Y') AS issue_date, + try_cast(regexp_extract(term, '([0-9]+)', 1) AS INTEGER) AS term_months, + try_cast(loan_amnt AS DOUBLE) AS loan_amnt, + try_cast(funded_amnt AS DOUBLE) AS funded_amnt + FROM {_raw_relation(raw_path)} + ), term36 AS ( + SELECT *, {cohort_sql} AS cohort + FROM parsed + WHERE issue_date IS NOT NULL AND term_months = 36 + ) + SELECT cohort, + count(*) AS rows, + sum(loan_amnt - funded_amnt) AS total_gap, + avg(loan_amnt - funded_amnt) AS mean_gap, + quantile_cont(loan_amnt - funded_amnt, 0.5) AS median_gap, + max(loan_amnt - funded_amnt) AS max_gap, + avg(CASE WHEN loan_amnt > funded_amnt THEN 1.0 ELSE 0.0 END) AS partial_share, + sum(funded_amnt) / sum(loan_amnt) AS funded_ratio + FROM term36 + GROUP BY cohort + ORDER BY cohort + """ + return connection.execute(query).fetchdf() + + +def _cutoff_label_availability( + connection: duckdb.DuckDBPyConnection, + raw_path: Path, + *, + design: Mapping[str, Any], + source: Mapping[str, Any], +) -> pd.DataFrame: + cutoff = _date(source["information_cutoff"]) + lag = int(source["charged_off_reporting_lag_months"]) + start = _date(pd.Timestamp(design["policy_development_end"]) + pd.Timedelta(days=1)) + query = f""" + WITH parsed AS ( + SELECT try_strptime(issue_d, '%b-%Y') AS issue_date, + try_cast(regexp_extract(term, '([0-9]+)', 1) AS INTEGER) AS term_months, + lower(trim(loan_status)) AS status, + try_strptime(last_pymnt_d, '%b-%Y') AS last_payment + FROM {_raw_relation(raw_path)} + ), gap AS ( + SELECT *, + status LIKE '%fully paid%' AS is_good, + status LIKE '%charged off%' AS is_bad, + CASE + WHEN status LIKE '%fully paid%' + THEN last_payment <= DATE '{cutoff}' + WHEN status LIKE '%charged off%' + THEN last_payment + INTERVAL {lag} MONTH <= DATE '{cutoff}' + ELSE false + END AS label_available + FROM parsed + WHERE term_months = 36 + AND issue_date BETWEEN DATE '{start}' AND DATE '{cutoff}' + ) + SELECT year(issue_date) AS issue_year, + count(*) AS rows, + sum(CASE WHEN is_good OR is_bad THEN 1 ELSE 0 END) AS terminal_at_snapshot, + sum(CASE WHEN label_available THEN 1 ELSE 0 END) AS available_by_cutoff, + avg(CASE WHEN label_available THEN 1.0 ELSE 0.0 END) AS available_rate, + sum(CASE WHEN is_bad AND label_available THEN 1 ELSE 0 END) AS available_bad, + sum(CASE WHEN is_good AND label_available THEN 1 ELSE 0 END) AS available_good + FROM gap + GROUP BY issue_year + ORDER BY issue_year + """ + return connection.execute(query).fetchdf() + + +def _feature_contract( + coverage: pd.DataFrame, + *, + columns: Sequence[str], + active_required: Sequence[str], + rules: Mapping[str, Any], +) -> pd.DataFrame: + fitting = {"pd_development", "probability_calibration", "conformal_fit"} + minimum_required = float(rules["minimum_fitting_feature_coverage"]) + late_fitting = float(rules["late_feature_fitting_coverage"]) + late_primary = float(rules["late_feature_primary_coverage"]) + exceptions = rules.get("active_feature_coverage_exceptions", {}) + if not isinstance(exceptions, Mapping): + raise TypeError("active_feature_coverage_exceptions must be a mapping.") + pivot = coverage.pivot(index="feature", columns="cohort", values="coverage") + rows: list[dict[str, Any]] = [] + for column in columns: + role, reason = classify_raw_column(column, active_required=active_required) + values = pivot.loc[column] if column in pivot.index else pd.Series(dtype=float) + fitting_values = [float(values.get(cohort, 0.0)) for cohort in sorted(fitting)] + minimum_fitting_coverage = min(fitting_values) if fitting_values else 0.0 + primary_coverage = float(values.get("primary_oot", 0.0)) + exception = exceptions.get(column) + if exception is not None and not isinstance(exception, Mapping): + raise TypeError(f"Coverage exception for {column} must be a mapping.") + exception_type = None if exception is None else str(exception["type"]) + requires_sensitivity = bool( + exception is not None and exception.get("requires_sensitivity", False) + ) + eligible = role in {"active_protocol_input", "candidate_origination"} and ( + minimum_fitting_coverage >= minimum_required or exception is not None + ) + late_feature = minimum_fitting_coverage < late_fitting and primary_coverage >= late_primary + rows.append( + { + "feature": column, + "provenance_role": role, + "loaded_by_active_protocol": column in active_required, + "minimum_fitting_coverage": minimum_fitting_coverage, + "primary_oot_coverage": primary_coverage, + "coverage_threshold": minimum_required, + "coverage_exception": exception_type, + "missingness_semantics": ( + None if exception is None else str(exception["missingness_semantics"]) + ), + "requires_sensitivity": requires_sensitivity, + "late_feature": late_feature, + "eligible_for_current_temporal_model": eligible, + "decision": ( + "eligible_with_declared_coverage_exception" + if eligible and exception is not None + else "eligible" + if eligible + else "exclude_post_outcome" + if role == "post_outcome_or_servicing" + else "exclude_late_schema" + if late_feature + else "exclude_by_role_or_coverage" + ), + "reason": reason, + } + ) + return pd.DataFrame(rows).sort_values(["decision", "feature"]).reset_index(drop=True) + + +def audit_raw_dataset( + raw_path: Path, + config: Mapping[str, Any], + *, + rules: Mapping[str, Any], +) -> RawDataAudit: + """Scan the complete CSV once per audit family and return compact evidence.""" + connection = duckdb.connect() + connection.execute("PRAGMA threads=8") + columns = _header(connection, raw_path) + archive_inventory, status_inventory = _archive_inventory( + connection, + raw_path, + servicing_cutoff=str(config["source"]["snapshot_date"]), + ) + cohort_sql = cohort_case_sql(config["design"], config["source"]) + inventory, coverage = _profile_coverage( + connection, raw_path, columns=columns, cohort_sql=cohort_sql + ) + active_required = [str(value) for value in config["source"]["required_raw_columns"]] + contract = _feature_contract( + coverage, + columns=columns, + active_required=active_required, + rules=rules, + ) + amount = _amount_alignment(connection, raw_path, cohort_sql=cohort_sql) + labels = _cutoff_label_availability( + connection, + raw_path, + design=config["design"], + source=config["source"], + ) + connection.close() + return RawDataAudit( + archive_inventory=archive_inventory, + status_inventory=status_inventory, + inventory=inventory, + feature_coverage=coverage, + feature_contract=contract, + amount_alignment=amount, + cutoff_label_availability=labels, + ) diff --git a/src/ijds_audit/robustness_evidence.py b/src/ijds_audit/robustness_evidence.py new file mode 100644 index 0000000..37d97db --- /dev/null +++ b/src/ijds_audit/robustness_evidence.py @@ -0,0 +1,527 @@ +"""Hash-verified fit-label and allocation-granularity evidence.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pandas as pd + +from src.ijds_audit.grid_contracts import require_exact_frame, require_exact_grid, require_finite +from src.utils.artifact_descriptor import verified_artifact_path + +WINDOW_IDS = ( + *(f"w{index:02d}_2012m{index:02d}_m{index + 5:02d}" for index in range(1, 8)), + "w08_2012m08_2013m01", +) +FIT_LABEL_SCENARIOS = ( + "observed_only", + "all_unavailable_nondefault", + "all_unavailable_default", + "hindsight_terminal", +) +FIT_EVALUATION_ARTIFACTS = ("coverage", "summary_table", "phase_stratum") +FIT_OUTCOME_FREE_ARTIFACTS = ("scores", "fit_audit", "scenario_audit", "recipes") +GRANULARITY_EVALUATION_ARTIFACTS = ("granularity_contrasts",) +GRANULARITY_OUTCOME_FREE_ARTIFACTS = ( + "rounded_allocations", + "rounded_solve_records", + "granularity_audit", +) +RULERS = ("objective_matched", "normalized_score") +COORDINATES = (0.25, 0.50, 0.75) +GAMMAS = (0.0, 1.0) +FIT_DESIGN_SPLITS = ("pd_development", "probability_calibration", "conformal_fit") + + +@dataclass(frozen=True) +class FitLabelCompletionEvidence: + """Verified completion-scenario freeze, evaluation, and derived findings.""" + + freeze: dict[str, Any] + summary: dict[str, Any] + frames: dict[str, pd.DataFrame] + outcome_free_artifacts: dict[str, Path] + evaluation_artifacts: dict[str, Path] + findings: dict[str, Any] + + +@dataclass(frozen=True) +class AllocationGranularityEvidence: + """Verified deterministic lot-rounding freeze, evaluation, and findings.""" + + freeze: dict[str, Any] + summary: dict[str, Any] + frames: dict[str, pd.DataFrame] + outcome_free_artifacts: dict[str, Path] + evaluation_artifacts: dict[str, Path] + findings: dict[str, Any] + + +def _load_json_object(path: Path, *, label: str) -> dict[str, Any]: + raw: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise TypeError(f"{label} must be a JSON object with string keys.") + return cast(dict[str, Any], raw) + + +def _require_identity( + payload: Mapping[str, Any], + identity: Mapping[str, Any], + *, + label: str, +) -> None: + for field in ("run_tag", "protocol_tag", "protocol_commit"): + if payload.get(field) != identity.get(field): + raise RuntimeError(f"{label} identity changed on {field}.") + + +def _require_no_side_effects(payload: Mapping[str, Any], *, label: str) -> None: + if payload.get("outcome_based_selection") is not False: + raise RuntimeError(f"{label} reports outcome-based selection.") + if payload.get("protected_stages_run") != []: + raise RuntimeError(f"{label} reports a protected-stage execution.") + if payload.get("protected_artifacts_written") != []: + raise RuntimeError(f"{label} reports a protected-artifact write.") + + +def _verified_inventory( + raw: object, + *, + expected_names: tuple[str, ...], + repo_root: Path, + label: str, +) -> dict[str, Path]: + if not isinstance(raw, Mapping) or set(raw) != set(expected_names): + raise RuntimeError(f"{label} artifact inventory changed.") + paths: dict[str, Path] = {} + for name, descriptor in raw.items(): + if not isinstance(descriptor, Mapping): + raise TypeError(f"{label} descriptor {name!r} is invalid.") + paths[str(name)] = verified_artifact_path( + cast(Mapping[str, Any], descriptor), + repo_root=repo_root, + label=f"{label} {name}", + ) + return paths + + +def _require_summary_freeze_descriptor( + summary: Mapping[str, Any], + *, + freeze_path: Path, + repo_root: Path, + label: str, +) -> None: + descriptor = summary.get("freeze") + if not isinstance(descriptor, Mapping): + raise TypeError(f"{label} summary omits its freeze descriptor.") + verified = verified_artifact_path( + descriptor, + repo_root=repo_root, + label=f"{label} freeze", + ) + if verified.resolve() != freeze_path.resolve(): + raise RuntimeError(f"{label} summary points to a different freeze.") + + +def _validate_fit_frames(frames: Mapping[str, pd.DataFrame]) -> dict[str, Any]: + coverage = frames["coverage"] + summary_table = frames["summary_table"] + phase = frames["phase_stratum"] + require_exact_grid( + coverage, + domains={ + "fit_label_scenario": FIT_LABEL_SCENARIOS, + "window_id": WINDOW_IDS, + "conformal_group": (-1, 2), + }, + label="fit-label completion coverage", + ) + require_exact_grid( + phase, + domains={ + "fit_label_scenario": FIT_LABEL_SCENARIOS, + "window_id": WINDOW_IDS, + }, + label="fit-label completion phase stratum", + ) + if not phase["conformal_group"].eq(2).all(): + raise RuntimeError("Fit-label phase table is not restricted to conformal stratum 2.") + require_exact_grid( + summary_table, + domains={"fit_label_scenario": FIT_LABEL_SCENARIOS}, + label="fit-label completion summary", + ) + require_finite( + coverage, + ( + "candidate_rows", + "resolved_rows", + "unresolved_rows", + "coverage_lower", + "coverage_upper", + "mean_width", + "fit_prevalence", + "fit_residual_quantile", + ), + label="fit-label completion coverage", + ) + if not ( + coverage["candidate_rows"].eq(coverage["resolved_rows"] + coverage["unresolved_rows"]).all() + and coverage["coverage_lower"].le(coverage["coverage_upper"]).all() + and coverage[["coverage_lower", "coverage_upper"]].ge(0.0).all(axis=None) + and coverage[["coverage_lower", "coverage_upper"]].le(1.0).all(axis=None) + ): + raise RuntimeError("Fit-label completion coverage bounds are incoherent.") + overall = coverage.loc[coverage["conformal_group"].eq(-1)] + derived = ( + overall.groupby("fit_label_scenario", observed=True, sort=True) + .agg( + windows=("window_id", "nunique"), + coverage_lower_min=("coverage_lower", "min"), + coverage_upper_max=("coverage_upper", "max"), + windows_upper_below_nominal=( + "coverage_upper", + lambda values: int((values < 0.90).sum()), + ), + all_windows_upper_below_nominal=( + "coverage_upper", + lambda values: bool((values < 0.90).all()), + ), + mean_width_min=("mean_width", "min"), + mean_width_max=("mean_width", "max"), + ) + .reset_index() + ) + require_exact_frame( + summary_table, + derived, + keys=("fit_label_scenario",), + label="fit-label completion derived summary", + ) + crossing_by_scenario: dict[str, bool] = {} + for scenario in FIT_LABEL_SCENARIOS: + scoped = phase.loc[phase["fit_label_scenario"].eq(scenario)].set_index("window_id") + w7 = scoped.loc["w07_2012m07_m12"] + w8 = scoped.loc["w08_2012m08_2013m01"] + crossing_by_scenario[scenario] = bool( + float(w7["fit_prevalence"]) >= 0.10 > float(w8["fit_prevalence"]) + and float(w7["fit_residual_quantile"]) > 0.5 + and float(w8["fit_residual_quantile"]) < 0.5 + ) + return { + "coverage_cells": int(len(overall)), + "phase_cells": int(len(phase)), + "all_scenarios_all_windows_upper_below_nominal": bool( + summary_table["all_windows_upper_below_nominal"].all() + ), + "w7_w8_crossing_by_scenario": crossing_by_scenario, + "w7_w8_crossing_scenarios": int(sum(crossing_by_scenario.values())), + "w7_w8_crossing_in_all_scenarios": bool(all(crossing_by_scenario.values())), + } + + +def _validate_fit_scenario_audit(frame: pd.DataFrame) -> dict[str, int]: + require_exact_grid( + frame, + domains={"scenario": FIT_LABEL_SCENARIOS, "design_split": FIT_DESIGN_SPLITS}, + label="fit-label completion scenario audit", + ) + require_finite( + frame, + ( + "rows", + "source_available_rows", + "source_unavailable_rows", + "active_available_rows", + "completed_rows", + "source_available_prevalence", + "active_prevalence", + ), + label="fit-label completion scenario audit", + ) + if ( + not frame["rows"] + .eq(frame["source_available_rows"] + frame["source_unavailable_rows"]) + .all() + ): + raise RuntimeError("Fit-label completion source-label census does not reconcile.") + observed = frame.loc[frame["scenario"].eq("observed_only")] + if not ( + observed["active_available_rows"].eq(observed["source_available_rows"]).all() + and observed["completed_rows"].eq(0).all() + ): + raise RuntimeError("Observed-only fit-label scenario unexpectedly completes labels.") + completed = frame.loc[~frame["scenario"].eq("observed_only")] + if not ( + completed["active_available_rows"].eq(completed["rows"]).all() + and completed["completed_rows"].eq(completed["source_unavailable_rows"]).all() + ): + raise RuntimeError("Fit-label completion scenarios do not fill every unavailable label.") + counts = ( + frame.groupby("design_split", observed=True)["source_unavailable_rows"].nunique().to_dict() + ) + if any(value != 1 for value in counts.values()): + raise RuntimeError("Unavailable fit-label counts vary by completion scenario.") + by_split = observed.set_index("design_split")["source_unavailable_rows"].astype(int).to_dict() + return {str(key): int(value) for key, value in by_split.items()} + + +def load_fit_label_completion_evidence( + summary_path: Path, + *, + freeze_path: Path, + identity: Mapping[str, Any], + repo_root: Path, +) -> FitLabelCompletionEvidence: + """Load and verify the four declared fit-label scenarios.""" + freeze = _load_json_object(freeze_path, label="Fit-label completion freeze") + summary = _load_json_object(summary_path, label="Fit-label completion summary") + _require_identity(freeze, identity, label="Fit-label completion freeze") + _require_identity(summary, identity, label="Fit-label completion summary") + _require_no_side_effects(freeze, label="Fit-label completion freeze") + _require_no_side_effects(summary, label="Fit-label completion summary") + if freeze.get("status") != "fit_labels_completed_before_evaluation_outcome_join": + raise RuntimeError("Fit-label completion freeze is incomplete.") + if summary.get("status") != "complete_fit_label_completion_corner_sensitivity": + raise RuntimeError("Fit-label completion evaluation is incomplete.") + if tuple(freeze.get("scenarios", ())) != FIT_LABEL_SCENARIOS: + raise RuntimeError("Fit-label completion freeze scenario order changed.") + if tuple(summary.get("scenarios", ())) != FIT_LABEL_SCENARIOS: + raise RuntimeError("Fit-label completion summary scenario order changed.") + if freeze.get("evaluation_outcome_columns_passed_to_fitting") != []: + raise RuntimeError("Fit-label completion freeze reports evaluation-outcome leakage.") + _require_summary_freeze_descriptor( + summary, + freeze_path=freeze_path, + repo_root=repo_root, + label="Fit-label completion", + ) + outcome_free = _verified_inventory( + freeze.get("artifacts"), + expected_names=FIT_OUTCOME_FREE_ARTIFACTS, + repo_root=repo_root, + label="Fit-label completion outcome-free", + ) + evaluation = _verified_inventory( + summary.get("artifacts"), + expected_names=FIT_EVALUATION_ARTIFACTS, + repo_root=repo_root, + label="Fit-label completion evaluation", + ) + frames = {name: pd.read_parquet(path) for name, path in evaluation.items()} + findings = _validate_fit_frames(frames) + unavailable_by_split = _validate_fit_scenario_audit( + pd.read_parquet(outcome_free["scenario_audit"]) + ) + findings = { + **findings, + "unavailable_fit_labels_by_split": unavailable_by_split, + "unavailable_fit_labels_total": int(sum(unavailable_by_split.values())), + } + results = summary.get("results") + if not isinstance(results, Mapping): + raise TypeError("Fit-label completion summary omits results.") + if ( + int(results.get("coverage_rows", -1)) != 64 + or int(results.get("overall_cells", -1)) != findings["coverage_cells"] + or int(results.get("phase_cells", -1)) != findings["phase_cells"] + or results.get("all_scenarios_all_windows_upper_below_nominal") + is not findings["all_scenarios_all_windows_upper_below_nominal"] + ): + raise RuntimeError("Fit-label completion top-level census changed.") + return FitLabelCompletionEvidence( + freeze=freeze, + summary=summary, + frames=frames, + outcome_free_artifacts=outcome_free, + evaluation_artifacts=evaluation, + findings=findings, + ) + + +def fit_label_completion_publication_table( + evidence: FitLabelCompletionEvidence, +) -> pd.DataFrame: + """Return one complete, nonselective paper row per completion scenario.""" + table = evidence.frames["summary_table"].copy() + crossing = evidence.findings["w7_w8_crossing_by_scenario"] + table["w7_w8_stratum2_crossing"] = table["fit_label_scenario"].map(crossing) + return table.sort_values("fit_label_scenario").reset_index(drop=True) + + +def _validate_granularity_frame(frame: pd.DataFrame) -> dict[str, Any]: + require_exact_grid( + frame, + domains={ + "window_id": WINDOW_IDS, + "frontier_ruler": RULERS, + "frontier_coordinate": COORDINATES, + "gamma": GAMMAS, + }, + label="allocation granularity contrasts", + ) + require_finite( + frame, + ( + "periods", + "cash_residual_total", + "cash_share", + "policy_a_capital", + "policy_b_capital", + "policy_a_normalization_capital", + "policy_b_normalization_capital", + "realized_payoff_rate_difference_lower", + "realized_payoff_rate_difference_upper", + "weighted_default_difference_lower", + "weighted_default_difference_upper", + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ), + label="allocation granularity contrasts", + ) + if not ( + frame["contrast"].eq("rounded_lot_minus_continuous").all() + and frame["role"].eq("primary_oot").all() + and frame["policy_a"].eq("rounded_lot").all() + and frame["policy_b"].eq("continuous").all() + and frame["periods"].eq(15).all() + and frame["causal_interpretation"].eq(False).all() + ): + raise RuntimeError("Allocation granularity contrast semantics changed.") + for lower, upper in ( + ("realized_payoff_rate_difference_lower", "realized_payoff_rate_difference_upper"), + ("weighted_default_difference_lower", "weighted_default_difference_upper"), + ("weighted_miscoverage_difference_lower", "weighted_miscoverage_difference_upper"), + ): + if not frame[lower].le(frame[upper]).all(): + raise RuntimeError(f"Allocation granularity bounds are reversed for {lower}.") + if not np.isclose(frame["policy_a_normalization_capital"], 15_000_000.0).all(): + raise RuntimeError("Rounded policies are not normalized by committed capital.") + if not np.isclose(frame["policy_b_normalization_capital"], 15_000_000.0).all(): + raise RuntimeError("Continuous policies are not normalized by committed capital.") + return { + "tracks": int(len(frame)), + "cash_share_max": float(frame["cash_share"].max()), + "payoff_rate_perturbation_abs_max": float( + frame[ + [ + "realized_payoff_rate_difference_lower", + "realized_payoff_rate_difference_upper", + ] + ] + .abs() + .to_numpy() + .max() + ), + "default_rate_perturbation_abs_max": float( + frame[["weighted_default_difference_lower", "weighted_default_difference_upper"]] + .abs() + .to_numpy() + .max() + ), + "miscoverage_rate_perturbation_abs_max": float( + frame[ + [ + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ] + ] + .abs() + .to_numpy() + .max() + ), + } + + +def load_allocation_granularity_evidence( + summary_path: Path, + *, + freeze_path: Path, + identity: Mapping[str, Any], + repo_root: Path, +) -> AllocationGranularityEvidence: + """Load and verify the deterministic USD-25 lot-rounding sensitivity.""" + freeze = _load_json_object(freeze_path, label="Allocation granularity freeze") + summary = _load_json_object(summary_path, label="Allocation granularity summary") + _require_identity(freeze, identity, label="Allocation granularity freeze") + _require_identity(summary, identity, label="Allocation granularity summary") + _require_no_side_effects(freeze, label="Allocation granularity freeze") + _require_no_side_effects(summary, label="Allocation granularity summary") + if freeze.get("status") != "allocation_granularity_frozen_before_outcome_join": + raise RuntimeError("Allocation granularity freeze is incomplete.") + if summary.get("status") != "complete_allocation_granularity_sensitivity": + raise RuntimeError("Allocation granularity evaluation is incomplete.") + if freeze.get("outcome_columns_passed_to_rounding") != []: + raise RuntimeError("Allocation rounding reports evaluation-outcome leakage.") + if ( + freeze.get("rounding_rule") != "floor_each_exposure_hold_residual_as_cash" + or float(freeze.get("lot_size_usd", -1.0)) != 25.0 + or float(freeze.get("committed_budget_usd", -1.0)) != 1_000_000.0 + ): + raise RuntimeError("Allocation granularity rounding contract changed.") + _require_summary_freeze_descriptor( + summary, + freeze_path=freeze_path, + repo_root=repo_root, + label="Allocation granularity", + ) + outcome_free = _verified_inventory( + freeze.get("artifacts"), + expected_names=GRANULARITY_OUTCOME_FREE_ARTIFACTS, + repo_root=repo_root, + label="Allocation granularity outcome-free", + ) + evaluation = _verified_inventory( + summary.get("artifacts"), + expected_names=GRANULARITY_EVALUATION_ARTIFACTS, + repo_root=repo_root, + label="Allocation granularity evaluation", + ) + frames = {name: pd.read_parquet(path) for name, path in evaluation.items()} + findings = _validate_granularity_frame(frames["granularity_contrasts"]) + results = summary.get("results") + if not isinstance(results, Mapping): + raise TypeError("Allocation granularity summary omits results.") + for key, value in findings.items(): + if not np.isclose(float(results.get(key, np.nan)), float(value), rtol=0.0, atol=1e-15): + raise RuntimeError(f"Allocation granularity summary changed on {key}.") + freeze_results = freeze.get("results") + if not isinstance(freeze_results, Mapping) or ( + int(freeze_results.get("portfolios", -1)) != 1440 + or int(freeze_results.get("source_rows", -1)) != 143_175 + or int(freeze_results.get("rounded_positive_rows", -1)) != 143_167 + or int(freeze_results.get("changed_rows", -1)) != 2_985 + ): + raise RuntimeError("Allocation granularity outcome-free census changed.") + findings = { + **findings, + "portfolios": int(freeze_results["portfolios"]), + "source_rows": int(freeze_results["source_rows"]), + "rounded_positive_rows": int(freeze_results["rounded_positive_rows"]), + "changed_rows": int(freeze_results["changed_rows"]), + "cash_residual_min": float(freeze_results["cash_residual_min"]), + "cash_residual_mean": float(freeze_results["cash_residual_mean"]), + "cash_residual_max": float(freeze_results["cash_residual_max"]), + "monthly_cash_share_max": float(freeze_results["cash_share_max"]), + } + return AllocationGranularityEvidence( + freeze=freeze, + summary=summary, + frames=frames, + outcome_free_artifacts=outcome_free, + evaluation_artifacts=evaluation, + findings=findings, + ) + + +def allocation_granularity_publication_table( + evidence: AllocationGranularityEvidence, +) -> pd.DataFrame: + """Return one compact row for the complete deterministic rounding diagnostic.""" + return pd.DataFrame([evidence.findings]) diff --git a/src/ijds_audit/sensitivity_evidence.py b/src/ijds_audit/sensitivity_evidence.py new file mode 100644 index 0000000..d6fef3f --- /dev/null +++ b/src/ijds_audit/sensitivity_evidence.py @@ -0,0 +1,493 @@ +"""Verified derivations for retrospective IJDS sensitivity evidence.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import Any, cast + +import pandas as pd + +from src.ijds_audit.grid_contracts import require_exact_frame, require_exact_grid, require_finite +from src.utils.artifact_descriptor import verified_artifact_path + +ENDPOINT_LAGS = (0, 3, 6, 8, 12) +IDENTIFICATION_WIDTH_COLUMNS = ( + "realized_payoff_identification_width", + "realized_payoff_rate_identification_width", + "weighted_default_identification_width", + "weighted_miscoverage_identification_width", +) +COVERAGE_LEARNERS = ( + "catboost_platt", + "numeric_logistic_platt", + "catboost_monotonic_platt", + "woe_scorecard_platform_platt", + "woe_scorecard_borrower_platt", +) +COVERAGE_ROLES = ("policy_development", "primary_oot", "censored_extension") +ENDPOINT_CENSUS_ROLES = ( + "pd_development", + "probability_calibration", + "conformal_fit", + "policy_development", + "primary_oot", + "censored_extension", +) +WINDOW_IDS = ( + *(f"w{index:02d}_2012m{index:02d}_m{index + 5:02d}" for index in range(1, 8)), + "w08_2012m08_2013m01", +) +RULERS = ("objective_matched", "normalized_score") +COORDINATES = (0.25, 0.50, 0.75) +TWO_RULER_METRICS = ( + "standardized_payoff", + "funded_default", + "funded_binary_miscoverage", +) +TWO_RULER_DIRECTIONS = ("gamma_1_lower", "gamma_1_higher", "crosses_zero", "exact_zero") +POLICY_IDS = tuple(f"linear-{index:03d}" for index in range(1, 10)) +SUPPORT_SCOPES = ( + "named_c0_c1_c2", + "development_admissible_exact_frontier", + "broad_stress_exact_frontier", +) +SUPPORT_METRICS = ("standardized_payoff", "terminal_default", "funded_miscoverage") + + +@dataclass(frozen=True) +class EndpointSensitivityEvidence: + """Hash-verified endpoint sensitivity and active-evaluation reconciliation.""" + + summary: dict[str, Any] + frames: dict[str, pd.DataFrame] + reconciliation: dict[str, Any] + + +def _verified_artifacts( + summary: Mapping[str, Any], + *, + repo_root: Path, + expected_names: Iterable[str], +) -> dict[str, Path]: + artifacts = summary.get("artifacts") + if not isinstance(artifacts, Mapping) or set(artifacts) != set(expected_names): + raise RuntimeError("Endpoint sensitivity artifact inventory changed.") + verified: dict[str, Path] = {} + for name, raw in artifacts.items(): + if not isinstance(raw, Mapping): + raise TypeError(f"Endpoint artifact {name!r} is not a descriptor.") + verified[str(name)] = verified_artifact_path( + raw, + repo_root=repo_root, + label=f"Endpoint {name}", + ) + return verified + + +def _validate_endpoint_frames(frames: Mapping[str, pd.DataFrame]) -> None: + census = frames["endpoint_census"] + require_exact_grid( + census, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "role": ENDPOINT_CENSUS_ROLES, + }, + label="endpoint census", + ) + require_finite( + census, + ( + "candidate_rows", + "resolved_rows", + "unresolved_rows", + "default_rows", + "nondefault_rows", + ), + label="endpoint census", + ) + if not ( + census["candidate_rows"].eq(census["resolved_rows"] + census["unresolved_rows"]).all() + and census["resolved_rows"].eq(census["default_rows"] + census["nondefault_rows"]).all() + ): + raise RuntimeError("Endpoint census arithmetic does not reconcile.") + + coverage = frames["coverage_cells"] + require_exact_grid( + coverage, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "learner": COVERAGE_LEARNERS, + "window_id": WINDOW_IDS, + "role": COVERAGE_ROLES, + "taxonomy_groups": (5,), + "conformal_group": (-1,), + }, + label="endpoint coverage", + ) + require_finite( + coverage, + ( + "candidate_rows", + "resolved_rows", + "unresolved_rows", + "coverage_resolved", + "coverage_lower", + "coverage_upper", + ), + label="endpoint coverage", + ) + if not ( + coverage["candidate_rows"].eq(coverage["resolved_rows"] + coverage["unresolved_rows"]).all() + and coverage["coverage_lower"].le(coverage["coverage_upper"]).all() + and coverage[["coverage_resolved", "coverage_lower", "coverage_upper"]] + .ge(0.0) + .all(axis=None) + and coverage[["coverage_resolved", "coverage_lower", "coverage_upper"]] + .le(1.0) + .all(axis=None) + ): + raise RuntimeError("Endpoint coverage bounds or censuses are incoherent.") + + coverage_summary = frames["coverage_summary"] + require_exact_grid( + coverage_summary, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "learner": COVERAGE_LEARNERS, + "role": COVERAGE_ROLES, + }, + label="endpoint coverage summary", + ) + derived_coverage = ( + coverage.groupby(["charged_off_lag_months", "learner", "role"], observed=True, sort=True) + .agg( + windows=("window_id", "nunique"), + coverage_lower_min=("coverage_lower", "min"), + coverage_upper_max=("coverage_upper", "max"), + ) + .reset_index() + ) + require_exact_frame( + coverage_summary, + derived_coverage, + keys=("charged_off_lag_months", "learner", "role"), + label="endpoint coverage summary", + ) + + contrasts = frames["two_ruler_window_contrasts"] + require_exact_grid( + contrasts, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + }, + label="endpoint two-ruler contrasts", + ) + directions = frames["two_ruler_directions"] + require_exact_grid( + directions, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + "metric": TWO_RULER_METRICS, + }, + label="endpoint two-ruler directions", + ) + if not set(directions["direction"]).issubset(TWO_RULER_DIRECTIONS): + raise RuntimeError("Endpoint two-ruler direction vocabulary changed.") + direction_census = frames["two_ruler_direction_census"] + require_exact_grid( + direction_census, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "metric": TWO_RULER_METRICS, + "direction": TWO_RULER_DIRECTIONS, + }, + label="endpoint two-ruler direction census", + ) + derived_direction_rows: list[dict[str, Any]] = [] + for lag, metric in product(ENDPOINT_LAGS, TWO_RULER_METRICS): + scoped = directions.loc[ + directions["charged_off_lag_months"].eq(lag) & directions["metric"].eq(metric) + ] + counts = scoped["direction"].value_counts().to_dict() + derived_direction_rows.extend( + { + "charged_off_lag_months": lag, + "metric": metric, + "direction": direction, + "cells": int(counts.get(direction, 0)), + } + for direction in TWO_RULER_DIRECTIONS + ) + require_exact_frame( + direction_census, + pd.DataFrame(derived_direction_rows), + keys=("charged_off_lag_months", "metric", "direction"), + label="endpoint two-ruler direction census", + ) + + envelopes = frames["exact_support_envelopes"] + require_exact_grid( + envelopes, + domains={ + "charged_off_lag_months": ENDPOINT_LAGS, + "window_id": WINDOW_IDS, + "paired_policy_id": POLICY_IDS, + "scope": SUPPORT_SCOPES, + "metric": SUPPORT_METRICS, + }, + label="endpoint exact-support envelopes", + ) + if bool(envelopes["nested_scope_independent_replications"].any()): + raise RuntimeError("Endpoint support scopes are incorrectly marked as replications.") + support_census = frames["exact_support_census"] + derived_support = ( + envelopes.groupby( + ["scope", "metric", "direction", "charged_off_lag_months"], + observed=True, + sort=True, + ) + .size() + .reset_index() + ) + derived_support.columns = [ + "scope", + "metric", + "direction", + "charged_off_lag_months", + "cells", + ] + require_exact_frame( + support_census, + derived_support, + keys=("charged_off_lag_months", "scope", "metric", "direction"), + label="endpoint exact-support census", + ) + + +def load_endpoint_sensitivity_evidence( + summary_path: Path, + *, + identity: Mapping[str, Any], + repo_root: Path, + reference_coverage: pd.DataFrame, + reference_two_ruler: pd.DataFrame, + reference_envelopes: pd.DataFrame, + float_atol: float = 0.0, + float_rtol: float = 0.0, +) -> EndpointSensitivityEvidence: + """Load endpoint V1, verify all grids, and reconcile its six-month slice.""" + summary_raw: object = json.loads(summary_path.read_text(encoding="utf-8")) + if not isinstance(summary_raw, dict): + raise TypeError("Endpoint sensitivity summary must be a JSON object.") + if not all(isinstance(key, str) for key in summary_raw): + raise TypeError("Endpoint sensitivity summary keys must be strings.") + summary = cast(dict[str, Any], summary_raw) + if summary.get("status") != "complete_retrospective_endpoint_availability_sensitivity": + raise RuntimeError("Endpoint sensitivity is incomplete.") + for field in ("run_tag", "protocol_tag", "protocol_commit"): + if summary.get(field) != identity.get(field): + raise RuntimeError(f"Endpoint sensitivity identity changed on {field}.") + if tuple(summary.get("lags", ())) != ENDPOINT_LAGS: + raise RuntimeError("Endpoint sensitivity lag grid changed.") + if ( + int(summary.get("coverage_cells", -1)) != 600 + or int(summary.get("two_ruler_direction_cells", -1)) != 720 + or int(summary.get("exact_support_envelopes", -1)) != 3240 + ): + raise RuntimeError("Endpoint sensitivity top-level census changed.") + boundary = summary.get("claim_boundary", {}) + for field in ( + "preregistered", + "confirmatory", + "prospective", + "outcome_based_selection", + "allocation_refit", + "policy_selection", + "model_selection", + "endpoint_selection", + ): + if boundary.get(field) is not False: + raise RuntimeError(f"Endpoint sensitivity unexpectedly allows {field}.") + if ( + summary.get("protected_stages_run") != [] + or summary.get("protected_artifacts_written") != [] + ): + raise RuntimeError("Endpoint sensitivity reports a protected-stage side effect.") + if any(value is not None for value in summary.get("selection", {}).values()): + raise RuntimeError("Endpoint sensitivity reports a selected result.") + + artifact_names = ( + "endpoint_census", + "coverage_cells", + "coverage_summary", + "two_ruler_window_contrasts", + "two_ruler_directions", + "two_ruler_direction_census", + "exact_support_envelopes", + "exact_support_census", + ) + paths = _verified_artifacts(summary, repo_root=repo_root, expected_names=artifact_names) + frames = {name: pd.read_parquet(path) for name, path in paths.items()} + _validate_endpoint_frames(frames) + + lag6_coverage = ( + frames["coverage_cells"] + .loc[frames["coverage_cells"]["charged_off_lag_months"].eq(6)] + .drop(columns="charged_off_lag_months") + ) + reference_coverage_slice = reference_coverage.loc[ + reference_coverage["learner"].isin(COVERAGE_LEARNERS) + & reference_coverage["window_id"].isin(WINDOW_IDS) + & reference_coverage["role"].isin(COVERAGE_ROLES) + & reference_coverage["taxonomy_groups"].eq(5) + & reference_coverage["conformal_group"].eq(-1) + ] + coverage_keys = ( + "learner", + "window_id", + "taxonomy_groups", + "role", + "conformal_group", + ) + require_exact_grid( + reference_coverage_slice, + domains={ + "learner": COVERAGE_LEARNERS, + "window_id": WINDOW_IDS, + "taxonomy_groups": (5,), + "role": COVERAGE_ROLES, + "conformal_group": (-1,), + }, + label="active coverage reference", + ) + require_exact_frame( + lag6_coverage, + reference_coverage_slice, + keys=coverage_keys, + label="endpoint lag-6 coverage", + float_atol=float_atol, + float_rtol=float_rtol, + ) + + lag6_two_ruler = ( + frames["two_ruler_window_contrasts"] + .loc[frames["two_ruler_window_contrasts"]["charged_off_lag_months"].eq(6)] + .drop(columns="charged_off_lag_months") + ) + contrast_keys = ("window_id", "ruler", "coordinate") + require_exact_frame( + lag6_two_ruler, + reference_two_ruler, + keys=contrast_keys, + label="endpoint lag-6 two-ruler contrasts", + float_atol=float_atol, + float_rtol=float_rtol, + allowed_expected_extra_columns=IDENTIFICATION_WIDTH_COLUMNS, + ) + + lag6_envelopes = ( + frames["exact_support_envelopes"] + .loc[frames["exact_support_envelopes"]["charged_off_lag_months"].eq(6)] + .drop(columns="charged_off_lag_months") + ) + envelope_keys = ("window_id", "paired_policy_id", "scope", "metric") + require_exact_frame( + lag6_envelopes, + reference_envelopes, + keys=envelope_keys, + label="endpoint lag-6 exact-support envelopes", + float_atol=float_atol, + float_rtol=float_rtol, + ) + reconciliation = { + "charged_off_lag_months": 6, + "coverage_cells_exact": int(len(lag6_coverage)), + "two_ruler_contrasts_exact": int(len(lag6_two_ruler)), + "exact_support_envelopes_exact": int(len(lag6_envelopes)), + "byte_value_equal_after_lag_column_removed": True, + } + return EndpointSensitivityEvidence( + summary=summary, + frames=frames, + reconciliation=reconciliation, + ) + + +def endpoint_publication_table(evidence: EndpointSensitivityEvidence) -> pd.DataFrame: + """Derive one complete, nonselective paper row per endpoint lag.""" + census = evidence.frames["endpoint_census"] + coverage = evidence.frames["coverage_cells"] + directions = evidence.frames["two_ruler_directions"] + support = evidence.frames["exact_support_envelopes"] + rows: list[dict[str, Any]] = [] + for lag in ENDPOINT_LAGS: + primary_census = census.loc[ + census["charged_off_lag_months"].eq(lag) & census["role"].eq("primary_oot") + ] + if len(primary_census) != 1: + raise RuntimeError(f"Endpoint lag {lag} has no unique primary census.") + primary_coverage = coverage.loc[ + coverage["charged_off_lag_months"].eq(lag) & coverage["role"].eq("primary_oot") + ] + if len(primary_coverage) != 40: + raise RuntimeError(f"Endpoint lag {lag} has an incomplete primary coverage grid.") + maximum = primary_coverage.loc[ + primary_coverage["coverage_upper"].eq(primary_coverage["coverage_upper"].max()) + ].sort_values(["learner", "window_id"]) + direction_counts = ( + directions.loc[directions["charged_off_lag_months"].eq(lag)] + .groupby(["metric", "direction"], observed=True) + .size() + .to_dict() + ) + support_counts = ( + support.loc[support["charged_off_lag_months"].eq(lag)] + .groupby(["scope", "metric", "direction"], observed=True) + .size() + .to_dict() + ) + census_row = primary_census.iloc[0] + row: dict[str, Any] = { + "charged_off_lag_months": lag, + "primary_candidates": int(census_row["candidate_rows"]), + "primary_resolved": int(census_row["resolved_rows"]), + "primary_unresolved": int(census_row["unresolved_rows"]), + "primary_defaults": int(census_row["default_rows"]), + "coverage_cells": int(len(primary_coverage)), + "coverage_upper_below_0_90_cells": int( + primary_coverage["coverage_upper"].lt(0.90).sum() + ), + "coverage_upper_at_or_above_0_90_cells": int( + primary_coverage["coverage_upper"].ge(0.90).sum() + ), + "coverage_upper_max": float(maximum["coverage_upper"].iloc[0]), + "coverage_upper_max_learner": ";".join(maximum["learner"].astype(str)), + "coverage_upper_max_window": ";".join(maximum["window_id"].astype(str)), + } + for metric in TWO_RULER_METRICS: + prefix = { + "standardized_payoff": "two_ruler_payoff", + "funded_default": "two_ruler_default", + "funded_binary_miscoverage": "two_ruler_miscoverage", + }[metric] + for direction in TWO_RULER_DIRECTIONS: + row[f"{prefix}_{direction}_cells"] = int( + direction_counts.get((metric, direction), 0) + ) + for metric in SUPPORT_METRICS: + for scope in ("development_admissible_exact_frontier", "broad_stress_exact_frontier"): + for direction in ("guardrail_lower", "guardrail_higher", "crosses_zero"): + key = f"{scope}_{metric}_{direction}_cells" + row[key] = int(support_counts.get((scope, metric, direction), 0)) + rows.append(row) + return pd.DataFrame(rows) diff --git a/src/ijds_audit/simulation.py b/src/ijds_audit/simulation.py new file mode 100644 index 0000000..6037acc --- /dev/null +++ b/src/ijds_audit/simulation.py @@ -0,0 +1,224 @@ +"""Locked factorial mechanism simulation for binary geometry and C2.""" + +from __future__ import annotations + +from collections.abc import Mapping +from itertools import product +from typing import Any + +import numpy as np +import pandas as pd + +from src.evaluation.coverage_transport import binary_miscoverage_bounds +from src.evaluation.standardized_credit_payoff import expected_objective_coefficients +from src.ijds_audit.geometry import summarize_binary_geometry +from src.ijds_audit.portfolio import ( + PointPortfolioSession, + PointPortfolioSolution, + c2_cap, + verify_c2_dominance, +) +from src.models.binary_conformal_guardrail import ( + apply_binary_outcome_recipe, + fit_binary_outcome_recipe, +) + + +def _allocation_distance(left: PointPortfolioSolution, right: PointPortfolioSolution) -> float: + total = float(left.total_allocated + right.total_allocated) + return float(np.abs(left.exposure - right.exposure).sum() / total) + + +def _sharp_payoff_difference( + exposure_a: np.ndarray, + exposure_b: np.ndarray, + rates: np.ndarray, + outcomes: np.ndarray, + *, + lgd: float, +) -> tuple[float, float]: + delta = exposure_a - exposure_b + value_zero = delta * rates + value_one = delta * -float(lgd) + observed = np.isfinite(outcomes) + exact = np.where(outcomes == 1.0, value_one, value_zero) + lower = np.where(observed, exact, np.minimum(value_zero, value_one)) + upper = np.where(observed, exact, np.maximum(value_zero, value_one)) + return float(lower.sum()), float(upper.sum()) + + +def _one_repetition( + *, + seed_sequence: np.random.SeedSequence, + sample_size: int, + score_shift: float, + prevalence_shift: float, + taxonomy_groups: int, + censoring_rate: float, + policy: Mapping[str, Any], + alpha: float, +) -> dict[str, Any]: + fit_rng, candidate_rng, censor_rng = [ + np.random.default_rng(stream) for stream in seed_sequence.spawn(3) + ] + fit_score = fit_rng.beta(2.0, 18.0, size=sample_size) + fit_outcome = fit_rng.binomial(1, fit_score).astype(int) + edges = np.quantile( + fit_score, + np.linspace(0.0, 1.0, int(taxonomy_groups) + 1), + method="linear", + ) + recipe = fit_binary_outcome_recipe( + fit_score, + fit_outcome, + alpha=float(alpha), + n_groups=int(taxonomy_groups), + bin_edges=tuple(float(value) for value in edges), + taxonomy_provenance="simulation_fit_score_quantiles", + taxonomy_method="fixed_empirical_linear_score_quantiles", + ) + candidate_score = np.clip( + candidate_rng.beta(2.0, 18.0, size=sample_size) + float(score_shift), + 1e-6, + 1.0 - 1e-6, + ) + candidate_probability = np.clip(candidate_score + float(prevalence_shift), 1e-6, 1.0 - 1e-6) + candidate_outcome = candidate_rng.binomial(1, candidate_probability).astype(float) + censored = censor_rng.random(sample_size) < float(censoring_rate) + observed_outcome = candidate_outcome.copy() + observed_outcome[censored] = np.nan + _, lower, upper = apply_binary_outcome_recipe(candidate_score, recipe) + observed = np.isfinite(observed_outcome) + miss_low, miss_high = binary_miscoverage_bounds(observed_outcome, lower, upper) + geometry = summarize_binary_geometry(lower, upper) + + frame = pd.DataFrame( + { + "loan_amnt": np.ones(sample_size, dtype=float), + "purpose": np.full(sample_size, "all", dtype=object), + } + ) + rates = np.clip( + 0.06 + 0.70 * candidate_score + candidate_rng.normal(0.0, 0.01, sample_size), 0.03, 0.40 + ) + lgd = float(policy["lgd"]) + objective = expected_objective_coefficients(candidate_score, rates, lgd=lgd) + gamma = float(policy["gamma"]) + effective = candidate_score + gamma * (upper - candidate_score) + guardrail = PointPortfolioSession( + frame, + point_score=effective, + objective_rate=objective, + budget=float(policy["budget_units"]), + purpose_cap=float(policy["purpose_cap"]), + threads=1, + ).solve(float(policy["risk_tolerance"])) + point_session = PointPortfolioSession( + frame, + point_score=candidate_score, + objective_rate=objective, + budget=float(policy["budget_units"]), + purpose_cap=float(policy["purpose_cap"]), + threads=1, + ) + same_cap = point_session.solve(float(policy["risk_tolerance"])) + contemporaneous_cap = c2_cap(guardrail.exposure, candidate_score) + contemporaneous = point_session.solve(contemporaneous_cap) + dominance = verify_c2_dominance( + guardrail_exposure=guardrail.exposure, + point_solution=contemporaneous, + point_score=candidate_score, + objective_rate=objective, + ) + c0_lower, c0_upper = _sharp_payoff_difference( + guardrail.exposure, + same_cap.exposure, + rates, + observed_outcome, + lgd=lgd, + ) + c2_lower, c2_upper = _sharp_payoff_difference( + guardrail.exposure, + contemporaneous.exposure, + rates, + observed_outcome, + lgd=lgd, + ) + return { + "fit_prevalence": float(np.mean(fit_outcome)), + "candidate_prevalence": float(np.mean(candidate_outcome)), + "resolved_rows": int(observed.sum()), + "coverage_resolved": float(1.0 - miss_low[observed].mean()), + "coverage_lower": float(1.0 - miss_high.mean()), + "coverage_upper": float(1.0 - miss_low.mean()), + "guardrail_weighted_effective_score": guardrail.weighted_point_score, + "guardrail_weighted_point_score": c2_cap(guardrail.exposure, candidate_score), + "same_cap_allocation_distance": _allocation_distance(guardrail, same_cap), + "c2_allocation_distance": _allocation_distance(guardrail, contemporaneous), + "same_cap_payoff_difference_lower": c0_lower, + "same_cap_payoff_difference_upper": c0_upper, + "c2_payoff_difference_lower": c2_lower, + "c2_payoff_difference_upper": c2_upper, + **dominance, + **geometry, + } + + +def run_factorial_simulation(config: Mapping[str, Any]) -> tuple[pd.DataFrame, pd.DataFrame]: + """Run every locked factorial cell and return repetitions plus cell summaries.""" + simulation = config["simulation"] + if simulation.get("enabled") is not True: + raise ValueError("The V4 simulation must remain enabled.") + rows: list[dict[str, Any]] = [] + grid = product( + enumerate(simulation["score_shift_grid"]), + enumerate(simulation["prevalence_shift_grid"]), + enumerate(simulation["taxonomy_groups_grid"]), + enumerate(simulation["censoring_rate_grid"]), + ) + for (score_index, score_shift), (prevalence_index, prevalence_shift), ( + group_index, + groups, + ), (censor_index, censoring) in grid: + for repetition in range(int(simulation["repetitions"])): + sequence = np.random.SeedSequence( + [ + int(simulation["random_seed"]), + score_index, + prevalence_index, + group_index, + censor_index, + repetition, + ] + ) + rows.append( + { + "score_shift": float(score_shift), + "prevalence_shift": float(prevalence_shift), + "taxonomy_groups": int(groups), + "censoring_rate": float(censoring), + "repetition": repetition, + **_one_repetition( + seed_sequence=sequence, + sample_size=int(simulation["sample_size"]), + score_shift=float(score_shift), + prevalence_shift=float(prevalence_shift), + taxonomy_groups=int(groups), + censoring_rate=float(censoring), + policy=simulation["mechanism_policy"], + alpha=float(config["conformal"]["alpha"]), + ), + } + ) + repetitions = pd.DataFrame(rows) + keys = ["score_shift", "prevalence_shift", "taxonomy_groups", "censoring_rate"] + numeric = [ + column + for column in repetitions.select_dtypes(include=[np.number]).columns + if column not in {*keys, "repetition"} + ] + summary = repetitions.groupby(keys, observed=True, sort=True)[numeric].agg( + ["mean", "std", "min", "max"] + ) + summary.columns = [f"{column}_{statistic}" for column, statistic in summary.columns] + return repetitions, summary.reset_index() diff --git a/src/ijds_audit/structural_checkpoint.py b/src/ijds_audit/structural_checkpoint.py new file mode 100644 index 0000000..3de0af2 --- /dev/null +++ b/src/ijds_audit/structural_checkpoint.py @@ -0,0 +1,134 @@ +"""Validation and zero-copy recovery for structural-sensitivity shards.""" + +from __future__ import annotations + +import os +from collections.abc import Collection +from dataclasses import dataclass +from pathlib import Path + +import pyarrow.parquet as pq + +ARTIFACT_FILES = { + "solve_records": "solve_records.parquet", + "allocations": "allocations.parquet", + "endpoint_diagnostics": "endpoint_diagnostics.parquet", + "minimum_endpoint_diagnostics": "minimum_endpoint_diagnostics.parquet", + "objective_optimum_diagnostics": "objective_optimum_diagnostics.parquet", + "order_sensitivity": "order_sensitivity.parquet", + "independent_validation": "independent_validation.parquet", +} +EXPECTED_ROWS = { + "solve_records": 1_440, + "endpoint_diagnostics": 720, + "minimum_endpoint_diagnostics": 600, + "objective_optimum_diagnostics": 15, + "order_sensitivity": 1_440, + "independent_validation": 288, +} +FORBIDDEN_OUTCOME_COLUMNS = { + "snapshot_default", + "snapshot_resolution", + "terminal_default", + "terminal_outcome", + "loan_status", + "last_pymnt_d", +} + + +@dataclass(frozen=True) +class StructuralShardInspection: + """Validated paths and row counts for one complete outcome-free scenario.""" + + scenario_id: str + paths: dict[str, Path] + rows: dict[str, int] + minimum_endpoint_retries: int + maximum_retry_slack: float + maximum_cap_residual: float + + +def inspect_structural_shard( + scenario_root: Path, + *, + scenario_id: str, + retry_slacks: Collection[float], + cap_residual_tolerance: float, +) -> StructuralShardInspection: + """Reject incomplete, mislabeled, outcome-bearing, or numerically invalid shards.""" + frontier = scenario_root / "frontier" + expected_names = set(ARTIFACT_FILES.values()) + actual_names = {path.name for path in frontier.glob("*.parquet")} + if actual_names != expected_names: + raise RuntimeError( + f"Structural shard {scenario_id} has artifact inventory {sorted(actual_names)}." + ) + paths: dict[str, Path] = {} + rows: dict[str, int] = {} + for name, filename in ARTIFACT_FILES.items(): + path = frontier / filename + parquet = pq.ParquetFile(path) + columns = set(parquet.schema_arrow.names) + forbidden = sorted(columns.intersection(FORBIDDEN_OUTCOME_COLUMNS)) + if forbidden: + raise RuntimeError(f"Structural shard {scenario_id}/{name} has outcomes: {forbidden}.") + if "scenario_id" not in columns: + raise RuntimeError(f"Structural shard {scenario_id}/{name} lacks scenario identity.") + identities = set(parquet.read(columns=["scenario_id"])["scenario_id"].to_pylist()) + if identities != {scenario_id}: + raise RuntimeError( + f"Structural shard {scenario_id}/{name} has identities {identities}." + ) + row_count = int(parquet.metadata.num_rows) + if name in EXPECTED_ROWS and row_count != EXPECTED_ROWS[name]: + raise RuntimeError( + f"Structural shard {scenario_id}/{name} has {row_count} rows, " + f"not {EXPECTED_ROWS[name]}." + ) + if name == "allocations" and row_count <= 0: + raise RuntimeError(f"Structural shard {scenario_id} has no funded rows.") + paths[name] = path + rows[name] = row_count + minimum = pq.read_table( + paths["minimum_endpoint_diagnostics"], + columns=[ + "minimum_endpoint_retried", + "minimum_endpoint_retry_slack", + "minimum_cap_residual", + ], + ).to_pandas() + allowed_slacks = {0.0, *(float(value) for value in retry_slacks)} + slack_values = set(minimum["minimum_endpoint_retry_slack"].astype(float).unique()) + if not slack_values.issubset(allowed_slacks): + raise RuntimeError(f"Structural shard {scenario_id} has undeclared retry slack.") + maximum_slack = float(minimum["minimum_endpoint_retry_slack"].max()) + maximum_residual = float(minimum["minimum_cap_residual"].abs().max()) + declared_maximum = max(allowed_slacks) + if maximum_slack > declared_maximum or maximum_residual > float(cap_residual_tolerance): + raise RuntimeError(f"Structural shard {scenario_id} exceeds its numerical contract.") + return StructuralShardInspection( + scenario_id=scenario_id, + paths=paths, + rows=rows, + minimum_endpoint_retries=int(minimum["minimum_endpoint_retried"].sum()), + maximum_retry_slack=maximum_slack, + maximum_cap_residual=maximum_residual, + ) + + +def hardlink_structural_shard( + inspection: StructuralShardInspection, + *, + destination_root: Path, +) -> dict[str, Path]: + """Link one validated shard into a fresh run without duplicating bytes.""" + if destination_root.exists(): + raise FileExistsError(f"Structural destination already exists: {destination_root}.") + frontier = destination_root / "frontier" + frontier.mkdir(parents=True) + linked: dict[str, Path] = {} + for name, source in inspection.paths.items(): + destination = frontier / ARTIFACT_FILES[name] + os.link(source, destination) + linked[name] = destination + return linked diff --git a/src/ijds_audit/structural_evidence.py b/src/ijds_audit/structural_evidence.py new file mode 100644 index 0000000..d24c0d9 --- /dev/null +++ b/src/ijds_audit/structural_evidence.py @@ -0,0 +1,563 @@ +"""Verified complete-grid portfolio-structure sensitivity evidence.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import pandas as pd +import yaml + +from src.ijds_audit.grid_contracts import require_exact_frame, require_exact_grid, require_finite +from src.ijds_audit.structural_checkpoint import ARTIFACT_FILES +from src.ijds_audit.structural_sensitivity import declared_scenarios +from src.utils.artifact_descriptor import verified_artifact_path + +WINDOW_IDS = ( + *(f"w{index:02d}_2012m{index:02d}_m{index + 5:02d}" for index in range(1, 8)), + "w08_2012m08_2013m01", +) +RULERS = ("objective_matched", "normalized_score") +COORDINATES = (0.25, 0.50, 0.75) +METRICS = ("standardized_payoff", "funded_default", "funded_binary_miscoverage") +DIRECTIONS = ("gamma_1_lower", "gamma_1_higher", "crosses_zero", "exact_zero") +SUMMARY_ARTIFACTS = ( + "scenario_summary", + "allocation_activity", + "window_contrasts", + "metric_directions", +) +SCENARIO_COLUMNS = ("scenario_id", "budget", "purpose_cap", "lgd", "is_baseline") +SCENARIO_TAG_COLUMNS = ( + "scenario_id", + "scenario_budget", + "scenario_purpose_cap", + "scenario_lgd", + "scenario_is_baseline", +) +REFERENCE_ONLY_IDENTIFICATION_WIDTH_COLUMNS = { + "realized_payoff_identification_width", + "realized_payoff_rate_identification_width", + "weighted_default_identification_width", + "weighted_miscoverage_identification_width", +} +DIRECTION_SUMMARY_COLUMNS = { + "standardized_payoff": { + "gamma_1_lower": "standardized_payoff_gamma_1_lower_cells", + "gamma_1_higher": "standardized_payoff_gamma_1_higher_cells", + "crosses_zero": "standardized_payoff_crosses_zero_cells", + "exact_zero": "standardized_payoff_exact_zero_cells", + }, + "funded_default": { + "gamma_1_lower": "funded_default_gamma_1_lower_cells", + "gamma_1_higher": "funded_default_gamma_1_higher_cells", + "crosses_zero": "funded_default_crosses_zero_cells", + "exact_zero": "funded_default_exact_zero_cells", + }, + "funded_binary_miscoverage": { + "gamma_1_lower": "funded_binary_miscoverage_gamma_1_lower_cells", + "gamma_1_higher": "funded_binary_miscoverage_gamma_1_higher_cells", + "crosses_zero": "funded_binary_miscoverage_crosses_zero_cells", + "exact_zero": "funded_binary_miscoverage_exact_zero_cells", + }, +} + + +@dataclass(frozen=True) +class StructuralSensitivityEvidence: + """Hash-verified V6 freeze, evaluation, frames, and derived findings.""" + + config: dict[str, Any] + freeze: dict[str, Any] + summary: dict[str, Any] + frames: dict[str, pd.DataFrame] + findings: dict[str, Any] + + +def _load_json_object(path: Path, *, label: str) -> dict[str, Any]: + raw: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise TypeError(f"{label} must be a JSON object with string keys.") + return cast(dict[str, Any], raw) + + +def _load_config(path: Path) -> dict[str, Any]: + raw: object = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + raise TypeError("Structural sensitivity config must be a mapping with string keys.") + config = cast(dict[str, Any], raw) + if config.get("protocol_status") != ( + "locked_retrospective_outcome_free_structural_sensitivity_v6_order_tolerance" + ): + raise RuntimeError("Paper-facing structural sensitivity is not locked V6.") + return config + + +def _require_identity( + payload: Mapping[str, Any], + identity: Mapping[str, Any], + *, + label: str, + fields: tuple[str, ...] = ("run_tag", "protocol_tag", "protocol_commit"), +) -> None: + for field in fields: + if payload.get(field) != identity.get(field): + raise RuntimeError(f"{label} identity changed on {field}.") + + +def _require_freeze_contract( + freeze: Mapping[str, Any], + *, + identity: Mapping[str, Any], + scenario_ids: tuple[str, ...], +) -> None: + _require_identity(freeze, identity, label="Structural freeze") + if freeze.get("status") != "outcome_free_structural_grid_frozen_before_endpoint_join": + raise RuntimeError("Structural freeze is incomplete.") + if int(freeze.get("scenario_count", -1)) != len(scenario_ids): + raise RuntimeError("Structural freeze scenario count changed.") + if freeze.get("outcome_columns_passed_to_frontier") != []: + raise RuntimeError("Structural freeze reports outcome leakage.") + if freeze.get("protected_stages_run") != [] or freeze.get("protected_artifacts_written") != []: + raise RuntimeError("Structural freeze reports a protected-stage side effect.") + recovery = freeze.get("recovery") + if not isinstance(recovery, Mapping) or ( + recovery.get("recovered_scenarios") != 35 + or recovery.get("recomputed_scenarios") != 1 + or recovery.get("missing_scenario_ids") != ["b0500k_p020_l025"] + ): + raise RuntimeError("Structural freeze recovery contract changed.") + + +def _verify_scenario_artifacts( + freeze: Mapping[str, Any], + *, + scenario_ids: tuple[str, ...], + repo_root: Path, +) -> None: + artifacts = freeze.get("scenario_artifacts") + if not isinstance(artifacts, Mapping) or set(artifacts) != set(scenario_ids): + raise RuntimeError("Structural freeze scenario inventory changed.") + for scenario_id, raw_inventory in artifacts.items(): + if not isinstance(raw_inventory, Mapping) or set(raw_inventory) != set(ARTIFACT_FILES): + raise RuntimeError(f"Structural shard inventory changed for {scenario_id}.") + for name, descriptor in raw_inventory.items(): + if not isinstance(descriptor, Mapping): + raise TypeError(f"Structural shard descriptor {scenario_id}/{name} is invalid.") + verified_artifact_path( + descriptor, + repo_root=repo_root, + label=f"Structural shard {scenario_id}/{name}", + ) + + +def _verify_freeze_sources(freeze: Mapping[str, Any], *, repo_root: Path) -> None: + for name in ("outcome_free_decision_base", "scenario_counts"): + descriptor = freeze.get(name) + if not isinstance(descriptor, Mapping): + raise TypeError(f"Structural freeze omits {name}.") + verified_artifact_path(descriptor, repo_root=repo_root, label=f"Structural {name}") + + +def _verify_freeze( + freeze_path: Path, + *, + identity: Mapping[str, Any], + scenario_ids: tuple[str, ...], + repo_root: Path, +) -> dict[str, Any]: + freeze = _load_json_object(freeze_path, label="Structural freeze") + _require_freeze_contract(freeze, identity=identity, scenario_ids=scenario_ids) + _verify_scenario_artifacts(freeze, scenario_ids=scenario_ids, repo_root=repo_root) + _verify_freeze_sources(freeze, repo_root=repo_root) + return freeze + + +def _verified_summary_artifacts(summary: Mapping[str, Any], *, repo_root: Path) -> dict[str, Path]: + artifacts = summary.get("artifacts") + if not isinstance(artifacts, Mapping) or set(artifacts) != set(SUMMARY_ARTIFACTS): + raise RuntimeError("Structural evaluation artifact inventory changed.") + verified: dict[str, Path] = {} + for name, descriptor in artifacts.items(): + if not isinstance(descriptor, Mapping): + raise TypeError(f"Structural evaluation descriptor {name!r} is invalid.") + verified[str(name)] = verified_artifact_path( + descriptor, + repo_root=repo_root, + label=f"Structural evaluation {name}", + ) + return verified + + +def _scenario_frame(config: Mapping[str, Any]) -> pd.DataFrame: + scenarios = declared_scenarios(config) + frame = pd.DataFrame(scenarios).rename( + columns={ + "scenario_budget": "budget", + "scenario_purpose_cap": "purpose_cap", + "scenario_lgd": "lgd", + "scenario_is_baseline": "is_baseline", + } + ) + return frame.loc[:, list(SCENARIO_COLUMNS)].sort_values("scenario_id").reset_index(drop=True) + + +def _require_scenario_columns( + frame: pd.DataFrame, + expected: pd.DataFrame, + *, + label: str, +) -> None: + require_exact_frame( + frame.loc[:, list(SCENARIO_COLUMNS)].drop_duplicates(), + expected, + keys=("scenario_id",), + label=label, + ) + + +def _validate_direction_counts(summary: pd.DataFrame, directions: pd.DataFrame) -> None: + counts = ( + directions.groupby(["scenario_id", "metric", "direction"], observed=True, sort=True) + .size() + .to_dict() + ) + for row_index in summary.index: + scenario_id = str(summary.at[row_index, "scenario_id"]) + for metric, columns in DIRECTION_SUMMARY_COLUMNS.items(): + total = 0 + for direction, column in columns.items(): + expected = int(counts.get((scenario_id, metric, direction), 0)) + actual = int(summary.at[row_index, column]) + if actual != expected: + raise RuntimeError( + f"Structural direction count changed for {scenario_id}/{metric}/{direction}." + ) + total += actual + if total != 48: + raise RuntimeError(f"Structural scenario {scenario_id}/{metric} is incomplete.") + + +def _validate_summary_bounds(summary: pd.DataFrame, directions: pd.DataFrame) -> None: + mappings = { + "standardized_payoff": ("payoff_lower_min", "payoff_upper_max"), + "funded_default": ("default_lower_min", "default_upper_max"), + "funded_binary_miscoverage": ("miscoverage_lower_min", "miscoverage_upper_max"), + } + for metric, (lower_column, upper_column) in mappings.items(): + scoped = directions.loc[directions["metric"].eq(metric)] + derived = ( + scoped.groupby("scenario_id", observed=True, sort=True) + .agg(**{lower_column: ("lower", "min"), upper_column: ("upper", "max")}) + .reset_index() + ) + require_exact_frame( + summary.loc[:, ["scenario_id", lower_column, upper_column]], + derived, + keys=("scenario_id",), + label=f"Structural {metric} bounds", + ) + + +def _validate_activity_summary(summary: pd.DataFrame, activity: pd.DataFrame) -> None: + value_columns = [column for column in activity if column not in SCENARIO_COLUMNS] + renamed = {column: f"activity_{column}" for column in value_columns} + expected = activity.loc[:, ["scenario_id", *value_columns]].rename(columns=renamed) + require_exact_frame( + summary.loc[:, ["scenario_id", *renamed.values()]], + expected, + keys=("scenario_id",), + label="Structural allocation-activity summary", + ) + + +def _validate_frames( + frames: Mapping[str, pd.DataFrame], + *, + expected_scenarios: pd.DataFrame, + reference_two_ruler: pd.DataFrame, +) -> None: + scenario_ids = tuple(expected_scenarios["scenario_id"].astype(str)) + summary = frames["scenario_summary"] + activity = frames["allocation_activity"] + contrasts = frames["window_contrasts"] + directions = frames["metric_directions"] + + require_exact_grid(summary, domains={"scenario_id": scenario_ids}, label="structural summary") + require_exact_grid(activity, domains={"scenario_id": scenario_ids}, label="structural activity") + _require_scenario_columns(summary, expected_scenarios, label="structural summary scenarios") + _require_scenario_columns(activity, expected_scenarios, label="structural activity scenarios") + require_exact_grid( + contrasts, + domains={ + "scenario_id": scenario_ids, + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + }, + label="structural window contrasts", + ) + require_exact_grid( + directions, + domains={ + "scenario_id": scenario_ids, + "window_id": WINDOW_IDS, + "ruler": RULERS, + "coordinate": COORDINATES, + "metric": METRICS, + }, + label="structural metric directions", + ) + if not set(directions["direction"]).issubset(DIRECTIONS): + raise RuntimeError("Structural direction vocabulary changed.") + require_finite(directions, ("lower", "upper"), label="structural metric directions") + if not directions["lower"].le(directions["upper"]).all(): + raise RuntimeError("Structural metric bounds are reversed.") + if bool(contrasts["causal_interpretation"].any()): + raise RuntimeError("Structural contrasts are incorrectly marked causal.") + require_finite( + activity, + ( + "purpose_cap_binding_share", + "frontier_constraint_binding_share", + "hhi_mean", + "maximum_loan_weight", + "maximum_purpose_share", + ), + label="structural allocation activity", + ) + if not ( + activity["purpose_cap_binding_share"].between(0.0, 1.0).all() + and activity["frontier_constraint_binding_share"].between(0.0, 1.0).all() + ): + raise RuntimeError("Structural binding shares lie outside [0, 1].") + if not activity["portfolios"].eq(1440).all(): + raise RuntimeError("Structural scenario portfolio census changed.") + _validate_activity_summary(summary, activity) + _validate_direction_counts(summary, directions) + _validate_summary_bounds(summary, directions) + + baseline = contrasts.loc[contrasts["scenario_is_baseline"]].drop( + columns=list(SCENARIO_TAG_COLUMNS) + ) + reference_only = set(reference_two_ruler.columns).difference(baseline.columns) + if reference_only.difference(REFERENCE_ONLY_IDENTIFICATION_WIDTH_COLUMNS): + raise RuntimeError( + "The active two-ruler reference added undeclared columns beyond identification widths." + ) + missing_reference = set(baseline.columns).difference(reference_two_ruler.columns) + if missing_reference: + raise RuntimeError( + f"The active two-ruler reference dropped structural baseline columns: " + f"{sorted(missing_reference)}." + ) + require_exact_frame( + baseline, + reference_two_ruler.loc[:, baseline.columns], + keys=("window_id", "ruler", "coordinate"), + label="structural baseline two-ruler reconciliation", + ) + + +def _group_scalar_map( + frame: pd.DataFrame, + *, + group_column: str, + value_column: str, + key_format: str, +) -> dict[str, float]: + result: dict[str, float] = {} + for group, scoped in frame.groupby(group_column, observed=True, sort=True): + values = scoped[value_column].drop_duplicates() + if len(values) != 1: + raise RuntimeError(f"Structural {value_column} varies within {group_column}={group}.") + result[format(float(str(group)), key_format)] = float(values.iloc[0]) + return result + + +def _group_rounded_max_map( + frame: pd.DataFrame, + *, + group_column: str, + value_column: str, + key_format: str, + decimals: int, +) -> dict[str, float]: + return { + format(float(str(group)), key_format): round(float(scoped[value_column].max()), decimals) + for group, scoped in frame.groupby(group_column, observed=True, sort=True) + } + + +def structural_findings(summary: pd.DataFrame, directions: pd.DataFrame) -> dict[str, Any]: + """Derive nonselective invariants over all 36 declared scenarios.""" + payoff_adverse = DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["gamma_1_lower"] + payoff_favorable = DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["gamma_1_higher"] + default_adverse = DIRECTION_SUMMARY_COLUMNS["funded_default"]["gamma_1_higher"] + default_favorable = DIRECTION_SUMMARY_COLUMNS["funded_default"]["gamma_1_lower"] + miscoverage_adverse = DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["gamma_1_higher"] + miscoverage_favorable = DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["gamma_1_lower"] + universally_favorable = ( + summary[payoff_favorable].eq(48) + & summary[default_favorable].eq(48) + & summary[miscoverage_favorable].eq(48) + ) + universally_adverse = ( + summary[payoff_adverse].eq(48) + & summary[default_adverse].eq(48) + & summary[miscoverage_adverse].eq(48) + ) + direction_totals = ( + directions.groupby(["metric", "direction"], observed=True, sort=True).size().to_dict() + ) + return { + "scenario_count": int(len(summary)), + "complete_cartesian_grid": int(len(summary)) == 36, + "every_scenario_has_adverse_default_and_miscoverage_cells": bool( + summary[default_adverse].gt(0).all() and summary[miscoverage_adverse].gt(0).all() + ), + "minimum_adverse_default_cells_per_scenario": int(summary[default_adverse].min()), + "minimum_adverse_miscoverage_cells_per_scenario": int(summary[miscoverage_adverse].min()), + "universally_favorable_scenarios": int(universally_favorable.sum()), + "universally_adverse_scenarios": int(universally_adverse.sum()), + "scenarios_with_any_favorable_payoff_cell": int(summary[payoff_favorable].gt(0).sum()), + "scenarios_with_any_favorable_default_cell": int(summary[default_favorable].gt(0).sum()), + "scenarios_with_any_favorable_miscoverage_cell": int( + summary[miscoverage_favorable].gt(0).sum() + ), + "portfolios_per_scenario": int(summary["activity_portfolios"].iloc[0]), + "purpose_cap_binding_share_by_cap": _group_scalar_map( + summary, + group_column="purpose_cap", + value_column="activity_purpose_cap_binding_share", + key_format=".2f", + ), + "frontier_constraint_binding_share_by_budget": _group_scalar_map( + summary, + group_column="budget", + value_column="activity_frontier_constraint_binding_share", + key_format=".0f", + ), + "maximum_loan_weight_by_budget": _group_rounded_max_map( + summary, + group_column="budget", + value_column="activity_maximum_loan_weight", + key_format=".0f", + decimals=12, + ), + "direction_totals": { + metric: { + direction: int(direction_totals.get((metric, direction), 0)) + for direction in DIRECTIONS + } + for metric in METRICS + }, + "interpretation": ( + "The complete structural grid rules out universal favorable direction, while " + "one-sided favorable cells in many scenarios also rule out universal adversity." + ), + } + + +def structural_publication_table(evidence: StructuralSensitivityEvidence) -> pd.DataFrame: + """Return every scenario with complete metric-direction counts.""" + summary = evidence.frames["scenario_summary"] + columns = [ + *SCENARIO_COLUMNS, + DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["gamma_1_lower"], + DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["gamma_1_higher"], + DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["crosses_zero"], + DIRECTION_SUMMARY_COLUMNS["standardized_payoff"]["exact_zero"], + DIRECTION_SUMMARY_COLUMNS["funded_default"]["gamma_1_higher"], + DIRECTION_SUMMARY_COLUMNS["funded_default"]["gamma_1_lower"], + DIRECTION_SUMMARY_COLUMNS["funded_default"]["crosses_zero"], + DIRECTION_SUMMARY_COLUMNS["funded_default"]["exact_zero"], + DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["gamma_1_higher"], + DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["gamma_1_lower"], + DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["crosses_zero"], + DIRECTION_SUMMARY_COLUMNS["funded_binary_miscoverage"]["exact_zero"], + "activity_purpose_cap_binding_share", + "activity_frontier_constraint_binding_share", + "activity_portfolios", + "activity_maximum_loan_weight", + ] + return ( + summary.loc[:, columns] + .sort_values(["budget", "purpose_cap", "lgd"], kind="stable") + .reset_index(drop=True) + ) + + +def load_structural_sensitivity_evidence( + summary_path: Path, + *, + freeze_path: Path, + config_path: Path, + identity: Mapping[str, Any], + repo_root: Path, + reference_two_ruler: pd.DataFrame, +) -> StructuralSensitivityEvidence: + """Load, hash-verify, and reconcile the complete structural V6 result.""" + config = _load_config(config_path) + _require_identity( + config, + identity, + label="Structural config", + fields=("run_tag", "protocol_tag"), + ) + expected_scenarios = _scenario_frame(config) + scenario_ids = tuple(expected_scenarios["scenario_id"].astype(str)) + freeze = _verify_freeze( + freeze_path, + identity=identity, + scenario_ids=scenario_ids, + repo_root=repo_root, + ) + summary = _load_json_object(summary_path, label="Structural evaluation summary") + _require_identity(summary, identity, label="Structural evaluation") + if summary.get("status") != "complete_post_freeze_structural_sensitivity_evaluation": + raise RuntimeError("Structural sensitivity evaluation is incomplete.") + if int(summary.get("scenario_count", -1)) != len(scenario_ids): + raise RuntimeError("Structural evaluation scenario count changed.") + if summary.get("selection") != { + "scenario": None, + "budget": None, + "purpose_cap": None, + "lgd": None, + }: + raise RuntimeError("Structural evaluation reports a selected scenario.") + if ( + summary.get("protected_stages_run") != [] + or summary.get("protected_artifacts_written") != [] + ): + raise RuntimeError("Structural evaluation reports a protected-stage side effect.") + if summary.get("baseline_reconciliation_maxima") != { + "realized_payoff_difference_lower": 0.0, + "realized_payoff_difference_upper": 0.0, + "weighted_default_difference_lower": 0.0, + "weighted_default_difference_upper": 0.0, + "weighted_miscoverage_difference_lower": 0.0, + "weighted_miscoverage_difference_upper": 0.0, + }: + raise RuntimeError("Structural baseline reconciliation is no longer exact.") + + paths = _verified_summary_artifacts(summary, repo_root=repo_root) + frames = {name: pd.read_parquet(path) for name, path in paths.items()} + _validate_frames( + frames, + expected_scenarios=expected_scenarios, + reference_two_ruler=reference_two_ruler, + ) + findings = structural_findings(frames["scenario_summary"], frames["metric_directions"]) + if findings["complete_cartesian_grid"] is not True: + raise RuntimeError("Structural sensitivity grid is incomplete.") + return StructuralSensitivityEvidence( + config=config, + freeze=freeze, + summary=summary, + frames=frames, + findings=findings, + ) diff --git a/src/ijds_audit/structural_sensitivity.py b/src/ijds_audit/structural_sensitivity.py new file mode 100644 index 0000000..f4a9bfa --- /dev/null +++ b/src/ijds_audit/structural_sensitivity.py @@ -0,0 +1,135 @@ +"""Declared scenario grid and diagnostics for IJDS portfolio structure.""" + +from __future__ import annotations + +from collections.abc import Mapping +from itertools import product +from typing import Any + +import numpy as np +import pandas as pd + + +def declared_scenarios(config: Mapping[str, Any]) -> tuple[dict[str, Any], ...]: + """Return the complete budget-purpose-LGD Cartesian product.""" + grid = config["structural_grid"] + budgets = tuple(float(value) for value in grid["budgets"]) + purpose_caps = tuple(float(value) for value in grid["purpose_caps"]) + lgds = tuple(float(value) for value in grid["lgds"]) + if budgets != (500_000.0, 1_000_000.0, 2_000_000.0): + raise ValueError("Budget grid must remain 0.5/1/2 million dollars.") + if purpose_caps != (0.20, 0.25, 0.30, 1.00): + raise ValueError("Purpose-cap grid must remain .20/.25/.30/1.00.") + if lgds != (0.25, 0.45, 0.65): + raise ValueError("LGD grid must remain .25/.45/.65.") + scenarios: list[dict[str, Any]] = [] + for budget, purpose_cap, lgd in product(budgets, purpose_caps, lgds): + scenarios.append( + { + "scenario_id": ( + f"b{round(budget / 1000):04d}k_" + f"p{round(purpose_cap * 100):03d}_l{round(lgd * 100):03d}" + ), + "budget": budget, + "purpose_cap": purpose_cap, + "lgd": lgd, + "is_baseline": bool(budget == 1_000_000.0 and purpose_cap == 0.25 and lgd == 0.45), + } + ) + if len(scenarios) != 36 or sum(item["is_baseline"] for item in scenarios) != 1: + raise RuntimeError("Structural scenario grid is incomplete or has no unique baseline.") + return tuple(scenarios) + + +def allocation_activity( + records: pd.DataFrame, + allocations: pd.DataFrame, + *, + scenario: Mapping[str, Any], + allocation_tolerance: float, +) -> dict[str, Any]: + """Summarize binding constraints and diversification for one scenario.""" + keys = ["window_id", "role", "period", "policy_label"] + if bool(records.duplicated(keys).any()): + raise RuntimeError("Structural solve records are not unique by portfolio.") + grouped = allocations.groupby(keys, observed=True, sort=False) + total = grouped["exposure"].sum().rename("total_exposure") + purpose = ( + allocations.groupby([*keys, "purpose"], observed=True, sort=False)["exposure"] + .sum() + .groupby(level=keys) + .max() + .rename("maximum_purpose_exposure") + ) + activity = pd.concat([total, purpose], axis=1) + activity["maximum_purpose_share"] = ( + activity["maximum_purpose_exposure"] / activity["total_exposure"] + ) + cap = float(scenario["purpose_cap"]) + purpose_binding = np.isclose( + activity["maximum_purpose_share"].to_numpy(dtype=float), + cap, + atol=1e-8, + rtol=0.0, + ) + fractions = allocations["allocation_fraction"].to_numpy(dtype=float) + partial = (fractions > float(allocation_tolerance)) & ( + fractions < 1.0 - float(allocation_tolerance) + ) + normalized_slack = records["constraint_slack"].abs().to_numpy(dtype=float) + ruler = records["frontier_ruler"].astype("string") + binding_tolerance = np.where(ruler.eq("normalized_score"), 1e-8, 1e-5) + weights = allocations["weight"].to_numpy(dtype=float) + hhi = ( + allocations.assign(_weight_squared=np.square(weights)) + .groupby(keys, observed=True, sort=False)["_weight_squared"] + .sum() + .to_numpy(dtype=float) + ) + return { + **dict(scenario), + "portfolios": int(len(records)), + "funded_rows": int(len(allocations)), + "purpose_cap_binding_portfolios": int(purpose_binding.sum()), + "purpose_cap_binding_share": float(purpose_binding.mean()), + "frontier_constraint_binding_portfolios": int( + (normalized_slack <= binding_tolerance).sum() + ), + "frontier_constraint_binding_share": float((normalized_slack <= binding_tolerance).mean()), + "partial_funded_rows": int(partial.sum()), + "partial_rows_per_portfolio_mean": float(partial.sum() / len(records)), + "funded_loans_per_portfolio_min": int(records["n_positive_exposure"].min()), + "funded_loans_per_portfolio_mean": float(records["n_positive_exposure"].mean()), + "funded_loans_per_portfolio_max": int(records["n_positive_exposure"].max()), + "hhi_min": float(hhi.min()), + "hhi_mean": float(hhi.mean()), + "hhi_max": float(hhi.max()), + "maximum_loan_weight": float(weights.max()), + "maximum_purpose_share": float(activity["maximum_purpose_share"].max()), + } + + +def scenario_result_summary( + contrasts: pd.DataFrame, + directions: pd.DataFrame, + *, + scenario: Mapping[str, Any], +) -> dict[str, Any]: + """Build one row containing every metric hull and direction count.""" + row: dict[str, Any] = {**dict(scenario)} + metrics = { + "payoff": ("realized_payoff_difference_lower", "realized_payoff_difference_upper"), + "default": ("weighted_default_difference_lower", "weighted_default_difference_upper"), + "miscoverage": ( + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + ), + } + for label, (lower, upper) in metrics.items(): + row[f"{label}_lower_min"] = float(contrasts[lower].min()) + row[f"{label}_upper_max"] = float(contrasts[upper].max()) + for metric, frame in directions.groupby("metric", observed=True, sort=True): + counts = frame["direction"].value_counts().to_dict() + for direction in ("gamma_1_lower", "gamma_1_higher", "crosses_zero", "exact_zero"): + row[f"{metric}_{direction}_cells"] = int(counts.get(direction, 0)) + return row diff --git a/src/ijds_challengers/__init__.py b/src/ijds_challengers/__init__.py new file mode 100644 index 0000000..f81f9c5 --- /dev/null +++ b/src/ijds_challengers/__init__.py @@ -0,0 +1 @@ +"""Isolated IJDS challenger methods that do not mutate active V4 evidence.""" diff --git a/src/ijds_challengers/archive.py b/src/ijds_challengers/archive.py new file mode 100644 index 0000000..c41a288 --- /dev/null +++ b/src/ijds_challengers/archive.py @@ -0,0 +1,111 @@ +"""Verified outcome-free parent loading for isolated IJDS challengers.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import pandas as pd + +from src.evaluation.standardized_credit_payoff import contractual_rate_decimal +from src.ijds_audit.protocol import verified_freeze_artifact_paths +from src.utils.isolated_experiment import relative_artifact_descriptor, sha256_file + + +def verified_parent_artifacts( + config: Mapping[str, Any], + *, + repo_root: Path, +) -> tuple[dict[str, Path], dict[str, Any]]: + """Verify the immutable parent freeze and return all referenced paths.""" + parent = config["parent"] + descriptor = parent["protocol_freeze"] + freeze_path = (repo_root / str(descriptor["path"])).resolve() + actual = relative_artifact_descriptor(freeze_path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor[field]: + raise RuntimeError(f"Parent protocol freeze mismatch for {field}.") + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + if not isinstance(freeze, dict): + raise TypeError("Parent protocol freeze must be a JSON object.") + expected = { + "run_tag": str(parent["run_tag"]), + "protocol_tag": str(parent["protocol_tag"]), + "protocol_commit": str(parent["protocol_commit"]), + "status": "outcome_free_allocations_frozen_before_archive_outcome_join", + } + for field, value in expected.items(): + if freeze.get(field) != value: + raise RuntimeError(f"Parent freeze field mismatch: {field}.") + if freeze.get("outcome_columns_passed_to_policy_or_comparator") != []: + raise RuntimeError("Parent outcome-free freeze reports an outcome column.") + return verified_freeze_artifact_paths(freeze, repo_root=repo_root), freeze + + +def load_outcome_free_decision_base( + *, + scores_path: Path, + raw_path: Path, + config: Mapping[str, Any], +) -> pd.DataFrame: + """Join frozen roles/scores to an explicit non-outcome raw-column allowlist.""" + source = config["source_ingest"] + if sha256_file(raw_path) != str(source["raw_sha256"]): + raise RuntimeError("Raw archive hash does not match the locked source.") + scores = pd.read_parquet(scores_path) + roles = {str(value) for value in config["frontier"]["roles"]} + score_frame = scores.loc[scores["design_split"].isin(roles)].copy() + score_frame["id"] = score_frame["id"].astype("string") + if bool(score_frame["id"].duplicated().any()): + raise RuntimeError("Frozen decision scores contain duplicate IDs.") + target_ids = set(score_frame["id"].astype(str)) + allowed = [str(value) for value in source["allowed_raw_columns"]] + pieces: list[pd.DataFrame] = [] + for chunk in pd.read_csv( + raw_path, + usecols=lambda column: str(column) in allowed, + dtype={"id": "string"}, + chunksize=int(source["chunksize"]), + low_memory=False, + ): + selected = chunk.loc[chunk["id"].astype(str).isin(target_ids)] + if not selected.empty: + pieces.append(selected) + if not pieces: + raise RuntimeError("No raw decision rows matched the frozen score universe.") + raw = pd.concat(pieces, ignore_index=True) + raw["id"] = raw["id"].astype("string") + if bool(raw["id"].duplicated().any()) or set(raw["id"].astype(str)) != target_ids: + raise RuntimeError("Raw decision fields do not align one-to-one with frozen scores.") + frame = score_frame.merge(raw, on="id", how="left", validate="one_to_one") + frame = frame.rename(columns={"pd_catboost_platt": "pd_point"}) + frame["issue_d"] = pd.to_datetime(frame["issue_d"]) + frame["loan_amnt"] = pd.to_numeric(frame["loan_amnt"], errors="raise").astype(float) + frame["purpose"] = frame["purpose"].astype("string").fillna("unknown") + frame["contractual_rate"] = contractual_rate_decimal(frame["int_rate"]) + frame = frame.drop(columns=["int_rate", "pd_numeric_logistic_platt"]) + tokens = tuple(str(token).casefold() for token in source["forbidden_tokens"]) + forbidden = [ + str(column) + for column in frame.columns + if any(token in str(column).casefold() for token in tokens) + ] + if forbidden: + raise RuntimeError(f"Decision base contains forbidden columns: {forbidden}.") + counts = frame["design_split"].value_counts().to_dict() + expected = {"policy_development": 94_885, "primary_oot": 376_890} + if counts != expected: + raise RuntimeError(f"Decision role census changed: {counts}.") + return frame.sort_values(["issue_d", "id"], kind="mergesort").reset_index(drop=True) + + +def monthly_frames(frame: pd.DataFrame, role: str) -> tuple[tuple[str, pd.DataFrame], ...]: + """Return stable monthly copies for one declared design role.""" + selected = frame.loc[frame["design_split"].eq(role)].copy() + periods = selected["issue_d"].dt.to_period("M") + return tuple( + (str(period), selected.loc[periods.eq(period)].copy()) + for period in sorted(periods.unique()) + ) diff --git a/src/ijds_challengers/config.py b/src/ijds_challengers/config.py new file mode 100644 index 0000000..e169cd1 --- /dev/null +++ b/src/ijds_challengers/config.py @@ -0,0 +1,157 @@ +"""Strict configuration contract for the normalized/objective frontier.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import yaml + + +def _exact_floats(values: Any, expected: list[float], *, label: str) -> None: + if [float(value) for value in values] != expected: + raise ValueError(f"The locked {label} changed.") + + +def _require_true(mapping: Any, names: set[str], *, label: str) -> None: + if not isinstance(mapping, dict) or any(mapping.get(name) is not True for name in names): + raise ValueError(f"Every {label} must remain enabled.") + + +def _validate_frontier(frontier: Any) -> None: + if not isinstance(frontier, dict): + raise TypeError("frontier must be a mapping.") + _exact_floats(frontier["gamma_grid"], [0.0, 0.25, 0.5, 0.75, 1.0], label="gamma path") + _exact_floats(frontier["coordinate_grid"], [0.25, 0.5, 0.75], label="coordinate grid") + _exact_floats(frontier["endpoint_contrast"], [1.0, 0.0], label="endpoint contrast") + if frontier["rulers"] != { + "primary": "objective_matched", + "secondary": "normalized_score", + }: + raise ValueError("The declared frontier ruler hierarchy changed.") + if frontier["roles"] != ["policy_development", "primary_oot"]: + raise ValueError("The outcome-free frontier roles changed.") + if (int(frontier["expected_development_months"]), int(frontier["expected_primary_months"])) != ( + 11, + 15, + ) or int(frontier["expected_windows"]) != 8: + raise ValueError("The locked window or month census changed.") + objective_optimum = frontier["objective_optimum"] + if objective_optimum.get("diagnostic") != "nonbasic_reduced_costs_plus_reversed_id_order": + raise ValueError("The objective-optimum uniqueness diagnostic changed.") + if any( + float(objective_optimum[name]) <= 0.0 for name in ("dual_tolerance", "primal_tolerance") + ): + raise ValueError("Objective-optimum basis tolerances must be positive.") + + +def _validate_source(source: Any) -> None: + if not isinstance(source, dict): + raise TypeError("source_ingest must be a mapping.") + if source["allowed_raw_columns"] != ["id", "loan_amnt", "int_rate", "purpose"]: + raise ValueError("The outcome-free raw-column allowlist changed.") + forbidden = tuple(str(token).casefold() for token in source["forbidden_tokens"]) + if any( + token in str(column).casefold() + for column in source["allowed_raw_columns"] + for token in forbidden + ): + raise ValueError("The raw-column allowlist contains an outcome-like name.") + + +def _validate_solver(solver: Any) -> None: + if not isinstance(solver, dict): + raise TypeError("solver must be a mapping.") + if solver["primary"] != "highspy_exact_budget_simplex" or int(solver["threads"]) != 1: + raise ValueError("The deterministic primary solver contract changed.") + if float(solver["budget_residual_tolerance_dollars"]) != 1.0e-4: + raise ValueError("The V1c budget-reconciliation correction changed.") + validation = solver["independent_validation"] + if validation["solver"] != "ortools_glop" or validation["periods"] != [ + "2016-04", + "2016-11", + "2017-06", + ]: + raise ValueError("The independent solver validation contract changed.") + positive = ( + solver["allocation_tolerance"], + solver["budget_residual_tolerance_dollars"], + solver["order_exposure_distance_tolerance"], + solver["order_objective_tolerance_dollars"], + solver["endpoint_pair_degeneracy_tolerance"], + validation["objective_rate_tolerance"], + validation["weighted_score_tolerance"], + ) + if any(float(value) <= 0.0 for value in positive): + raise ValueError("Every numerical solver tolerance must be positive.") + + +def load_frontier_config(path: Path) -> dict[str, Any]: + """Load and validate the locked outcome-free V1b challenger config.""" + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Frontier config must be a YAML mapping.") + required = { + "protocol_status", + "protocol_tag", + "run_tag", + "lineage", + "parent", + "source_ingest", + "frontier", + "solver", + "claim_boundary", + "stop_rules", + "output", + } + missing = sorted(required.difference(payload)) + if missing: + raise ValueError(f"Frontier config is missing sections: {missing}.") + if payload["protocol_status"] != "locked_outcome_free_frontier_before_execution": + raise ValueError("Frontier protocol is not locked for outcome-free execution.") + lineage = payload["lineage"] + expected_lineage = { + "original_failed_protocol_tag": ( + "protocol/ijds-normalized-objective-frontier-2026-07-12-v1" + ), + "objective_tie_erratum_tag": ("protocol/ijds-normalized-objective-frontier-2026-07-13-v1b"), + "objective_tie_correction": ( + "replace_slack_floor_score_span_with_basis_reduced_cost_and_id_reversal" + ), + "v1b_stop": "maximum_budget_residual_6.366e-6_exceeded_1e-6", + "budget_correction": ("align_final_budget_tolerance_with_solver_wrappers_at_1e-4_dollars"), + } + if lineage != expected_lineage: + raise ValueError("The V1-to-V1c numerical correction lineage changed.") + _validate_frontier(payload["frontier"]) + _validate_source(payload["source_ingest"]) + _validate_solver(payload["solver"]) + boundary = payload["claim_boundary"] + if boundary.get("outcome_columns_passed") != []: + raise ValueError("The V1 frontier cannot accept outcome columns.") + required_true = { + "no_policy_selection", + "no_policy_winner", + "no_conformal_guarantee_repair", + "no_equal_true_risk_claim", + "no_equal_shadow_price_claim", + "no_causal_claim", + "no_submission_freeze", + } + _require_true(boundary, required_true, label="V1 frontier claim boundary") + _require_true( + payload["stop_rules"], + { + "stop_on_incomplete_cell", + "stop_on_score_range_failure", + "stop_on_objective_range_failure", + "stop_on_objective_optimum_tie", + "stop_on_order_sensitivity", + "stop_on_independent_solver_mismatch", + "stop_before_outcomes_if_endpoint_allocations_all_identical", + }, + label="V1 frontier stop rule", + ) + if payload["output"].get("immutability") != "hard_no_overwrite_choose_fresh_run_tag": + raise ValueError("Frontier outputs must remain immutable.") + return cast(dict[str, Any], payload) diff --git a/src/ijds_challengers/evaluation.py b/src/ijds_challengers/evaluation.py new file mode 100644 index 0000000..645e96f --- /dev/null +++ b/src/ijds_challengers/evaluation.py @@ -0,0 +1,336 @@ +"""Hash-verified post-freeze evaluation for the two-ruler challenger.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from src.evaluation.policy_contrast_bounds import sharp_policy_contrast_bounds +from src.utils.isolated_experiment import relative_artifact_descriptor + + +@dataclass(frozen=True) +class FrozenFrontier: + """Verified V1c paths and metadata available before the outcome join.""" + + artifacts: dict[str, Path] + freeze: dict[str, Any] + summary: dict[str, Any] + + +def verify_frontier_freeze( + config: Mapping[str, Any], + *, + repo_root: Path, +) -> FrozenFrontier: + """Verify the exact V1c freeze and every outcome-free artifact descriptor.""" + source = config["source_frontier"] + descriptor = source["freeze"] + freeze_path = (repo_root / str(descriptor["path"])).resolve() + actual = relative_artifact_descriptor(freeze_path, repo_root=repo_root) + if actual != descriptor: + raise RuntimeError("The V1c freeze descriptor does not match the V2 protocol.") + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + if not isinstance(freeze, dict): + raise TypeError("The V1c freeze must be a JSON object.") + expected = { + "status": str(source["status"]), + "run_tag": str(source["run_tag"]), + "protocol_tag": str(source["protocol_tag"]), + "protocol_commit": str(source["protocol_commit"]), + } + for field, value in expected.items(): + if freeze.get(field) != value: + raise RuntimeError(f"V1c freeze field mismatch: {field}.") + if freeze.get("outcome_columns_passed_to_frontier") != []: + raise RuntimeError("The V1c freeze reports outcome leakage.") + for field in ("policy_selection", "window_selection", "ruler_selection"): + if freeze.get(field) is not None: + raise RuntimeError(f"The V1c freeze reports forbidden selection: {field}.") + + expected_artifacts = { + "solve_records", + "allocations", + "endpoint_diagnostics", + "objective_optimum_diagnostics", + "order_sensitivity", + "independent_validation", + } + descriptors = freeze.get("outcome_free_artifacts") + if not isinstance(descriptors, dict) or set(descriptors) != expected_artifacts: + raise RuntimeError("The V1c outcome-free artifact inventory changed.") + artifacts: dict[str, Path] = {} + for name, artifact_descriptor in descriptors.items(): + path = (repo_root / str(artifact_descriptor["path"])).resolve() + if relative_artifact_descriptor(path, repo_root=repo_root) != artifact_descriptor: + raise RuntimeError(f"V1c artifact descriptor mismatch: {name}.") + artifacts[str(name)] = path + for name in ("summary", "execution_receipt"): + item = freeze.get(name) + if not isinstance(item, dict): + raise RuntimeError(f"V1c freeze is missing {name} metadata.") + path = (repo_root / str(item["path"])).resolve() + if relative_artifact_descriptor(path, repo_root=repo_root) != item: + raise RuntimeError(f"V1c {name} descriptor mismatch.") + summary_descriptor = freeze["summary"] + summary_path = (repo_root / str(summary_descriptor["path"])).resolve() + summary = json.loads(summary_path.read_text(encoding="utf-8")) + if not isinstance(summary, dict) or summary.get("status") != str(source["status"]): + raise RuntimeError("The V1c summary status is not evaluation eligible.") + expected_counts = config["evaluation"] + if int(summary["counts"]["solve_records"]) != int( + expected_counts["expected_solve_records"] + ) or int(summary["counts"]["funded_rows"]) != int(expected_counts["expected_funded_rows"]): + raise RuntimeError("The V1c summary census changed.") + return FrozenFrontier(artifacts=artifacts, freeze=freeze, summary=summary) + + +def validate_outcome_alignment( + allocations: pd.DataFrame, + outcomes: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> pd.DataFrame: + """Validate one ID/role/period outcome universe before portfolio evaluation.""" + if bool(outcomes["id"].duplicated().any()): + raise RuntimeError("The outcome universe contains duplicate IDs.") + observed = pd.to_numeric(outcomes["snapshot_default"], errors="coerce") + finite = observed.notna() + if not bool(observed.loc[finite].isin([0, 1]).all()): + raise RuntimeError("The outcome universe contains a nonbinary observed outcome.") + roles = [str(value) for value in config["evaluation"]["evaluated_roles"]] + selected = outcomes.loc[outcomes["role"].isin(roles)].copy() + counts = selected["role"].value_counts().to_dict() + expected = { + str(role): int(value) + for role, value in config["evaluation"]["expected_candidate_counts"].items() + } + if counts != expected: + raise RuntimeError(f"The V2 candidate outcome census changed: {counts}.") + + funded_keys = allocations[["id", "role", "period"]].drop_duplicates() + if bool(funded_keys["id"].duplicated().any()): + raise RuntimeError("A funded ID maps to multiple decision role/period keys.") + outcome_keys = outcomes[["id", "role", "period"]].rename( + columns={"role": "outcome_role", "period": "outcome_period"} + ) + aligned = funded_keys.merge(outcome_keys, on="id", how="left", validate="one_to_one") + if bool(aligned[["outcome_role", "outcome_period"]].isna().any().any()): + raise RuntimeError("At least one funded ID has no archive outcome row.") + if not bool( + aligned["role"].astype(str).eq(aligned["outcome_role"].astype(str)).all() + and aligned["period"].astype(str).eq(aligned["outcome_period"].astype(str)).all() + ): + raise RuntimeError("Funded role/period keys do not align with archive outcomes.") + + audit = ( + selected.groupby(["role", "snapshot_resolution"], observed=True, sort=True) + .agg(candidate_rows=("id", "size"), resolved_rows=("snapshot_default", "count")) + .reset_index() + ) + audit["unresolved_rows"] = audit["candidate_rows"] - audit["resolved_rows"] + return audit + + +def build_endpoint_contrasts( + joined_allocations: pd.DataFrame, + endpoint_diagnostics: pd.DataFrame, + *, + config: Mapping[str, Any], + lgd: float, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Build every locked monthly and complete-window gamma endpoint contrast.""" + evaluation = config["evaluation"] + role = str(evaluation["primary_contrast_role"]) + rulers = tuple(str(value) for value in evaluation["rulers"]) + coordinates = tuple(float(value) for value in evaluation["coordinates"]) + windows = tuple(sorted(endpoint_diagnostics["window_id"].astype(str).unique())) + monthly_rows: list[dict[str, Any]] = [] + window_rows: list[dict[str, Any]] = [] + for window_id in windows: + window_allocations = joined_allocations.loc[ + joined_allocations["window_id"].eq(window_id) & joined_allocations["role"].eq(role) + ] + window_structure = endpoint_diagnostics.loc[ + endpoint_diagnostics["window_id"].eq(window_id) & endpoint_diagnostics["role"].eq(role) + ] + for ruler in rulers: + for coordinate in coordinates: + policy_a = _policy_label(ruler, 1.0, coordinate) + policy_b = _policy_label(ruler, 0.0, coordinate) + pair = window_allocations.loc[ + window_allocations["policy_label"].isin([policy_a, policy_b]) + ] + _validate_policy_pair_attributes(pair) + structure = window_structure.loc[ + window_structure["ruler"].eq(ruler) + & np.isclose( + window_structure["coordinate"].to_numpy(dtype=float), + coordinate, + atol=0.0, + rtol=0.0, + ) + ] + if len(structure) != int(evaluation["expected_primary_months"]): + raise RuntimeError( + f"Endpoint structure is incomplete for {window_id}/{ruler}/{coordinate}." + ) + window_rows.append( + { + "window_id": window_id, + "ruler": ruler, + "coordinate": coordinate, + "months": int(len(structure)), + "nonidentical_months": int( + structure["normalized_exposure_distance"].gt(1.0e-6).sum() + ), + "monthly_exposure_distance_sum": float( + structure["normalized_exposure_distance"].sum() + ), + "monthly_exposure_distance_mean": float( + structure["normalized_exposure_distance"].mean() + ), + "monthly_exposure_distance_maximum": float( + structure["normalized_exposure_distance"].max() + ), + **sharp_policy_contrast_bounds( + pair, + policy_a=policy_a, + policy_b=policy_b, + role=role, + lgd=lgd, + ), + } + ) + for period, month_pair in pair.groupby("period", observed=True, sort=True): + period_structure = structure.loc[structure["period"].eq(str(period))] + if len(period_structure) != 1: + raise RuntimeError( + f"Monthly structure is incomplete for {window_id}/{period}." + ) + monthly_rows.append( + { + "window_id": window_id, + "period": str(period), + "ruler": ruler, + "coordinate": coordinate, + "normalized_exposure_distance": float( + period_structure["normalized_exposure_distance"].iloc[0] + ), + **sharp_policy_contrast_bounds( + month_pair, + policy_a=policy_a, + policy_b=policy_b, + role=role, + lgd=lgd, + ), + } + ) + return pd.DataFrame(window_rows), pd.DataFrame(monthly_rows) + + +def build_metric_directions( + window_contrasts: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> pd.DataFrame: + """Classify every predeclared metric bound without selecting a subset.""" + rows: list[dict[str, Any]] = [] + for contrast in window_contrasts.to_dict(orient="records"): + for metric, spec in config["metrics"].items(): + lower = float(contrast[str(spec["lower"])]) + upper = float(contrast[str(spec["upper"])]) + tolerance = float(spec["direction_tolerance"]) + rows.append( + { + "window_id": str(contrast["window_id"]), + "ruler": str(contrast["ruler"]), + "coordinate": float(contrast["coordinate"]), + "metric": str(metric), + "lower": lower, + "upper": upper, + "direction_tolerance": tolerance, + "direction": direction_from_bounds( + lower, + upper, + tolerance=tolerance, + ), + } + ) + return pd.DataFrame(rows) + + +def direction_from_bounds(lower: float, upper: float, *, tolerance: float) -> str: + """Classify one ordered interval with an explicit zero tolerance.""" + low = float(lower) + high = float(upper) + tol = float(tolerance) + if not np.isfinite(low) or not np.isfinite(high) or low > high + tol: + raise ValueError("Direction classification requires an ordered finite interval.") + if low > tol: + return "gamma_1_higher" + if high < -tol: + return "gamma_1_lower" + if abs(low) <= tol and abs(high) <= tol: + return "exact_zero" + return "crosses_zero" + + +def validate_complete_evaluation( + evaluated: pd.DataFrame, + joined: pd.DataFrame, + window_contrasts: pd.DataFrame, + monthly_contrasts: pd.DataFrame, + directions: pd.DataFrame, + *, + config: Mapping[str, Any], +) -> None: + """Fail unless every frozen portfolio and endpoint contrast is retained.""" + expected = config["evaluation"] + actual = { + "evaluated": len(evaluated), + "joined": len(joined), + "window": len(window_contrasts), + "monthly": len(monthly_contrasts), + "directions": len(directions), + } + required = { + "evaluated": int(expected["expected_solve_records"]), + "joined": int(expected["expected_funded_rows"]), + "window": int(expected["expected_window_contrasts"]), + "monthly": int(expected["expected_monthly_contrasts"]), + "directions": int(expected["expected_metric_directions"]), + } + if actual != required: + raise RuntimeError(f"The V2 evaluation census is incomplete: {actual} != {required}.") + if window_contrasts["window_id"].nunique() != int(expected["expected_windows"]): + raise RuntimeError("The V2 window census is incomplete.") + keys = ["window_id", "ruler", "coordinate"] + if bool(window_contrasts.duplicated(keys).any()): + raise RuntimeError("The V2 window contrast grid contains duplicates.") + direction_keys = [*keys, "metric"] + if bool(directions.duplicated(direction_keys).any()): + raise RuntimeError("The V2 metric-direction grid contains duplicates.") + + +def _policy_label(ruler: str, gamma: float, coordinate: float) -> str: + return f"{ruler}_g{round(gamma * 100):03d}_c{round(coordinate * 100):03d}" + + +def _validate_policy_pair_attributes(allocations: pd.DataFrame) -> None: + """Require loan attributes to agree across the two frozen endpoints.""" + attributes = [ + "contractual_rate", + "conformal_lower", + "conformal_upper", + "snapshot_default", + ] + counts = allocations.groupby("id", observed=True)[attributes].nunique(dropna=False) + if bool(counts.gt(1).any(axis=1).any()): + raise RuntimeError("Loan attributes disagree across frozen endpoint policies.") diff --git a/src/ijds_challengers/evaluation_config.py b/src/ijds_challengers/evaluation_config.py new file mode 100644 index 0000000..fb95a92 --- /dev/null +++ b/src/ijds_challengers/evaluation_config.py @@ -0,0 +1,183 @@ +"""Strict configuration contract for the post-freeze frontier evaluation.""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +from src.ijds_audit.config import load_config_payload + +EXPECTED_FREEZE_SHA256 = "7877c5e460772a0093e4132eaa542e9049f7ec15d2ddaa35c2df389892a0e185" + + +def load_v2_config(path: Path) -> dict[str, Any]: + """Load and validate the locked V2 outcome-evaluation config.""" + payload = load_config_payload(path) + if not isinstance(payload, dict): + raise TypeError("V2 config must be a YAML mapping.") + required = { + "schema_version", + "protocol_status", + "protocol_tag", + "run_tag", + "source_frontier", + "parent", + "evaluation", + "metrics", + "outcomes", + "claim_boundary", + "stop_rules", + "output", + } + missing = sorted(required.difference(payload)) + if missing: + raise ValueError(f"V2 config is missing sections: {missing}.") + if payload["protocol_status"] != "locked_post_freeze_outcome_evaluation": + raise ValueError("V2 protocol is not locked for post-freeze evaluation.") + _validate_source_frontier(payload["source_frontier"]) + _validate_evaluation(payload["evaluation"]) + _validate_metrics(payload["metrics"]) + _require_true( + payload["outcomes"], + {"no_outcome_refit", "no_outcome_resolve", "no_outcome_selection"}, + label="outcome boundary", + ) + _require_true( + payload["claim_boundary"], + { + "no_policy_selection", + "no_window_selection", + "no_ruler_selection", + "no_coordinate_selection", + "no_gamma_selection", + "no_policy_winner", + "no_causal_claim", + "no_conformal_guarantee_repair", + "no_selected_set_validity", + "no_submission_freeze", + }, + label="V2 claim boundary", + ) + _require_true( + payload["stop_rules"], + { + "stop_on_v1c_freeze_mismatch", + "stop_on_artifact_descriptor_mismatch", + "stop_on_outcome_alignment_failure", + "stop_on_incomplete_portfolio_evaluation", + "stop_on_incomplete_contrast_census", + "stop_on_nonbinary_observed_outcome", + "retain_all_predeclared_results", + }, + label="V2 stop rule", + ) + if payload["output"].get("immutability") != "hard_no_overwrite_choose_fresh_run_tag": + raise ValueError("V2 outputs must remain immutable.") + recovery = payload.get("endpoint_reason_recovery") + if recovery is not None: + if not isinstance(recovery, dict): + raise TypeError("Endpoint reason recovery must be a mapping.") + status = recovery.get("status") + if status == "reason_taxonomy_only_no_scientific_metric_change": + if recovery.get("require_exact_reference_column_equivalence") is not True: + raise ValueError("Endpoint reason recovery must require exact equivalence.") + elif status == "reason_taxonomy_only_machine_tolerance_recovery": + if recovery.get("require_exact_non_float_reference_equivalence") is not True: + raise ValueError("Endpoint recovery must retain exact non-floating equivalence.") + if recovery.get("equivalence_mode") != "exact_non_float_machine_tolerant_float": + raise ValueError("Endpoint recovery has an invalid equivalence mode.") + for field in ("float_atol", "float_rtol"): + tolerance = float(recovery.get(field, -1.0)) + if not math.isfinite(tolerance) or not 0.0 <= tolerance <= 1.0e-12: + raise ValueError(f"Endpoint recovery {field} exceeds its ceiling.") + else: + raise ValueError("Unexpected endpoint reason recovery status.") + if recovery.get("artifact_section") != "evaluation_artifacts": + raise ValueError("Two-ruler recovery must reference evaluation_artifacts.") + if not isinstance(recovery.get("reference_json"), dict): + raise KeyError("Two-ruler recovery requires a reference JSON descriptor.") + return payload + + +def _validate_source_frontier(source: Any) -> None: + if not isinstance(source, dict): + raise TypeError("source_frontier must be a mapping.") + expected = { + "run_tag": "ijds-normalized-objective-frontier-2026-07-13-v1c", + "protocol_tag": "protocol/ijds-normalized-objective-frontier-2026-07-13-v1c", + "protocol_commit": "46f4df915d38eb5a6cc144484c6e6fe56d8ed397", + "status": "outcome_free_frontiers_frozen_before_archive_outcome_join", + } + if any(source.get(field) != value for field, value in expected.items()): + raise ValueError("The locked V1c source identity changed.") + descriptor = source.get("freeze") + if not isinstance(descriptor, dict) or descriptor != { + "path": ( + "models/experiments/ijds_audit/" + "ijds-normalized-objective-frontier-2026-07-13-v1c/protocol_freeze.json" + ), + "bytes": 15192, + "sha256": EXPECTED_FREEZE_SHA256, + }: + raise ValueError("The locked V1c freeze descriptor changed.") + + +def _validate_evaluation(evaluation: Any) -> None: + if not isinstance(evaluation, dict): + raise TypeError("evaluation must be a mapping.") + exact = { + "evaluated_roles": ["policy_development", "primary_oot"], + "primary_contrast_role": "primary_oot", + "endpoint_contrast": [1.0, 0.0], + "rulers": ["objective_matched", "normalized_score"], + "coordinates": [0.25, 0.5, 0.75], + "expected_solve_records": 6240, + "expected_funded_rows": 622455, + "expected_window_contrasts": 48, + "expected_monthly_contrasts": 720, + "expected_metric_directions": 144, + "expected_windows": 8, + "expected_primary_months": 15, + "expected_candidate_counts": { + "policy_development": 94885, + "primary_oot": 376890, + }, + "outcome_join": "single_validated_join_after_hash_verified_v1c_freeze", + "unresolved_outcomes": "sharp_common_loanwise_assignment_on_funded_union", + } + if evaluation != exact: + raise ValueError("The complete V2 evaluation census changed.") + + +def _validate_metrics(metrics: Any) -> None: + expected_columns = { + "standardized_payoff": ( + "realized_payoff_difference_lower", + "realized_payoff_difference_upper", + 1.0e-4, + ), + "funded_default": ( + "weighted_default_difference_lower", + "weighted_default_difference_upper", + 1.0e-10, + ), + "funded_binary_miscoverage": ( + "weighted_miscoverage_difference_lower", + "weighted_miscoverage_difference_upper", + 1.0e-10, + ), + } + if not isinstance(metrics, dict) or set(metrics) != set(expected_columns): + raise ValueError("The V2 metric family changed.") + for metric, (lower, upper, tolerance) in expected_columns.items(): + spec = metrics[metric] + if spec.get("lower") != lower or spec.get("upper") != upper: + raise ValueError(f"The {metric} bound columns changed.") + if float(spec.get("direction_tolerance")) != tolerance: + raise ValueError(f"The {metric} direction tolerance changed.") + + +def _require_true(mapping: Any, names: set[str], *, label: str) -> None: + if not isinstance(mapping, dict) or any(mapping.get(name) is not True for name in names): + raise ValueError(f"Every {label} must remain enabled.") diff --git a/src/ijds_challengers/frontier.py b/src/ijds_challengers/frontier.py new file mode 100644 index 0000000..7652a0a --- /dev/null +++ b/src/ijds_challengers/frontier.py @@ -0,0 +1,294 @@ +"""Outcome-free score frontiers for the normalized-stringency challenger.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal + +import highspy +import numpy as np +import pandas as pd +from scipy.sparse import csc_matrix + + +@dataclass(frozen=True) +class ScoreFrontierSolution: + """Reconciled exact-budget solution on a score/objective frontier.""" + + allocation_fraction: np.ndarray + exposure: np.ndarray + objective_value: float + weighted_score: float + total_allocated: float + simplex_iterations: int + + +class ObjectiveFloorPortfolioSession: + """Warm-started score optimization under a changing plug-in objective floor.""" + + def __init__( + self, + frame: pd.DataFrame, + *, + score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + budget: float, + purpose_cap: float, + sense: Literal["minimize", "maximize"] = "minimize", + time_limit: int = 300, + threads: int = 1, + ) -> None: + n = len(frame) + if n == 0 or float(budget) <= 0.0: + raise ValueError("The frontier requires candidates and a positive budget.") + self.score = _validated_vector(score, name="score", n=n) + self.objective = _validated_vector(objective_rate, name="objective_rate", n=n) + if bool(np.any((self.score < 0.0) | (self.score > 1.0))): + raise ValueError("score must lie in [0, 1].") + if sense not in {"minimize", "maximize"}: + raise ValueError("sense must be 'minimize' or 'maximize'.") + self.budget = float(budget) + lp, self.objective_floor_row, self.amount = _objective_floor_lp( + frame, + self.score, + self.objective, + budget=self.budget, + purpose_cap=float(purpose_cap), + sense=sense, + ) + self.solver = highspy.Highs() + if hasattr(self.solver, "resetGlobalScheduler"): + self.solver.resetGlobalScheduler(True) + self.solver.setOptionValue("output_flag", False) + self.solver.setOptionValue("log_to_console", False) + self.solver.setOptionValue("solver", "simplex") + self.solver.setOptionValue("presolve", "on") + self.solver.setOptionValue("time_limit", float(time_limit)) + self.solver.setOptionValue("threads", max(1, int(threads))) + if self.solver.passModel(lp) != highspy.HighsStatus.kOk: + raise RuntimeError("HiGHS rejected the objective-floor portfolio LP.") + + def solve(self, objective_floor: float | None = None) -> ScoreFrontierSolution: + """Optimize score with an optional absolute plug-in objective floor.""" + lower = -highspy.kHighsInf if objective_floor is None else float(objective_floor) + status = self.solver.changeRowBounds( + int(self.objective_floor_row), + lower, + highspy.kHighsInf, + ) + if status != highspy.HighsStatus.kOk: + raise RuntimeError("HiGHS rejected the plug-in objective-floor update.") + if self.solver.run() == highspy.HighsStatus.kError: + raise RuntimeError("HiGHS failed while solving the objective-floor LP.") + model_status = self.solver.modelStatusToString(self.solver.getModelStatus()) + if "Optimal" not in str(model_status): + raise RuntimeError(f"Objective-floor LP is not optimal: {model_status}.") + fraction = np.clip( + np.asarray(self.solver.getSolution().col_value, dtype=float), + 0.0, + 1.0, + ) + exposure = self.amount * fraction + total = float(exposure.sum()) + if not np.isclose(total, self.budget, rtol=0.0, atol=1e-4): + raise RuntimeError(f"Objective-floor LP did not fill its budget: {total}.") + info = self.solver.getInfo() + return ScoreFrontierSolution( + allocation_fraction=fraction, + exposure=exposure, + objective_value=float(exposure @ self.objective), + weighted_score=float(exposure @ self.score / total), + total_allocated=total, + simplex_iterations=int(getattr(info, "simplex_iteration_count", 0) or 0), + ) + + +def normalized_score_cap( + *, + minimum_score: float, + score_at_objective: float, + coordinate: float, + minimum_range: float = 0.0, +) -> float: + """Map a unit coordinate to one score's attainable decision range.""" + lower = float(minimum_score) + upper = float(score_at_objective) + value = float(coordinate) + if not 0.0 <= value <= 1.0: + raise ValueError("Normalized score coordinate must lie in [0, 1].") + score_range = upper - lower + if score_range < float(minimum_range): + raise ValueError( + f"Normalized score range {score_range:.12g} is below {minimum_range:.12g}." + ) + return lower + value * score_range + + +def common_objective_target( + *, + minimum_objectives: Sequence[float] | np.ndarray, + objective_optimum: float, + coordinate: float, + minimum_range: float = 0.0, +) -> tuple[float, float]: + """Return the common lower endpoint and matched absolute objective target.""" + minima = np.asarray(minimum_objectives, dtype=float) + if minima.ndim != 1 or len(minima) == 0 or not bool(np.isfinite(minima).all()): + raise ValueError("minimum_objectives must be a nonempty finite vector.") + value = float(coordinate) + if not 0.0 <= value <= 1.0: + raise ValueError("Objective coordinate must lie in [0, 1].") + lower = float(minima.max()) + objective_range = float(objective_optimum) - lower + if objective_range < float(minimum_range): + raise ValueError( + f"Common objective range {objective_range:.12g} is below {minimum_range:.12g}." + ) + return lower, lower + value * objective_range + + +def normalized_exposure_distance( + first: Sequence[float] | np.ndarray, + second: Sequence[float] | np.ndarray, + *, + budget: float, +) -> float: + """Return turnover-like L1 exposure distance normalized by twice the budget.""" + left = np.asarray(first, dtype=float) + right = np.asarray(second, dtype=float) + if left.shape != right.shape or left.ndim != 1: + raise ValueError("Exposure vectors must be aligned one-dimensional arrays.") + if float(budget) <= 0.0: + raise ValueError("budget must be positive.") + return float(np.abs(left - right).sum() / (2.0 * float(budget))) + + +def solve_glop_portfolio( + frame: pd.DataFrame, + *, + score: Sequence[float] | np.ndarray, + objective_rate: Sequence[float] | np.ndarray, + budget: float, + purpose_cap: float, + mode: Literal["normalized_score", "objective_matched"], + threshold: float, +) -> ScoreFrontierSolution: + """Independently resolve one declared frontier cell with OR-Tools GLOP.""" + from ortools.linear_solver import pywraplp + + n = len(frame) + amount = _validated_vector(frame["loan_amnt"], name="loan_amnt", n=n) + score_array = _validated_vector(score, name="score", n=n) + objective = _validated_vector(objective_rate, name="objective_rate", n=n) + solver = pywraplp.Solver.CreateSolver("GLOP") + if solver is None: + raise RuntimeError("OR-Tools GLOP is unavailable.") + variables = [solver.NumVar(0.0, 1.0, f"x_{index}") for index in range(n)] + solver.Add(solver.Sum(float(amount[index]) * variables[index] for index in range(n)) == budget) + purposes = frame["purpose"].astype("string").fillna("unknown") + for purpose in sorted(purposes.unique()): + mask = purposes.eq(purpose).to_numpy(dtype=bool) + solver.Add( + solver.Sum(float(amount[index]) * variables[index] for index in np.flatnonzero(mask)) + <= float(purpose_cap) * float(budget) + ) + if mode == "normalized_score": + solver.Add( + solver.Sum( + float(amount[index] * score_array[index]) * variables[index] for index in range(n) + ) + <= float(threshold) * float(budget) + ) + solver.Maximize( + solver.Sum( + float(amount[index] * objective[index]) * variables[index] for index in range(n) + ) + ) + elif mode == "objective_matched": + solver.Add( + solver.Sum( + float(amount[index] * objective[index]) * variables[index] for index in range(n) + ) + >= float(threshold) + ) + solver.Minimize( + solver.Sum( + float(amount[index] * score_array[index]) * variables[index] for index in range(n) + ) + ) + else: + raise ValueError(f"Unknown frontier mode: {mode}.") + status = solver.Solve() + if status != pywraplp.Solver.OPTIMAL: + raise RuntimeError(f"GLOP frontier solve is not optimal: status {status}.") + fraction = np.asarray([variable.solution_value() for variable in variables], dtype=float) + fraction = np.clip(fraction, 0.0, 1.0) + exposure = amount * fraction + total = float(exposure.sum()) + return ScoreFrontierSolution( + allocation_fraction=fraction, + exposure=exposure, + objective_value=float(exposure @ objective), + weighted_score=float(exposure @ score_array / total), + total_allocated=total, + simplex_iterations=int(solver.iterations()), + ) + + +def _validated_vector( + values: Sequence[float] | np.ndarray, + *, + name: str, + n: int, +) -> np.ndarray: + array = np.asarray(values, dtype=float) + if array.shape != (n,) or not bool(np.isfinite(array).all()): + raise ValueError(f"{name} must be a finite vector with {n} rows.") + return array + + +def _objective_floor_lp( + frame: pd.DataFrame, + score: np.ndarray, + objective_rate: np.ndarray, + *, + budget: float, + purpose_cap: float, + sense: Literal["minimize", "maximize"], +) -> tuple[highspy.HighsLp, int, np.ndarray]: + n = len(frame) + amount = _validated_vector(frame["loan_amnt"], name="loan_amnt", n=n) + if bool(np.any(amount <= 0.0)): + raise ValueError("loan_amnt must be positive.") + if not 0.0 < float(purpose_cap) <= 1.0: + raise ValueError("purpose_cap must lie in (0, 1].") + rows = [amount] + lower = [float(budget)] + upper = [float(budget)] + objective_floor_row = len(rows) + rows.append(amount * objective_rate) + lower.append(-highspy.kHighsInf) + upper.append(highspy.kHighsInf) + purposes = frame["purpose"].astype("string").fillna("unknown") + for purpose in sorted(purposes.unique()): + rows.append(amount * purposes.eq(purpose).to_numpy(dtype=float)) + lower.append(-highspy.kHighsInf) + upper.append(float(purpose_cap) * float(budget)) + matrix = csc_matrix(np.vstack(rows)) + lp = highspy.HighsLp() + lp.num_col_ = n + lp.num_row_ = len(rows) + lp.col_cost_ = (amount * score).tolist() + lp.col_lower_ = np.zeros(n).tolist() + lp.col_upper_ = np.ones(n).tolist() + lp.row_lower_ = lower + lp.row_upper_ = upper + lp.sense_ = highspy.ObjSense.kMinimize if sense == "minimize" else highspy.ObjSense.kMaximize + lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise + lp.a_matrix_.num_col_ = n + lp.a_matrix_.num_row_ = len(rows) + lp.a_matrix_.start_ = matrix.indptr.astype(np.int32).tolist() + lp.a_matrix_.index_ = matrix.indices.astype(np.int32).tolist() + lp.a_matrix_.value_ = matrix.data.astype(float).tolist() + return lp, objective_floor_row, amount diff --git a/src/ijds_challengers/normalized_frontier.py b/src/ijds_challengers/normalized_frontier.py new file mode 100644 index 0000000..bfd6644 --- /dev/null +++ b/src/ijds_challengers/normalized_frontier.py @@ -0,0 +1,995 @@ +"""Build the complete outcome-free normalized/objective frontier census.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Protocol + +import numpy as np +import pandas as pd +from loguru import logger + +from src.evaluation.standardized_credit_payoff import expected_objective_coefficients +from src.ijds_audit.policy_support import point_basis_diagnostics +from src.ijds_audit.portfolio import PointPortfolioSession, PointPortfolioSolution +from src.ijds_challengers.archive import monthly_frames +from src.ijds_challengers.frontier import ( + ObjectiveFloorPortfolioSession, + ScoreFrontierSolution, + common_objective_target, + normalized_exposure_distance, + normalized_score_cap, + solve_glop_portfolio, +) +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + apply_binary_outcome_recipe, +) + + +@dataclass(frozen=True) +class FrontierBuild: + """All deterministic artifacts produced before the outcome join.""" + + solve_records: pd.DataFrame + allocations: pd.DataFrame + endpoint_diagnostics: pd.DataFrame + minimum_endpoint_diagnostics: pd.DataFrame + objective_optimum_diagnostics: pd.DataFrame + order_sensitivity: pd.DataFrame + independent_validation: pd.DataFrame + + +@dataclass(frozen=True) +class _GammaState: + score: np.ndarray + minimum_score: float + score_at_objective: float + score_range: float + minimum_objective: float + minimum_cap_residual: float + minimum_endpoint_retry_slack: float + normalized_session: PointPortfolioSession + objective_session: ObjectiveFloorPortfolioSession + + +@dataclass(frozen=True) +class _RulerSolution: + ruler: str + threshold: float + solution: ScoreFrontierSolution + cap: float | None + objective_target: float | None + + +@dataclass(frozen=True) +class _ObjectiveOptimum: + solution: ScoreFrontierSolution + diagnostics: dict[str, Any] + + +@dataclass(frozen=True) +class _FrontierCensus: + roles: tuple[str, ...] + role_months: dict[str, int] + windows: int + coordinates: int + all_gammas: tuple[float, ...] + reported_gammas: tuple[float, ...] + expected_records: int + expected_endpoints: int + expected_minimum_endpoints: int + expected_validation: int + + +class _PointEndpointSession(Protocol): + def solve(self, risk_cap: float) -> PointPortfolioSolution: + """Solve one point-score cap.""" + + +def build_outcome_free_frontiers( + base: pd.DataFrame, + recipes: Mapping[str, Mapping[str, Mapping[int, BinaryOutcomeConformalRecipe]]], + *, + config: Mapping[str, Any], + parent_config: Mapping[str, Any], +) -> FrontierBuild: + """Solve every locked V1b cell without accepting an outcome dataframe.""" + _assert_outcome_free(base, config=config) + frontier = config["frontier"] + solver_config = config["solver"] + gamma_grid = tuple(float(value) for value in frontier["gamma_grid"]) + reported_gamma_grid = tuple( + float(value) for value in frontier.get("reported_gamma_grid", gamma_grid) + ) + if not set(reported_gamma_grid).issubset(gamma_grid) or not {0.0, 1.0}.issubset( + reported_gamma_grid + ): + raise ValueError("Reported gammas must be a subset containing both endpoints.") + coordinates = tuple(float(value) for value in frontier["coordinate_grid"]) + budget = float(parent_config["policy"]["budget"]) + purpose_cap = float(parent_config["policy"]["max_concentration_by_purpose"]) + lgd = float(parent_config["payoff"]["lgd"]) + threads = int(solver_config["threads"]) + time_limit = int(solver_config["time_limit_seconds"]) + allocation_tolerance = float(solver_config["allocation_tolerance"]) + budget_tolerance = float(solver_config["budget_residual_tolerance_dollars"]) + normalized_config = frontier["normalized_score"] + objective_config = frontier["objective_matched"] + optimum_config = frontier["objective_optimum"] + validation_periods = { + str(value) for value in solver_config["independent_validation"]["periods"] + } + + records: list[dict[str, Any]] = [] + allocation_frames: list[pd.DataFrame] = [] + endpoint_rows: list[dict[str, Any]] = [] + minimum_endpoint_rows: list[dict[str, Any]] = [] + optimum_rows: list[dict[str, Any]] = [] + order_rows: list[dict[str, Any]] = [] + validation_rows: list[dict[str, Any]] = [] + objective_cache: dict[tuple[str, str], _ObjectiveOptimum] = {} + windows = recipes["catboost_platt"] + + for window_index, (window_id, group_recipes) in enumerate(sorted(windows.items()), start=1): + logger.info("Frontier window {}/8: {}", window_index, window_id) + point_all = base["pd_point"].to_numpy(dtype=float) + _, lower_all, upper_all = apply_binary_outcome_recipe(point_all, group_recipes[5]) + window_base = base.assign( + conformal_lower=lower_all, + conformal_upper=upper_all, + ) + for role in frontier["roles"]: + monthly = monthly_frames(window_base, str(role)) + expected_months = ( + int(frontier["expected_development_months"]) + if role == "policy_development" + else int(frontier["expected_primary_months"]) + ) + if len(monthly) != expected_months: + raise RuntimeError( + f"{window_id} {role} has {len(monthly)} months, not {expected_months}." + ) + for period, month in monthly: + point = month["pd_point"].to_numpy(dtype=float) + upper = month["conformal_upper"].to_numpy(dtype=float) + rates = month["contractual_rate"].to_numpy(dtype=float) + objective = expected_objective_coefficients(point, rates, lgd=lgd) + cache_key = (str(role), period) + optimum = objective_cache.get(cache_key) + if optimum is None: + optimum = _solve_objective_optimum( + month, + point_score=point, + objective_rate=objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + role=str(role), + period=period, + optimum_config=optimum_config, + solver_config=solver_config, + ) + objective_cache[cache_key] = optimum + optimum_rows.append(optimum.diagnostics) + unconstrained = optimum.solution + gamma_states = _build_gamma_states( + month, + point=point, + upper=upper, + objective=objective, + unconstrained=unconstrained, + gamma_grid=gamma_grid, + window_id=window_id, + role=str(role), + period=period, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + normalized_config=normalized_config, + ) + minimum_endpoint_rows.extend( + { + "window_id": window_id, + "role": str(role), + "period": period, + "gamma": gamma, + "minimum_score": state.minimum_score, + "minimum_cap_residual": state.minimum_cap_residual, + "minimum_endpoint_retry_slack": state.minimum_endpoint_retry_slack, + "minimum_endpoint_retried": bool(state.minimum_endpoint_retry_slack > 0.0), + } + for gamma, state in sorted(gamma_states.items()) + ) + common_lower, _ = common_objective_target( + minimum_objectives=[state.minimum_objective for state in gamma_states.values()], + objective_optimum=unconstrained.objective_value, + coordinate=0.0, + minimum_range=float(objective_config["minimum_objective_range_dollars"]), + ) + endpoint_solutions: dict[tuple[str, float, float], ScoreFrontierSolution] = {} + for gamma in reported_gamma_grid: + state = gamma_states[gamma] + for coordinate in coordinates: + ruler_solutions = _solve_rulers( + state, + gamma_states=gamma_states, + coordinate=coordinate, + unconstrained_objective=unconstrained.objective_value, + window_id=window_id, + role=str(role), + period=period, + gamma=gamma, + normalized_config=normalized_config, + objective_config=objective_config, + ) + for solved in ruler_solutions: + _append_solution( + records, + allocation_frames, + month=month, + score=state.score, + objective=objective, + solution=solved.solution, + window_id=window_id, + role=str(role), + period=period, + gamma=gamma, + ruler=solved.ruler, + coordinate=coordinate, + cap=solved.cap, + objective_target=solved.objective_target, + common_objective_lower=common_lower, + state=state, + unconstrained_objective=unconstrained.objective_value, + allocation_tolerance=allocation_tolerance, + budget=budget, + ) + if str(role) == "primary_oot" and gamma in {0.0, 1.0}: + endpoint_solutions[(solved.ruler, coordinate, gamma)] = ( + solved.solution + ) + order_rows.append( + _order_diagnostic( + month, + score=state.score, + objective=objective, + original=solved.solution, + window_id=window_id, + role=str(role), + period=period, + gamma=gamma, + ruler=solved.ruler, + coordinate=coordinate, + threshold=solved.threshold, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ) + ) + if period in validation_periods: + validation_rows.append( + _independent_diagnostic( + month, + score=state.score, + objective=objective, + original=solved.solution, + window_id=window_id, + role=str(role), + period=period, + gamma=gamma, + ruler=solved.ruler, + coordinate=coordinate, + threshold=solved.threshold, + budget=budget, + purpose_cap=purpose_cap, + ) + ) + if str(role) == "primary_oot": + for ruler in ("objective_matched", "normalized_score"): + for coordinate in coordinates: + full = endpoint_solutions[(ruler, coordinate, 1.0)] + point_solution = endpoint_solutions[(ruler, coordinate, 0.0)] + endpoint_rows.append( + { + "window_id": window_id, + "role": str(role), + "period": period, + "ruler": ruler, + "coordinate": coordinate, + "endpoint_contrast": "gamma_1_minus_gamma_0", + "normalized_exposure_distance": normalized_exposure_distance( + full.exposure, + point_solution.exposure, + budget=budget, + ), + "objective_difference": float( + full.objective_value - point_solution.objective_value + ), + "full_endpoint_weighted_score": float(full.weighted_score), + "point_endpoint_weighted_score": float( + point_solution.weighted_score + ), + "full_endpoint_point_moment": float( + full.exposure @ point / budget + ), + "point_endpoint_point_moment": float( + point_solution.exposure @ point / budget + ), + "unconstrained_objective": float(unconstrained.objective_value), + } + ) + + result = FrontierBuild( + solve_records=pd.DataFrame(records), + allocations=pd.concat(allocation_frames, ignore_index=True), + endpoint_diagnostics=pd.DataFrame(endpoint_rows), + minimum_endpoint_diagnostics=pd.DataFrame(minimum_endpoint_rows), + objective_optimum_diagnostics=pd.DataFrame(optimum_rows), + order_sensitivity=pd.DataFrame(order_rows), + independent_validation=pd.DataFrame(validation_rows), + ) + _validate_complete_build( + result, config=config, budget=budget, budget_tolerance=budget_tolerance + ) + return result + + +def _solve_objective_optimum( + month: pd.DataFrame, + *, + point_score: np.ndarray, + objective_rate: np.ndarray, + budget: float, + purpose_cap: float, + time_limit: int, + threads: int, + role: str, + period: str, + optimum_config: Mapping[str, Any], + solver_config: Mapping[str, Any], +) -> _ObjectiveOptimum: + session = PointPortfolioSession( + month, + point_score=point_score, + objective_rate=objective_rate, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ) + raw_solution = session.solve(1.0) + solution = _from_point_solution(raw_solution) + basis = point_basis_diagnostics( + session, + raw_solution, + dual_tolerance=float(optimum_config["dual_tolerance"]), + primal_tolerance=float(optimum_config["primal_tolerance"]), + ) + minimum_reduced_cost = float(basis["minimum_absolute_nonbasic_reduced_cost"]) + near_zero = int(basis["near_zero_nonbasic_reduced_costs"]) + if near_zero > 0 or minimum_reduced_cost <= float(optimum_config["dual_tolerance"]): + raise RuntimeError( + f"Objective optimum has a near-zero nonbasic reduced cost for " + f"{role} {period}: minimum={minimum_reduced_cost:.12g}, count={near_zero}." + ) + + reverse = month.iloc[::-1].reset_index(drop=True) + reversed_solution = PointPortfolioSession( + reverse, + point_score=point_score[::-1], + objective_rate=objective_rate[::-1], + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ).solve(1.0) + reverse_exposure = pd.Series( + reversed_solution.exposure, + index=reverse["id"].astype("string"), + ) + aligned = month["id"].astype("string").map(reverse_exposure).to_numpy(dtype=float) + exposure_distance = normalized_exposure_distance( + solution.exposure, + aligned, + budget=budget, + ) + objective_difference = float(reversed_solution.objective_value - solution.objective_value) + if exposure_distance > float(solver_config["order_exposure_distance_tolerance"]): + raise RuntimeError(f"ID reversal changed the objective optimum for {role} {period}.") + if abs(objective_difference) > float(solver_config["order_objective_tolerance_dollars"]): + raise RuntimeError(f"ID reversal changed the objective value for {role} {period}.") + + return _ObjectiveOptimum( + solution=solution, + diagnostics={ + "role": role, + "period": period, + "n_candidates": int(len(month)), + "objective_value": float(solution.objective_value), + "weighted_point_score": float(solution.weighted_score), + "minimum_absolute_nonbasic_reduced_cost": minimum_reduced_cost, + "minimum_scaled_nonbasic_reduced_cost": float( + basis["minimum_scaled_nonbasic_reduced_cost"] + ), + "near_zero_nonbasic_reduced_costs": near_zero, + "primal_degenerate_basic_columns": int(basis["primal_degenerate_basic_columns"]), + "primal_degenerate_basic_rows": int(basis["primal_degenerate_basic_rows"]), + "basis_primal_degenerate": bool(basis["basis_primal_degenerate"]), + "maximum_dual_sign_violation": float(basis["maximum_dual_sign_violation"]), + "objective_reconciliation_error": float(basis["objective_reconciliation_error"]), + "reversed_id_exposure_distance": exposure_distance, + "reversed_id_objective_difference": objective_difference, + "reversed_id_weighted_point_score_difference": float( + reversed_solution.weighted_point_score - solution.weighted_score + ), + }, + ) + + +def _build_gamma_states( + month: pd.DataFrame, + *, + point: np.ndarray, + upper: np.ndarray, + objective: np.ndarray, + unconstrained: ScoreFrontierSolution, + gamma_grid: tuple[float, ...], + window_id: str, + role: str, + period: str, + budget: float, + purpose_cap: float, + time_limit: int, + threads: int, + normalized_config: Mapping[str, Any], +) -> dict[float, _GammaState]: + states: dict[float, _GammaState] = {} + cap_tolerance = float(normalized_config["cap_residual_tolerance"]) + configured_slacks = normalized_config.get("minimum_endpoint_retry_slacks") + retry_slacks: tuple[float, ...] + if configured_slacks is None: + single_slack = float(normalized_config.get("minimum_endpoint_retry_slack", 0.0)) + retry_slacks = () if single_slack == 0.0 else (single_slack,) + else: + retry_slacks = tuple(float(value) for value in configured_slacks) + if ( + any(slack <= 0.0 or slack > cap_tolerance for slack in retry_slacks) + or tuple(sorted(set(retry_slacks))) != retry_slacks + ): + raise ValueError( + "Minimum-endpoint retry slacks must be unique, increasing, positive, and no " + "larger than the cap tolerance." + ) + for gamma in gamma_grid: + score = point + gamma * (upper - point) + normalized_session = PointPortfolioSession( + month, + point_score=score, + objective_rate=objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ) + raw_minimum = ObjectiveFloorPortfolioSession( + month, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ).solve() + minimum_score = float(raw_minimum.weighted_score) + point_minimum, applied_retry_slack = _solve_minimum_endpoint( + normalized_session, + minimum_score=minimum_score, + retry_slacks=retry_slacks, + ) + efficient_minimum = _from_point_solution(point_minimum) + minimum_cap_residual = float(efficient_minimum.weighted_score - minimum_score) + if abs(minimum_cap_residual) > cap_tolerance: + raise RuntimeError( + f"Minimum-score endpoint failed for {window_id} {role} " + f"{period} gamma={gamma}: {minimum_cap_residual:.3e}." + ) + score_at_objective = float(unconstrained.exposure @ score / budget) + score_range = score_at_objective - minimum_score + if score_range < float(normalized_config["minimum_score_range"]): + raise RuntimeError( + f"Normalized score range failed for {window_id} {role} " + f"{period} gamma={gamma}: {score_range:.12g}." + ) + objective_session = ObjectiveFloorPortfolioSession( + month, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ) + states[gamma] = _GammaState( + score=score, + minimum_score=minimum_score, + score_at_objective=score_at_objective, + score_range=score_range, + minimum_objective=float(efficient_minimum.objective_value), + minimum_cap_residual=minimum_cap_residual, + minimum_endpoint_retry_slack=applied_retry_slack, + normalized_session=normalized_session, + objective_session=objective_session, + ) + return states + + +def _solve_minimum_endpoint( + session: _PointEndpointSession, + *, + minimum_score: float, + retry_slacks: Sequence[float], +) -> tuple[PointPortfolioSolution, float]: + """Reconcile one cross-solver endpoint, retrying only known boundary failures.""" + try: + return session.solve(float(minimum_score)), 0.0 + except RuntimeError as error: + if not _is_minimum_endpoint_boundary_failure(error): + raise + if not retry_slacks: + raise + last_error = error + for retry_slack in retry_slacks: + try: + solution = session.solve(float(minimum_score) + float(retry_slack)) + return solution, float(retry_slack) + except RuntimeError as retry_error: + if not _is_minimum_endpoint_boundary_failure(retry_error): + raise + last_error = retry_error + raise last_error from error + + +def _is_minimum_endpoint_boundary_failure(error: RuntimeError) -> bool: + message = str(error) + return message in { + "Point LP is not optimal: Infeasible.", + "Point LP is not optimal: Unknown.", + } or message.startswith("Point LP did not fill its budget:") + + +def _solve_rulers( + state: _GammaState, + *, + gamma_states: Mapping[float, _GammaState], + coordinate: float, + unconstrained_objective: float, + window_id: str, + role: str, + period: str, + gamma: float, + normalized_config: Mapping[str, Any], + objective_config: Mapping[str, Any], +) -> tuple[_RulerSolution, _RulerSolution]: + cap = normalized_score_cap( + minimum_score=state.minimum_score, + score_at_objective=state.score_at_objective, + coordinate=coordinate, + minimum_range=float(normalized_config["minimum_score_range"]), + ) + normalized_solution = _from_point_solution(state.normalized_session.solve(cap)) + normalized_slack = float(cap - normalized_solution.weighted_score) + if abs(normalized_slack) > float(normalized_config["cap_residual_tolerance"]): + raise RuntimeError( + f"Normalized cap did not bind for {window_id} {role} {period} " + f"gamma={gamma} coordinate={coordinate}: {normalized_slack:.3e}." + ) + + _, objective_target = common_objective_target( + minimum_objectives=[item.minimum_objective for item in gamma_states.values()], + objective_optimum=unconstrained_objective, + coordinate=coordinate, + minimum_range=float(objective_config["minimum_objective_range_dollars"]), + ) + matched_solution = state.objective_session.solve(objective_target) + floor_slack = float(matched_solution.objective_value - objective_target) + if abs(floor_slack) > float(objective_config["floor_residual_tolerance_dollars"]): + raise RuntimeError( + f"Objective floor failed for {window_id} {role} {period} " + f"gamma={gamma} coordinate={coordinate}: {floor_slack:.3e}." + ) + return ( + _RulerSolution( + ruler="normalized_score", + threshold=cap, + solution=normalized_solution, + cap=cap, + objective_target=None, + ), + _RulerSolution( + ruler="objective_matched", + threshold=objective_target, + solution=matched_solution, + cap=None, + objective_target=objective_target, + ), + ) + + +def _from_point_solution(solution: PointPortfolioSolution) -> ScoreFrontierSolution: + return ScoreFrontierSolution( + allocation_fraction=solution.allocation_fraction, + exposure=solution.exposure, + objective_value=float(solution.objective_value), + weighted_score=float(solution.weighted_point_score), + total_allocated=float(solution.total_allocated), + simplex_iterations=int(solution.simplex_iterations), + ) + + +def _policy_label(ruler: str, gamma: float, coordinate: float) -> str: + return f"{ruler}_g{round(gamma * 100):03d}_c{round(coordinate * 100):03d}" + + +def _append_solution( + records: list[dict[str, Any]], + allocations: list[pd.DataFrame], + *, + month: pd.DataFrame, + score: np.ndarray, + objective: np.ndarray, + solution: ScoreFrontierSolution, + window_id: str, + role: str, + period: str, + gamma: float, + ruler: str, + coordinate: float, + cap: float | None, + objective_target: float | None, + common_objective_lower: float, + state: _GammaState, + unconstrained_objective: float, + allocation_tolerance: float, + budget: float, +) -> None: + label = _policy_label(ruler, gamma, coordinate) + if cap is not None: + constraint_slack = float(cap - solution.weighted_score) + elif objective_target is not None: + constraint_slack = float(solution.objective_value - objective_target) + else: + raise ValueError("A frontier solution requires a cap or objective target.") + active = solution.exposure > float(allocation_tolerance) + funded = month.loc[active].copy() + funded["allocation_fraction"] = solution.allocation_fraction[active] + funded["exposure"] = solution.exposure[active] + funded["weight"] = funded["exposure"] / solution.total_allocated + funded["pd_effective"] = score[active] + funded["expected_payoff_rate"] = objective[active] + funded["expected_payoff_contribution"] = funded["exposure"] * objective[active] + metadata = { + "window_id": window_id, + "role": role, + "period": period, + "policy_label": label, + "candidate_id": label, + "comparator_rule": ruler, + "paired_policy_id": label, + "frontier_ruler": ruler, + "frontier_coordinate": coordinate, + "frontier_cap": np.nan if cap is None else float(cap), + "objective_target": np.nan if objective_target is None else float(objective_target), + "gamma": gamma, + } + funded = funded.assign(**metadata) + allocations.append(funded) + records.append( + { + **metadata, + "risk_tolerance": np.nan if cap is None else float(cap), + "uncertainty_aversion": gamma, + "policy_mode": ruler, + "robust_guardrail": bool(gamma > 0.0), + "solver_status": "Optimal", + "solver_backend_actual": "highspy_exact_budget_simplex", + "expected_objective": float(solution.objective_value), + "n_candidates": int(len(month)), + "n_positive_exposure": int(active.sum()), + "total_allocated": float(solution.total_allocated), + "budget_residual": float(solution.total_allocated - budget), + "weighted_pd_point": float(solution.exposure @ month["pd_point"] / budget), + "weighted_pd_effective": float(solution.weighted_score), + "weighted_conformal_upper": float( + solution.exposure @ month["conformal_upper"] / budget + ), + "minimum_score": state.minimum_score, + "score_at_objective": state.score_at_objective, + "score_range": state.score_range, + "minimum_score_portfolio_objective": state.minimum_objective, + "common_objective_lower": common_objective_lower, + "unconstrained_objective": unconstrained_objective, + "objective_retention": float( + (solution.objective_value - common_objective_lower) + / (unconstrained_objective - common_objective_lower) + ), + "constraint_slack": constraint_slack, + "highs_simplex_iterations": int(solution.simplex_iterations), + } + ) + + +def _order_diagnostic( + month: pd.DataFrame, + *, + score: np.ndarray, + objective: np.ndarray, + original: ScoreFrontierSolution, + window_id: str, + role: str, + period: str, + gamma: float, + ruler: str, + coordinate: float, + threshold: float, + budget: float, + purpose_cap: float, + time_limit: int, + threads: int, +) -> dict[str, Any]: + reverse = month.iloc[::-1].reset_index(drop=True) + reverse_score = score[::-1] + reverse_objective = objective[::-1] + if ruler == "normalized_score": + reversed_solution = _from_point_solution( + PointPortfolioSession( + reverse, + point_score=reverse_score, + objective_rate=reverse_objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ).solve(float(threshold)) + ) + else: + reversed_solution = ObjectiveFloorPortfolioSession( + reverse, + score=reverse_score, + objective_rate=reverse_objective, + budget=budget, + purpose_cap=purpose_cap, + time_limit=time_limit, + threads=threads, + ).solve(float(threshold)) + reverse_exposure = pd.Series( + reversed_solution.exposure, + index=reverse["id"].astype("string"), + ) + aligned = month["id"].astype("string").map(reverse_exposure).to_numpy(dtype=float) + return { + "window_id": window_id, + "role": role, + "period": period, + "gamma": gamma, + "ruler": ruler, + "coordinate": coordinate, + "threshold": threshold, + "normalized_exposure_distance": normalized_exposure_distance( + original.exposure, + aligned, + budget=budget, + ), + "objective_difference": float(reversed_solution.objective_value - original.objective_value), + "weighted_score_difference": float( + reversed_solution.weighted_score - original.weighted_score + ), + } + + +def _independent_diagnostic( + month: pd.DataFrame, + *, + score: np.ndarray, + objective: np.ndarray, + original: ScoreFrontierSolution, + window_id: str, + role: str, + period: str, + gamma: float, + ruler: str, + coordinate: float, + threshold: float, + budget: float, + purpose_cap: float, +) -> dict[str, Any]: + glop = solve_glop_portfolio( + month, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=purpose_cap, + mode="normalized_score" if ruler == "normalized_score" else "objective_matched", + threshold=threshold, + ) + return { + "window_id": window_id, + "role": role, + "period": period, + "gamma": gamma, + "ruler": ruler, + "coordinate": coordinate, + "threshold": threshold, + "highs_objective": float(original.objective_value), + "glop_objective": float(glop.objective_value), + "objective_rate_difference": float( + (glop.objective_value - original.objective_value) / budget + ), + "highs_weighted_score": float(original.weighted_score), + "glop_weighted_score": float(glop.weighted_score), + "weighted_score_difference": float(glop.weighted_score - original.weighted_score), + "glop_iterations": int(glop.simplex_iterations), + } + + +def _frontier_census(result: FrontierBuild, config: Mapping[str, Any]) -> _FrontierCensus: + frontier = config["frontier"] + roles = tuple(str(value) for value in frontier["roles"]) + role_months = { + "policy_development": int(frontier["expected_development_months"]), + "primary_oot": int(frontier["expected_primary_months"]), + } + unknown_roles = sorted(set(roles).difference(role_months)) + if unknown_roles: + raise RuntimeError(f"Unsupported frontier roles: {unknown_roles}.") + windows = int(result.solve_records["window_id"].nunique()) + coordinates = len(frontier["coordinate_grid"]) + all_gammas = tuple(float(value) for value in frontier["gamma_grid"]) + reported_gammas = tuple( + float(value) for value in frontier.get("reported_gamma_grid", all_gammas) + ) + expected_records = ( + windows * sum(role_months[role] for role in roles) * len(reported_gammas) * coordinates * 2 + ) + expected_endpoints = ( + windows * role_months["primary_oot"] * coordinates * 2 + if "primary_oot" in roles and {0.0, 1.0}.issubset(reported_gammas) + else 0 + ) + expected_minimum_endpoints = ( + windows * sum(role_months[role] for role in roles) * len(all_gammas) + ) + validation_periods = len(config["solver"]["independent_validation"]["periods"]) + expected_validation = ( + windows * validation_periods * 2 * coordinates * 2 + if "primary_oot" in roles and {0.0, 1.0}.issubset(reported_gammas) + else 0 + ) + return _FrontierCensus( + roles=roles, + role_months=role_months, + windows=windows, + coordinates=coordinates, + all_gammas=all_gammas, + reported_gammas=reported_gammas, + expected_records=expected_records, + expected_endpoints=expected_endpoints, + expected_minimum_endpoints=expected_minimum_endpoints, + expected_validation=expected_validation, + ) + + +def _validate_frontier_censuses( + result: FrontierBuild, + *, + config: Mapping[str, Any], + census: _FrontierCensus, +) -> None: + frontier = config["frontier"] + if len(result.solve_records) != census.expected_records: + raise RuntimeError( + f"Frontier produced {len(result.solve_records)} records, not {census.expected_records}." + ) + if len(result.endpoint_diagnostics) != census.expected_endpoints: + raise RuntimeError("Primary endpoint diagnostic census is incomplete.") + minimum_endpoints = result.minimum_endpoint_diagnostics + if len(minimum_endpoints) != census.expected_minimum_endpoints: + raise RuntimeError("Minimum-endpoint diagnostic census is incomplete.") + if float(minimum_endpoints["minimum_cap_residual"].abs().max()) > float( + frontier["normalized_score"]["cap_residual_tolerance"] + ): + raise RuntimeError("A minimum-endpoint retry exceeded the declared cap tolerance.") + optimum = result.objective_optimum_diagnostics + expected_optima = sum(census.role_months[role] for role in census.roles) + if len(optimum) != expected_optima: + raise RuntimeError("Objective-optimum diagnostic census is incomplete.") + optimum_config = config["frontier"]["objective_optimum"] + if int(optimum["near_zero_nonbasic_reduced_costs"].sum()) != 0 or float( + optimum["minimum_absolute_nonbasic_reduced_cost"].min() + ) <= float(optimum_config["dual_tolerance"]): + raise RuntimeError("An objective optimum has an unresolved nonbasic reduced cost.") + expected_order = census.expected_endpoints * 2 + if len(result.order_sensitivity) != expected_order: + raise RuntimeError("Primary endpoint ID-order audit census is incomplete.") + if len(result.independent_validation) != census.expected_validation: + raise RuntimeError("Independent GLOP validation census is incomplete.") + + +def _validate_frontier_numerics( + result: FrontierBuild, + *, + config: Mapping[str, Any], + budget: float, + budget_tolerance: float, +) -> None: + maximum_budget = float(result.solve_records["budget_residual"].abs().max()) + if maximum_budget > budget_tolerance: + raise RuntimeError(f"Frontier budget residual reached {maximum_budget:.3e} dollars.") + order = result.order_sensitivity + optimum = result.objective_optimum_diagnostics + order_config = config["solver"] + if float(optimum["reversed_id_exposure_distance"].max()) > float( + order_config["order_exposure_distance_tolerance"] + ): + raise RuntimeError("ID reversal changed a score-independent objective optimum.") + if float(optimum["reversed_id_objective_difference"].abs().max()) > float( + order_config["order_objective_tolerance_dollars"] + ): + raise RuntimeError("ID reversal changed a score-independent optimum objective.") + if float(order["normalized_exposure_distance"].max()) > float( + order_config["order_exposure_distance_tolerance"] + ): + raise RuntimeError("ID reversal changed a primary endpoint allocation.") + if float(order["objective_difference"].abs().max()) > float( + order_config["order_objective_tolerance_dollars"] + ): + raise RuntimeError("ID reversal changed a primary endpoint objective.") + validation = result.independent_validation + independent = order_config["independent_validation"] + if float(validation["objective_rate_difference"].abs().max()) > float( + independent["objective_rate_tolerance"] + ): + raise RuntimeError("GLOP disagrees with HiGHS on a frontier objective rate.") + if float(validation["weighted_score_difference"].abs().max()) > float( + independent["weighted_score_tolerance"] + ): + raise RuntimeError("GLOP disagrees with HiGHS on a funded score.") + if not np.isclose( + result.solve_records["total_allocated"].to_numpy(dtype=float), + float(budget), + atol=budget_tolerance, + rtol=0.0, + ).all(): + raise RuntimeError("At least one frontier solve failed the exact-budget contract.") + + +def _validate_complete_build( + result: FrontierBuild, + *, + config: Mapping[str, Any], + budget: float, + budget_tolerance: float, +) -> None: + census = _frontier_census(result, config) + _validate_frontier_censuses(result, config=config, census=census) + _validate_frontier_numerics( + result, + config=config, + budget=budget, + budget_tolerance=budget_tolerance, + ) + + +def _assert_outcome_free(frame: pd.DataFrame, *, config: Mapping[str, Any]) -> None: + tokens = tuple(str(value).casefold() for value in config["source_ingest"]["forbidden_tokens"]) + forbidden = [ + str(column) + for column in frame.columns + if any(token in str(column).casefold() for token in tokens) + ] + if forbidden: + raise ValueError(f"Outcome-like columns reached frontier construction: {forbidden}.") diff --git a/src/models/__init__.py b/src/models/__init__.py index 90b6c8a..2ddb206 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -1 +1 @@ -"""ML models for CRPTO: PD, calibration and conformal prediction.""" +"""Maturity-safe PD and binary conformal score primitives.""" diff --git a/src/models/binary_conformal_guardrail.py b/src/models/binary_conformal_guardrail.py new file mode 100644 index 0000000..1b37591 --- /dev/null +++ b/src/models/binary_conformal_guardrail.py @@ -0,0 +1,154 @@ +"""Mondrian split-conformal intervals for the observed binary outcome. + +These intervals predict the binary repayment outcome. They are not confidence +intervals for a latent probability of default. Their upper endpoint may be +used as a decision score, but that use does not transport marginal coverage to +an optimizer-selected portfolio. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + + +@dataclass(frozen=True) +class BinaryOutcomeConformalRecipe: + """Exact absolute-residual quantiles fitted within score strata.""" + + alpha: float + requested_groups: int + bin_edges: tuple[float, ...] + residual_quantiles: tuple[float, ...] + group_counts: tuple[int, ...] + finite_sample_ranks: tuple[int, ...] + raw_finite_sample_ranks: tuple[int, ...] + estimand: str = "binary_outcome_prediction_interval" + method: str = "exact_split_mondrian_absolute_residual" + learned_widening: bool = False + learned_floor: bool = False + taxonomy_provenance: str = "unspecified_legacy_recipe" + taxonomy_method: str = "unspecified_legacy_taxonomy" + + +def _validated_bin_edges(bin_edges: Sequence[float]) -> np.ndarray: + edges = np.asarray(tuple(bin_edges), dtype=float) + if ( + edges.ndim != 1 + or len(edges) < 2 + or not bool(np.isfinite(edges).all()) + or bool(np.any(np.diff(edges) <= 0.0)) + ): + raise ValueError( + "Conformal bin edges must be a finite, strictly increasing one-dimensional vector." + ) + return edges + + +def assign_conformal_groups( + probabilities: np.ndarray, + bin_edges: Sequence[float], +) -> np.ndarray: + """Assign calibrated scores to the frozen Mondrian strata.""" + values = np.asarray(probabilities, dtype=float) + edges = _validated_bin_edges(bin_edges) + return np.searchsorted(edges[1:-1], values, side="right").astype(int) + + +def fit_binary_outcome_recipe( + probabilities: np.ndarray, + outcomes: np.ndarray, + *, + alpha: float, + n_groups: int | None = None, + bin_edges: Sequence[float] | None = None, + taxonomy_provenance: str | None = None, + taxonomy_method: str | None = None, + method: str | None = None, +) -> BinaryOutcomeConformalRecipe: + """Fit exact residual quantiles within a fixed or legacy score taxonomy. + + When ``bin_edges`` is supplied, the taxonomy is treated as an upstream, + outcome-free input. Only residuals from ``probabilities`` and ``outcomes`` + determine the group counts, ranks, and quantiles. Omitting ``bin_edges`` + preserves the historical behavior that learns score quantiles from this + same calibration sample. + """ + point = np.asarray(probabilities, dtype=float) + raw_outcomes = np.asarray(outcomes) + if len(point) != len(raw_outcomes) or len(point) == 0: + raise ValueError("Conformal fit arrays must be nonempty and aligned.") + if not bool(np.isfinite(point).all()): + raise ValueError("Conformal probabilities must be finite.") + if not bool(np.isin(raw_outcomes, (0, 1)).all()): + raise ValueError("Conformal outcomes must contain only binary labels 0 and 1.") + y_true = raw_outcomes.astype(int) + if not 0.0 < float(alpha) < 1.0: + raise ValueError("alpha must lie in (0, 1).") + if bin_edges is None: + if n_groups is None or int(n_groups) < 1: + raise ValueError("n_groups must be positive when bin_edges are not supplied.") + group_count = int(n_groups) + edges = np.quantile( + point, + np.linspace(0.0, 1.0, group_count + 1), + method="linear", + ) + if bool(np.any(np.diff(edges) <= 0.0)): + raise RuntimeError("Calibrated-score quantiles do not define distinct groups.") + provenance = taxonomy_provenance or "conformal_calibration_sample" + taxonomy = taxonomy_method or "empirical_linear_score_quantiles" + recipe_method = method or "exact_split_mondrian_absolute_residual" + else: + edges = _validated_bin_edges(bin_edges) + group_count = len(edges) - 1 + if n_groups is not None and int(n_groups) != group_count: + raise ValueError("n_groups must match the number of supplied score bins.") + provenance = taxonomy_provenance or "externally_supplied" + taxonomy = taxonomy_method or "fixed_strictly_increasing_score_bin_edges" + recipe_method = method or "fixed_taxonomy_split_mondrian_absolute_residual" + groups = assign_conformal_groups(point, tuple(float(value) for value in edges)) + residual = np.abs(y_true.astype(float) - point) + quantiles: list[float] = [] + counts: list[int] = [] + ranks: list[int] = [] + raw_ranks: list[int] = [] + for group in range(group_count): + scores = np.sort(residual[groups == group]) + count = int(len(scores)) + if count == 0: + raise RuntimeError(f"Conformal group {group} is empty.") + raw_rank = int(np.ceil((count + 1) * (1.0 - float(alpha)))) + rank = min(max(raw_rank, 1), count) + quantile = 1.0 if raw_rank > count else float(scores[rank - 1]) + counts.append(count) + raw_ranks.append(raw_rank) + ranks.append(rank) + quantiles.append(float(np.clip(quantile, 0.0, 1.0))) + return BinaryOutcomeConformalRecipe( + alpha=float(alpha), + requested_groups=group_count, + bin_edges=tuple(float(value) for value in edges), + residual_quantiles=tuple(quantiles), + group_counts=tuple(counts), + finite_sample_ranks=tuple(ranks), + raw_finite_sample_ranks=tuple(raw_ranks), + method=recipe_method, + taxonomy_provenance=provenance, + taxonomy_method=taxonomy, + ) + + +def apply_binary_outcome_recipe( + probabilities: np.ndarray, + recipe: BinaryOutcomeConformalRecipe, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Apply the frozen recipe without holdout-learned widening or floors.""" + point = np.asarray(probabilities, dtype=float) + groups = assign_conformal_groups(point, recipe.bin_edges) + quantiles = np.asarray(recipe.residual_quantiles, dtype=float)[groups] + lower = np.clip(point - quantiles, 0.0, 1.0) + upper = np.clip(point + quantiles, 0.0, 1.0) + return groups, lower, upper diff --git a/src/models/calibration.py b/src/models/calibration.py deleted file mode 100644 index 06266cc..0000000 --- a/src/models/calibration.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Probability calibration methods. - -Available methods: Isotonic, Platt (Sigmoid), Beta, and Venn-Abers. -Canonical calibrator is selected at training time via temporal multi-metric validation. -""" - -from __future__ import annotations - -from typing import Any, cast - -import numpy as np -import pandas as pd -from loguru import logger -from scipy.optimize import minimize_scalar -from sklearn.isotonic import IsotonicRegression -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import log_loss - -from src.utils.io_utils import load_pickle_compat - - -class LogitShiftCalibrator: - """Lightweight serializable calibrator that shifts log-odds by a fixed delta.""" - - def __init__(self, delta: float): - self.delta = float(delta) - - def transform(self, scores: np.ndarray) -> np.ndarray: - scores_arr = np.clip(np.asarray(scores, dtype=float), 1e-6, 1.0 - 1e-6) - logits = np.log(scores_arr / (1.0 - scores_arr)) - shifted = 1.0 / (1.0 + np.exp(-(logits + self.delta))) - clipped: np.ndarray = np.clip(np.asarray(shifted, dtype=float), 0.0, 1.0) - return clipped - - def predict(self, scores: np.ndarray) -> np.ndarray: - return self.transform(scores) - - def get_params(self, deep: bool = True) -> dict[str, float]: - return {"delta": float(self.delta)} - - def __repr__(self) -> str: - return f"LogitShiftCalibrator(delta={self.delta:.6f})" - - -class TemperatureScalingCalibrator: - """Serializable binary temperature scaler over probability logits.""" - - def __init__(self, temperature: float = 1.0) -> None: - self.temperature = float(temperature) - self._is_fitted = False - - @staticmethod - def _logit(scores: np.ndarray) -> np.ndarray: - clipped = np.clip(np.asarray(scores, dtype=float).reshape(-1), 1e-6, 1.0 - 1e-6) - logits: np.ndarray = np.log(clipped / (1.0 - clipped)) - return logits - - @staticmethod - def _sigmoid(logits: np.ndarray) -> np.ndarray: - return cast(np.ndarray, 1.0 / (1.0 + np.exp(-logits))) - - def fit(self, y_prob_raw: np.ndarray, y_true: np.ndarray) -> TemperatureScalingCalibrator: - logits = self._logit(y_prob_raw) - y = np.asarray(y_true, dtype=int) - - def objective(raw_temperature: float) -> float: - temperature = max(float(raw_temperature), 1e-3) - pred = np.clip(self._sigmoid(logits / temperature), 1e-6, 1.0 - 1e-6) - return float(log_loss(y, pred)) - - result = minimize_scalar(objective, bounds=(0.05, 10.0), method="bounded") - self.temperature = float(result.x if result.success else 1.0) - self._is_fitted = True - logger.info("Fitted temperature scaling calibrator (T={:.6f})", self.temperature) - return self - - def predict(self, y_prob_raw: np.ndarray) -> np.ndarray: - if not self._is_fitted: - raise RuntimeError("TemperatureScalingCalibrator is not fitted.") - logits = self._logit(y_prob_raw) - clipped: np.ndarray = np.clip(self._sigmoid(logits / max(self.temperature, 1e-3)), 0.0, 1.0) - return clipped - - -class QuadraticLogitCalibrator: - """Logistic calibrator over logit(p) and logit(p)^2.""" - - def __init__(self) -> None: - self.model = LogisticRegression(max_iter=1000) - self._is_fitted = False - - @staticmethod - def _design_matrix(y_prob_raw: np.ndarray) -> np.ndarray: - clipped = np.clip(np.asarray(y_prob_raw, dtype=float).reshape(-1), 1e-6, 1.0 - 1e-6) - logits = np.log(clipped / (1.0 - clipped)) - return np.column_stack([logits, logits**2]) - - def fit(self, y_prob_raw: np.ndarray, y_true: np.ndarray) -> QuadraticLogitCalibrator: - self.model.fit(self._design_matrix(y_prob_raw), np.asarray(y_true, dtype=int)) - self._is_fitted = True - logger.info("Fitted quadratic-logit calibrator") - return self - - def predict(self, y_prob_raw: np.ndarray) -> np.ndarray: - if not self._is_fitted: - raise RuntimeError("QuadraticLogitCalibrator is not fitted.") - return cast(np.ndarray, self.model.predict_proba(self._design_matrix(y_prob_raw))[:, 1]) - - -def expected_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: int = 10) -> float: - """Compute Expected Calibration Error (ECE).""" - bin_edges = np.linspace(0, 1, n_bins + 1) - ece = 0.0 - for i in range(n_bins): - mask = (y_prob >= bin_edges[i]) & (y_prob < bin_edges[i + 1]) - if mask.sum() == 0: - continue - bin_acc = y_true[mask].mean() - bin_conf = y_prob[mask].mean() - ece += mask.sum() / len(y_true) * abs(bin_acc - bin_conf) - return ece - - -def adaptive_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: int = 10) -> float: - """Compute quantile-binned ECE to reduce sensitivity to sparse fixed bins.""" - y_arr = np.asarray(y_true, dtype=float).reshape(-1) - p_arr = np.asarray(y_prob, dtype=float).reshape(-1) - if len(y_arr) == 0: - return 0.0 - order = np.argsort(p_arr, kind="mergesort") - y_sorted = y_arr[order] - p_sorted = p_arr[order] - bins = [ - chunk - for chunk in np.array_split(np.arange(len(y_sorted)), max(1, int(n_bins))) - if len(chunk) - ] - ace = 0.0 - for idx in bins: - bin_acc = float(y_sorted[idx].mean()) - bin_conf = float(p_sorted[idx].mean()) - ace += len(idx) / len(y_sorted) * abs(bin_acc - bin_conf) - return float(ace) - - -def calibrate_isotonic( - y_cal: np.ndarray, - proba_cal: np.ndarray, -) -> IsotonicRegression: - """Fit isotonic regression calibrator.""" - iso = IsotonicRegression(y_min=0, y_max=1, out_of_bounds="clip") - iso.fit(proba_cal, y_cal) - logger.info("Fitted isotonic calibrator") - return iso - - -def calibrate_platt( - model: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, -) -> LogisticRegression: - """Fit Platt scaling as logistic regression over raw model scores. - - Returning a score-based calibrator keeps downstream conformal code agnostic - to feature-space requirements of the base classifier. - """ - proba_cal = model.predict_proba(X_cal)[:, 1] - cal_model = LogisticRegression(max_iter=1000) - cal_model.fit(proba_cal.reshape(-1, 1), y_cal) - logger.info("Fitted Platt scaling calibrator") - return cal_model - - -def calibrate_beta( - y_cal: np.ndarray, - proba_cal: np.ndarray, - parameters: str = "abm", -) -> Any: - """Fit beta calibration (Kull et al. 2017). - - Args: - y_cal: Binary labels. - proba_cal: Raw probabilities from base model. - parameters: Beta calibration parameterisation. - "abm" = 3 parameters (handles asymmetric distortions). - "am" = 2 parameters (equivalent to Platt when a=b). - - Returns: - Fitted BetaCalibration object with .predict() method. - """ - from betacal import BetaCalibration - - bc = BetaCalibration(parameters=parameters) - bc.fit(proba_cal, y_cal) - logger.info(f"Fitted beta calibrator (parameters={parameters})") - return bc - - -def evaluate_calibration( - y_true: np.ndarray, - y_prob: np.ndarray, - name: str = "model", - n_bins: int = 10, -) -> dict[str, float]: - """Evaluate calibration quality.""" - from sklearn.metrics import brier_score_loss, log_loss - - ece = expected_calibration_error(y_true, y_prob, n_bins) - adaptive_ece = adaptive_calibration_error(y_true, y_prob, n_bins) - brier = brier_score_loss(y_true, y_prob) - logloss = log_loss(y_true, y_prob) - metrics = { - "ece": ece, - "adaptive_ece": adaptive_ece, - "brier_score": brier, - "log_loss": logloss, - } - logger.info( - f"Calibration [{name}] — ECE: {ece:.4f}, Adaptive ECE: {adaptive_ece:.4f}, " - f"Brier: {brier:.4f}, Log-loss: {logloss:.4f}" - ) - return metrics - - -def load_probability_calibrator(path: str | None) -> Any | None: - """Load a canonical or shadow calibrator from disk.""" - if not path: - return None - return load_pickle_compat(path) diff --git a/src/models/conformal/__init__.py b/src/models/conformal/__init__.py deleted file mode 100644 index 97967b0..0000000 --- a/src/models/conformal/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Public facade for conformal prediction utilities. - -The implementation lives in focused submodules, but the stable public import -path remains ``src.models.conformal`` for scripts, tests, and old pickles. -""" - -from __future__ import annotations - -from src.models.conformal._scores import ( - _compute_score_scale, - _conformal_quantile, - _resolve_score_scale_family, -) -from src.models.conformal.classification import ( - build_mondrian_partition_labels, - create_classification_sets, - create_classification_sets_mondrian, - create_cross_conformal_score_intervals, -) -from src.models.conformal.pd_intervals import ( - apply_probability_calibrator, - conditional_coverage_by_group, - create_pd_intervals, - create_pd_intervals_mondrian, - create_pd_intervals_mondrian_from_predictions, - create_pd_intervals_venn_abers, -) -from src.models.conformal.regression import ( - create_regression_intervals, - create_residual_intervals, -) -from src.models.conformal_adapters import ( - PrefitCalibratedClassifierAdapter, - PrefitClassifierAdapter, - ProbabilityRegressor, -) -from src.models.conformal_diagnostics import ( - summarize_prediction_sets, - validate_coverage, -) - -__all__ = [ - "PrefitCalibratedClassifierAdapter", - "PrefitClassifierAdapter", - "ProbabilityRegressor", - "_compute_score_scale", - "_conformal_quantile", - "_resolve_score_scale_family", - "apply_probability_calibrator", - "build_mondrian_partition_labels", - "conditional_coverage_by_group", - "create_classification_sets", - "create_classification_sets_mondrian", - "create_cross_conformal_score_intervals", - "create_pd_intervals", - "create_pd_intervals_mondrian", - "create_pd_intervals_mondrian_from_predictions", - "create_pd_intervals_venn_abers", - "create_regression_intervals", - "create_residual_intervals", - "summarize_prediction_sets", - "validate_coverage", -] diff --git a/src/models/conformal/_scores.py b/src/models/conformal/_scores.py deleted file mode 100644 index 6378881..0000000 --- a/src/models/conformal/_scores.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Score-space helpers shared by conformal interval builders.""" - -from __future__ import annotations - -import numpy as np - - -def _conformal_quantile(scores: np.ndarray, alpha: float) -> float: - """Finite-sample conformal quantile with ``higher`` interpolation.""" - scores = np.asarray(scores, dtype=float) - if scores.size == 0: - return 0.0 - n = scores.size - q_level = min(1.0, np.ceil((n + 1) * (1 - alpha)) / n) - return float(np.quantile(scores, q_level, method="higher")) - - -def _resolve_score_scale_family(*, scaled_scores: bool, score_scale_family: str | None) -> str: - family = str(score_scale_family or "").strip().lower() - if family in {"", "auto"}: - family = "bernoulli_sqrt" if scaled_scores else "none" - valid = { - "none", - "bernoulli_sqrt", - "bernoulli_sqrt_clipped_0.02", - "bernoulli_sqrt_clipped_0.05", - } - if family not in valid: - raise ValueError(f"Unsupported score_scale_family: {score_scale_family}") - return family - - -def _compute_score_scale(y_prob: np.ndarray, score_scale_family: str) -> np.ndarray: - y_prob_arr = np.clip(np.asarray(y_prob, dtype=float), 1e-6, 1.0 - 1e-6) - if score_scale_family == "none": - return np.ones_like(y_prob_arr) - if score_scale_family == "bernoulli_sqrt": - return np.asarray(np.sqrt(np.clip(y_prob_arr * (1.0 - y_prob_arr), 1e-6, None))) - if score_scale_family == "bernoulli_sqrt_clipped_0.02": - clipped = np.clip(y_prob_arr, 0.02, 0.98) - return np.asarray(np.sqrt(np.clip(clipped * (1.0 - clipped), 1e-6, None))) - if score_scale_family == "bernoulli_sqrt_clipped_0.05": - clipped = np.clip(y_prob_arr, 0.05, 0.95) - return np.asarray(np.sqrt(np.clip(clipped * (1.0 - clipped), 1e-6, None))) - raise ValueError(f"Unsupported score_scale_family: {score_scale_family}") diff --git a/src/models/conformal/classification.py b/src/models/conformal/classification.py deleted file mode 100644 index a7ec1f9..0000000 --- a/src/models/conformal/classification.py +++ /dev/null @@ -1,309 +0,0 @@ -"""Classification conformal sets and Mondrian partition helpers.""" - -from __future__ import annotations - -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -from src.models.conformal._scores import _conformal_quantile -from src.models.conformal.pd_intervals import apply_probability_calibrator -from src.models.conformal_adapters import PrefitCalibratedClassifierAdapter - - -def create_classification_sets( - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - alpha: float = 0.1, - method: str = "lac", - calibrator: Any | None = None, -) -> tuple[np.ndarray, np.ndarray]: - """Generate conformal prediction sets for classification.""" - method_key = str(method or "lac").strip().lower() - if method_key == "margin": - y_pred, y_sets = _create_margin_classification_sets( - classifier=classifier, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - calibrator=calibrator, - ) - else: - from mapie.classification import SplitConformalClassifier - - adapted = PrefitCalibratedClassifierAdapter( - classifier, - calibrator=calibrator, - n_features_in=X_cal.shape[1], - ) - mapie = SplitConformalClassifier( - estimator=adapted, - confidence_level=1 - alpha, - conformity_score=method_key, - prefit=True, - ) - mapie.conformalize(X_cal, y_cal) - - y_pred = mapie.predict(X_test) - _, y_sets_raw = mapie.predict_set(X_test) - y_sets = np.asarray(y_sets_raw[:, :, 0], dtype=int) - - singleton_rate = (y_sets.sum(axis=1) == 1).mean() - logger.info( - f"Conformal sets (alpha={alpha}, method={method_key}): singleton_rate={singleton_rate:.2%}" - ) - return y_pred, y_sets - - -def _create_margin_classification_sets( - *, - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - alpha: float, - calibrator: Any | None, -) -> tuple[np.ndarray, np.ndarray]: - """Binary margin-style conformal sets over calibrated probabilities.""" - p_cal = apply_probability_calibrator(calibrator, classifier.predict_proba(X_cal)[:, 1]) - p_test = apply_probability_calibrator(calibrator, classifier.predict_proba(X_test)[:, 1]) - y_cal_arr = np.asarray(y_cal, dtype=int).reshape(-1) - - p_true = np.where(y_cal_arr == 1, p_cal, 1.0 - p_cal) - nonconformity = 2.0 * (1.0 - p_true) - q_alpha = _conformal_quantile(nonconformity, alpha) - - include_pos = (2.0 * (1.0 - p_test)) <= q_alpha - include_neg = (2.0 * p_test) <= q_alpha - y_sets = np.column_stack([include_neg.astype(int), include_pos.astype(int)]) - y_pred = (p_test >= 0.5).astype(int) - return y_pred, y_sets - - -def create_classification_sets_mondrian( - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - group_cal: pd.Series, - group_test: pd.Series, - alpha: float = 0.1, - method: str = "lac", - min_group_size: int = 500, - calibrator: Any | None = None, -) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: - """Generate group-conditional conformal prediction sets for binary classification.""" - g_cal = pd.Series(group_cal).fillna("UNKNOWN").astype(str).reset_index(drop=True) - g_test = pd.Series(group_test).fillna("UNKNOWN").astype(str).reset_index(drop=True) - all_groups = sorted(set(g_cal).union(set(g_test))) - group_counts = {group: int((g_cal == group).sum()) for group in all_groups} - fallback_groups: list[str] = [] - - global_pred, global_sets = create_classification_sets( - classifier=classifier, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - method=method, - calibrator=calibrator, - ) - y_pred = np.asarray(global_pred, dtype=int).copy() - y_sets = np.asarray(global_sets, dtype=int).copy() - - for group in all_groups: - cal_mask = g_cal == group - test_mask = g_test == group - if not test_mask.any(): - continue - if int(cal_mask.sum()) < int(min_group_size): - fallback_groups.append(group) - continue - group_pred, group_sets = create_classification_sets( - classifier=classifier, - X_cal=X_cal.loc[cal_mask].reset_index(drop=True), - y_cal=y_cal.loc[cal_mask].reset_index(drop=True), - X_test=X_test.loc[test_mask].reset_index(drop=True), - alpha=alpha, - method=method, - calibrator=calibrator, - ) - y_pred[np.asarray(test_mask)] = np.asarray(group_pred, dtype=int) - y_sets[np.asarray(test_mask)] = np.asarray(group_sets, dtype=int) - - diagnostics = { - "alpha": float(alpha), - "method": str(method), - "min_group_size": int(min_group_size), - "group_cal_counts": group_counts, - "fallback_groups": sorted(set(fallback_groups)), - } - return y_pred, y_sets, diagnostics - - -def build_mondrian_partition_labels( - *, - y_prob_cal: np.ndarray, - y_prob_eval: np.ndarray, - partition: str, - base_groups_cal: pd.Series | np.ndarray | None = None, - base_groups_eval: pd.Series | np.ndarray | None = None, - n_score_bins: int = 10, - min_group_size: int = 500, - fallback_mode: str = "grade_then_global", -) -> tuple[pd.Series, pd.Series, dict[str, Any]]: - """Build partition labels for Mondrian-style calibration.""" - partition_key = str(partition).strip().lower() - if partition_key in {"grade", "group", "default"}: - if base_groups_cal is None or base_groups_eval is None: - raise ValueError("grade partition requires base group labels for calibration/eval.") - g_cal = pd.Series(base_groups_cal).fillna("UNKNOWN").astype(str).reset_index(drop=True) - g_eval = pd.Series(base_groups_eval).fillna("UNKNOWN").astype(str).reset_index(drop=True) - return ( - g_cal, - g_eval, - { - "partition": "grade", - "score_band_count": 0, - "fallback_groups": [], - }, - ) - - cal_scores = pd.Series(np.asarray(y_prob_cal, dtype=float).reshape(-1)).clip(0.0, 1.0) - eval_scores = pd.Series(np.asarray(y_prob_eval, dtype=float).reshape(-1)).clip(0.0, 1.0) - rank_source = cal_scores.rank(method="first") - n_bins_effective = int(max(1, min(int(n_score_bins), int(rank_source.nunique())))) - if n_bins_effective <= 1: - cal_band = pd.Series(["score_q0"] * len(cal_scores), dtype="string") - eval_band = pd.Series(["score_q0"] * len(eval_scores), dtype="string") - edges = np.array([0.0, 1.0], dtype=float) - else: - quantiles = np.linspace(0.0, 1.0, n_bins_effective + 1) - edges = np.unique(np.quantile(cal_scores.to_numpy(dtype=float), quantiles)) - if len(edges) <= 2: - cal_band = pd.Series(["score_q0"] * len(cal_scores), dtype="string") - eval_band = pd.Series(["score_q0"] * len(eval_scores), dtype="string") - else: - labels = [f"score_q{i:02d}" for i in range(len(edges) - 1)] - cal_band = pd.cut( - cal_scores, - bins=edges, - labels=labels, - include_lowest=True, - duplicates="drop", - ).astype("string") - eval_band = pd.cut( - eval_scores, - bins=edges, - labels=labels, - include_lowest=True, - duplicates="drop", - ).astype("string") - cal_band = cal_band.fillna(labels[0]) - eval_band = eval_band.fillna(labels[0]) - - if partition_key in {"score_decile_mondrian", "score_decile", "scoreband"}: - return ( - cal_band.astype(str).reset_index(drop=True), - eval_band.astype(str).reset_index(drop=True), - { - "partition": "score_decile_mondrian", - "score_band_count": int(cal_band.nunique()), - "score_band_edges": [float(x) for x in np.asarray(edges, dtype=float)], - "fallback_groups": [], - "fallback_mode": "score_only", - }, - ) - - if partition_key not in {"grade_x_scoreband_mondrian", "grade_scoreband", "hybrid"}: - raise ValueError(f"Unsupported partition mode: {partition}") - - if base_groups_cal is None or base_groups_eval is None: - raise ValueError("grade_x_scoreband_mondrian requires base group labels.") - - grade_cal = pd.Series(base_groups_cal).fillna("UNKNOWN").astype(str).reset_index(drop=True) - grade_eval = pd.Series(base_groups_eval).fillna("UNKNOWN").astype(str).reset_index(drop=True) - hybrid_cal = (grade_cal + "|" + cal_band.astype(str)).reset_index(drop=True) - hybrid_eval = (grade_eval + "|" + eval_band.astype(str)).reset_index(drop=True) - - counts = hybrid_cal.value_counts(dropna=False).to_dict() - grade_counts = grade_cal.value_counts(dropna=False).to_dict() - fallback_groups: list[str] = [] - fallback_mode_key = str(fallback_mode or "grade_then_global").strip().lower() - if fallback_mode_key not in {"grade_then_global", "global_only"}: - raise ValueError(f"Unsupported fallback_mode: {fallback_mode}") - - def _resolve_label(label: str, base_grade: str) -> str: - n_label = int(counts.get(label, 0)) - if n_label >= int(min_group_size): - return label - fallback_groups.append(label) - if fallback_mode_key == "grade_then_global" and int(grade_counts.get(base_grade, 0)) >= int( - min_group_size - ): - return base_grade - return "GLOBAL" - - resolved_cal = pd.Series( - [ - _resolve_label(str(label), str(grade)) - for label, grade in zip(hybrid_cal, grade_cal, strict=False) - ], - dtype="string", - ) - resolved_eval = pd.Series( - [ - _resolve_label(str(label), str(grade)) - for label, grade in zip(hybrid_eval, grade_eval, strict=False) - ], - dtype="string", - ) - return ( - resolved_cal.astype(str).reset_index(drop=True), - resolved_eval.astype(str).reset_index(drop=True), - { - "partition": "grade_x_scoreband_mondrian", - "score_band_count": int(cal_band.nunique()), - "score_band_edges": [float(x) for x in np.asarray(edges, dtype=float)], - "fallback_groups": sorted(set(fallback_groups)), - "hybrid_group_count_cal": int(hybrid_cal.nunique()), - "resolved_group_count_cal": int(resolved_cal.nunique()), - "fallback_mode": fallback_mode_key, - }, - ) - - -def create_cross_conformal_score_intervals( - y_cal: pd.Series | np.ndarray, - y_prob_cal: np.ndarray, - y_prob_test: np.ndarray, - *, - alpha: float = 0.1, - method: str = "plus", - cv: int = 5, -) -> tuple[np.ndarray, np.ndarray]: - """Run a lightweight cross conformal benchmark on raw score space.""" - from mapie.regression import CrossConformalRegressor - from sklearn.linear_model import LinearRegression - - y_cal_arr = np.asarray(y_cal, dtype=float).reshape(-1) - X_cal = np.asarray(y_prob_cal, dtype=float).reshape(-1, 1) - X_test = np.asarray(y_prob_test, dtype=float).reshape(-1, 1) - - regressor = CrossConformalRegressor( - estimator=LinearRegression(), - confidence_level=1 - alpha, - method=method, - cv=cv, - ) - regressor.fit_conformalize(X_cal, y_cal_arr) - y_pred, y_intervals_raw = regressor.predict_interval(X_test) - y_pred_arr = np.clip(np.asarray(y_pred, dtype=float).reshape(-1), 0.0, 1.0) - y_intervals = np.asarray(y_intervals_raw[:, :, 0], dtype=float) - y_intervals = np.clip(y_intervals, 0.0, 1.0) - return y_pred_arr, y_intervals diff --git a/src/models/conformal/pd_intervals.py b/src/models/conformal/pd_intervals.py deleted file mode 100644 index f0013ce..0000000 --- a/src/models/conformal/pd_intervals.py +++ /dev/null @@ -1,291 +0,0 @@ -"""PD conformal interval builders.""" - -from __future__ import annotations - -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - -from src.models.conformal._scores import ( - _compute_score_scale, - _conformal_quantile, - _resolve_score_scale_family, -) -from src.models.conformal_adapters import ProbabilityRegressor - - -def apply_probability_calibrator(calibrator: Any, scores: np.ndarray) -> np.ndarray: - """Apply calibrator robustly across sklearn calibrator API variants.""" - scores = np.asarray(scores, dtype=float) - if calibrator is None: - return np.asarray(np.clip(scores, 0.0, 1.0), dtype=float) - - if hasattr(calibrator, "transform"): - out = calibrator.transform(scores) - return np.asarray(np.clip(np.asarray(out, dtype=float), 0.0, 1.0), dtype=float) - - if hasattr(calibrator, "predict_proba"): - out = calibrator.predict_proba(scores.reshape(-1, 1))[:, 1] - return np.asarray(np.clip(np.asarray(out, dtype=float), 0.0, 1.0), dtype=float) - - try: - out = calibrator.predict(scores) - out = np.asarray(out, dtype=float) - if out.shape[0] != scores.shape[0]: - out = np.asarray(calibrator.predict(scores.reshape(-1, 1)), dtype=float) - except (ValueError, TypeError, IndexError): - out = np.asarray(calibrator.predict(scores.reshape(-1, 1)), dtype=float) - - return np.asarray(np.clip(out, 0.0, 1.0), dtype=float) - - -def create_pd_intervals( - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - alpha: float = 0.1, - calibrator: Any | None = None, -) -> tuple[np.ndarray, np.ndarray]: - """Generate global split-conformal PD intervals via MAPIE.""" - from mapie.regression import SplitConformalRegressor - - prob_reg = ProbabilityRegressor(classifier, calibrator=calibrator) - mapie = SplitConformalRegressor( - estimator=prob_reg, - confidence_level=1 - alpha, - prefit=True, - ) - mapie.conformalize(X_cal, y_cal.astype(float)) - - y_pred, y_intervals_raw = mapie.predict_interval(X_test) - y_intervals = y_intervals_raw[:, :, 0] - - y_intervals = np.clip(y_intervals, 0, 1) - y_pred = np.clip(y_pred, 0, 1) - - avg_width = float((y_intervals[:, 1] - y_intervals[:, 0]).mean()) - logger.info( - f"Conformal PD intervals (global, alpha={alpha}): " - f"avg_width={avg_width:.4f}, target_coverage={1 - alpha:.0%}" - ) - return y_pred, y_intervals - - -def create_pd_intervals_mondrian( - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - group_cal: pd.Series, - group_test: pd.Series, - alpha: float = 0.1, - min_group_size: int = 500, - calibrator: Any | None = None, - scaled_scores: bool = False, - score_scale_family: str = "none", - log_summary: bool = True, -) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: - """Create group-conditional split-conformal PD intervals.""" - y_cal_pred_raw = classifier.predict_proba(X_cal)[:, 1] - y_test_pred_raw = classifier.predict_proba(X_test)[:, 1] - y_cal_pred = apply_probability_calibrator(calibrator, y_cal_pred_raw) - y_test_pred = apply_probability_calibrator(calibrator, y_test_pred_raw) - - return create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_cal_pred, - y_test_pred=y_test_pred, - y_cal=y_cal, - group_cal=group_cal, - group_test=group_test, - alpha=alpha, - min_group_size=min_group_size, - scaled_scores=scaled_scores, - score_scale_family=score_scale_family, - log_summary=log_summary, - ) - - -def create_pd_intervals_mondrian_from_predictions( - *, - y_cal_pred: np.ndarray, - y_test_pred: np.ndarray, - y_cal: pd.Series | np.ndarray, - group_cal: pd.Series | np.ndarray, - group_test: pd.Series | np.ndarray, - alpha: float = 0.1, - min_group_size: int = 500, - scaled_scores: bool = False, - score_scale_family: str = "none", - log_summary: bool = True, -) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: - """Create group-conditional PD intervals from precomputed probabilities. - - The conformal reopen search evaluates many interval designs against the same - model scores. This helper preserves the exact interval math used by - ``create_pd_intervals_mondrian`` while avoiding repeated classifier - inference inside each grid cell. - """ - y_cal_pred = np.clip(np.asarray(y_cal_pred, dtype=float).reshape(-1), 0.0, 1.0) - y_test_pred = np.clip(np.asarray(y_test_pred, dtype=float).reshape(-1), 0.0, 1.0) - if len(y_cal_pred) != len(y_cal): - raise ValueError( - f"Calibration prediction length mismatch: pred={len(y_cal_pred)}, y={len(y_cal)}" - ) - if len(y_test_pred) != len(group_test): - raise ValueError( - f"Test prediction length mismatch: pred={len(y_test_pred)}, groups={len(group_test)}" - ) - - y_cal_arr = np.asarray(y_cal, dtype=float) - g_cal = pd.Series(group_cal).fillna("UNKNOWN").astype(str).to_numpy() - g_test = pd.Series(group_test).fillna("UNKNOWN").astype(str).to_numpy() - - scores = np.abs(y_cal_arr - y_cal_pred) - resolved_scale_family = _resolve_score_scale_family( - scaled_scores=scaled_scores, - score_scale_family=score_scale_family, - ) - cal_scale = _compute_score_scale(y_cal_pred, resolved_scale_family) - test_scale = _compute_score_scale(y_test_pred, resolved_scale_family) - if resolved_scale_family != "none": - scores = scores / cal_scale - - global_q = _conformal_quantile(scores, alpha) - group_quantiles: dict[str, float] = {} - group_cal_counts: dict[str, int] = {} - fallback_groups: list[str] = [] - - all_groups = sorted(set(g_cal).union(set(g_test))) - for g in all_groups: - mask = g_cal == g - n_g = int(mask.sum()) - group_cal_counts[g] = n_g - if n_g >= min_group_size: - group_quantiles[g] = _conformal_quantile(scores[mask], alpha) - else: - group_quantiles[g] = global_q - fallback_groups.append(g) - - radii = np.array([group_quantiles[str(g)] for g in g_test], dtype=float) * test_scale - low = np.clip(y_test_pred - radii, 0.0, 1.0) - high = np.clip(y_test_pred + radii, 0.0, 1.0) - y_intervals = np.column_stack([low, high]) - - diagnostics = { - "alpha": alpha, - "global_quantile": global_q, - "group_quantiles": group_quantiles, - "group_cal_counts": group_cal_counts, - "fallback_groups": fallback_groups, - "scaled_scores": bool(resolved_scale_family != "none"), - "score_scale_family": resolved_scale_family, - "min_group_size": min_group_size, - "avg_width": float((high - low).mean()), - "median_width": float(np.median(high - low)), - } - if log_summary: - logger.info( - "Conformal PD intervals (mondrian): " - f"groups={len(all_groups)}, avg_width={diagnostics['avg_width']:.4f}, " - f"fallback_groups={len(fallback_groups)}" - ) - return y_test_pred, y_intervals, diagnostics - - -def conditional_coverage_by_group( - y_true: np.ndarray, - y_intervals: np.ndarray, - groups: pd.Series | np.ndarray, -) -> pd.DataFrame: - """Compute conditional coverage and width per segment.""" - g = pd.Series(groups).fillna("UNKNOWN").astype(str) - y_true_arr = np.asarray(y_true, dtype=float) - low = y_intervals[:, 0] - high = y_intervals[:, 1] - covered = (y_true_arr >= low) & (y_true_arr <= high) - widths = high - low - - df = pd.DataFrame( - { - "group": g, - "covered": covered.astype(float), - "width": widths, - } - ) - out = ( - df.groupby("group", observed=True) - .agg( - n=("covered", "size"), - coverage=("covered", "mean"), - avg_width=("width", "mean"), - median_width=("width", "median"), - ) - .reset_index() - .sort_values("group") - ) - return out - - -def create_pd_intervals_venn_abers( - classifier: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Generate Venn-Abers multi-probability PD intervals.""" - y_cal_arr = np.asarray(y_cal.values if hasattr(y_cal, "values") else y_cal, dtype=int).reshape( - -1 - ) - - try: - from mapie.calibration import VennAbersCalibrator - - va = VennAbersCalibrator(estimator=classifier, inductive=True, random_state=42) - va.fit( - X_cal, - y_cal_arr, - X_calib=X_cal, - y_calib=y_cal_arr, - ) - probs = va.predict_proba(X_test) - p_low_raw = np.clip(np.asarray(probs[:, 0], dtype=float), 0.0, 1.0) - p_high_raw = np.clip(np.asarray(probs[:, 1], dtype=float), 0.0, 1.0) - p_low = np.minimum(p_low_raw, p_high_raw) - p_high = np.maximum(p_low_raw, p_high_raw) - y_pred_point = np.clip((p_low + p_high) / 2.0, 0.0, 1.0) - avg_width = float((p_high - p_low).mean()) - logger.info( - f"Venn-Abers PD intervals [MAPIE]: avg_width={avg_width:.4f}, n_test={len(X_test)}" - ) - return y_pred_point, p_low, p_high - - except (ImportError, ValueError, TypeError, RuntimeError, AttributeError) as exc: - logger.warning( - f"MAPIE VennAbersCalibrator failed ({exc}) - falling back to venn_abers library." - ) - - from venn_abers import VennAbers - - p_cal_pos = np.asarray(classifier.predict_proba(X_cal)[:, 1], dtype=float).reshape(-1) - p_cal = np.column_stack([1.0 - p_cal_pos, p_cal_pos]) - p_test_pos = np.asarray(classifier.predict_proba(X_test)[:, 1], dtype=float).reshape(-1) - p_test = np.column_stack([1.0 - p_test_pos, p_test_pos]) - - wrapped = VennAbers() - wrapped.fit(p_cal, y_cal_arr) - y_pred_binary, p_result = wrapped.predict_proba(p_test) - p0 = np.clip(np.asarray(p_result[:, 0], dtype=float), 0.0, 1.0) - p1 = np.clip(np.asarray(p_result[:, 1], dtype=float), 0.0, 1.0) - - p_low = np.minimum(p0, p1) - p_high = np.maximum(p0, p1) - y_pred_point = np.clip(np.asarray(y_pred_binary[:, 1], dtype=float), 0.0, 1.0) - - avg_width = float((p_high - p_low).mean()) - logger.info( - f"Venn-Abers PD intervals [fallback]: avg_width={avg_width:.4f}, n_test={len(X_test)}" - ) - return y_pred_point, p_low, p_high diff --git a/src/models/conformal/regression.py b/src/models/conformal/regression.py deleted file mode 100644 index d26b44f..0000000 --- a/src/models/conformal/regression.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Regression-style conformal interval helpers.""" - -from __future__ import annotations - -from typing import Any - -import numpy as np -import pandas as pd -from loguru import logger - - -def create_regression_intervals( - regressor: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - alpha: float = 0.1, -) -> tuple[np.ndarray, np.ndarray]: - """Generate regression intervals using MAPIE.""" - from mapie.regression import SplitConformalRegressor - - mapie = SplitConformalRegressor( - estimator=regressor, - confidence_level=1 - alpha, - prefit=True, - ) - mapie.conformalize(X_cal, y_cal) - - y_pred, y_intervals_raw = mapie.predict_interval(X_test) - y_intervals = y_intervals_raw[:, :, 0] - - avg_width = (y_intervals[:, 1] - y_intervals[:, 0]).mean() - logger.info(f"Conformal regression intervals (alpha={alpha}): avg_width={avg_width:.4f}") - return y_pred, y_intervals - - -def create_residual_intervals( - regressor: Any, - X_cal: pd.DataFrame, - y_cal: pd.Series, - X_test: pd.DataFrame, - alpha: float = 0.1, -) -> tuple[np.ndarray, np.ndarray]: - """Generate naive residual-based prediction intervals as a benchmark.""" - if hasattr(regressor, "predict_proba"): - cal_preds = regressor.predict_proba(X_cal)[:, 1] - test_preds = regressor.predict_proba(X_test)[:, 1] - else: - cal_preds = regressor.predict(X_cal) - test_preds = regressor.predict(X_test) - - cal_preds = np.asarray(cal_preds, dtype=float) - test_preds = np.asarray(test_preds, dtype=float) - y_cal_arr = np.asarray(y_cal, dtype=float) - - residuals = y_cal_arr - cal_preds - q_low = np.percentile(residuals, 100 * (alpha / 2)) - q_high = np.percentile(residuals, 100 * (1 - alpha / 2)) - - low = test_preds + q_low - high = test_preds + q_high - y_intervals = np.column_stack([low, high]) - - avg_width = float((high - low).mean()) - logger.info(f"Residual intervals (bootstrap-style, alpha={alpha}): avg_width={avg_width:.4f}") - return test_preds, y_intervals diff --git a/src/models/conformal_adapters.py b/src/models/conformal_adapters.py deleted file mode 100644 index 8918709..0000000 --- a/src/models/conformal_adapters.py +++ /dev/null @@ -1,125 +0,0 @@ -"""sklearn-style adapter classes for the conformal stack. - -This module was split out of ``src.models.conformal`` to make the conformal -stack easier to navigate. The three classes here are -exactly the ones the frozen ``models/pd_canonical_calibrator.pkl`` (and any -older calibrator pickles) reference by their fully-qualified -``src.models.conformal.`` path. - -To keep those pickles working without re-pickling, every class below sets -its ``__module__`` attribute back to ``"src.models.conformal"`` after -definition. ``src.models.conformal`` then re-imports each class so the public -name is reachable at both ``src.models.conformal.`` (the legacy -path) and ``src.models.conformal_adapters.`` (the implementation -home). - -Pickle behaviour: - -* ``pickle.load`` on a pre-refactor pickle resolves - ``src.models.conformal.ProbabilityRegressor`` to the re-exported package - name, which points to the class here. -* ``pickle.dump`` of a freshly constructed instance writes - ``src.models.conformal.ProbabilityRegressor`` because of the - ``__module__`` override, so future loads remain stable. -""" - -from __future__ import annotations - -from typing import Any, Self - -import numpy as np -import pandas as pd -from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin - -# Canonical module path that pickled instances must report. -_PICKLE_MODULE = "src.models.conformal" - - -class ProbabilityRegressor(BaseEstimator, RegressorMixin): - """Wrap classifier predict_proba as a regression predictor. - - Optionally applies a probability calibrator after raw predictions. - """ - - def __init__(self, classifier: Any, calibrator: Any | None = None) -> None: - self.classifier = classifier - self.calibrator = calibrator - self.is_fitted_ = True # required for MAPIE prefit checks - - def fit(self, X: Any, y: Any) -> Self: - """Already fitted — no-op for MAPIE interface.""" - return self - - def predict(self, X: Any) -> np.ndarray: - """Return calibrated P(default) in [0, 1].""" - # Local import keeps the heavy ``apply_probability_calibrator`` lazy - # and avoids a circular import between ``conformal`` and this module. - from src.models.conformal import apply_probability_calibrator - - raw = self.classifier.predict_proba(X)[:, 1] - return apply_probability_calibrator(self.calibrator, raw) - - -class PrefitClassifierAdapter(ClassifierMixin, BaseEstimator): - """Small sklearn-style adapter for prefit classifiers inside MAPIE checks.""" - - def __init__(self, classifier: Any, n_features_in: int | None = None) -> None: - self.classifier = classifier - classes = getattr(classifier, "classes_", np.array([0, 1])) - self.classes_ = np.asarray(classes) - self.n_features_in_ = int(n_features_in or getattr(classifier, "n_features_in_", 0) or 0) - self.feature_names_in_ = np.asarray( - [f"f{i}" for i in range(self.n_features_in_)], dtype=object - ) - self.is_fitted_ = True - - def fit(self, X: Any, y: Any) -> Self: - return self - - def _is_minimal_probe(self, X: pd.DataFrame) -> bool: - if X.shape[0] != 1 or X.shape[1] != self.n_features_in_: - return False - numeric = X.apply(pd.to_numeric, errors="coerce") - return bool(np.isfinite(numeric.to_numpy()).all() and np.allclose(numeric.to_numpy(), 0.0)) - - def predict(self, X: Any) -> np.ndarray: - X_df = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X - if self._is_minimal_probe(X_df): - return np.zeros(len(X_df), dtype=int) - return np.asarray(self.classifier.predict(X_df)) - - def predict_proba(self, X: Any) -> np.ndarray: - X_df = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X - if self._is_minimal_probe(X_df): - return np.column_stack([np.ones(len(X_df)), np.zeros(len(X_df))]) - return np.asarray(self.classifier.predict_proba(X_df)) - - -class PrefitCalibratedClassifierAdapter(PrefitClassifierAdapter): - """Prefit classifier adapter that applies a probability calibrator.""" - - def __init__( - self, - classifier: Any, - calibrator: Any | None = None, - n_features_in: int | None = None, - ) -> None: - super().__init__(classifier, n_features_in=n_features_in) - self.calibrator = calibrator - - def predict_proba(self, X: Any) -> np.ndarray: - from src.models.conformal import apply_probability_calibrator - - raw = super().predict_proba(X) - if self.calibrator is None: - return raw - p_pos = apply_probability_calibrator(self.calibrator, raw[:, 1]) - p_neg = np.clip(1.0 - p_pos, 0.0, 1.0) - return np.column_stack([p_neg, p_pos]) - - -# --------------------------------------------------------------------------- -# Pickle compatibility shim: report the legacy fully-qualified path. -# --------------------------------------------------------------------------- -for _cls in (ProbabilityRegressor, PrefitClassifierAdapter, PrefitCalibratedClassifierAdapter): - _cls.__module__ = _PICKLE_MODULE diff --git a/src/models/conformal_alpha_grid.py b/src/models/conformal_alpha_grid.py deleted file mode 100644 index cad3e6a..0000000 --- a/src/models/conformal_alpha_grid.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Exact alpha-grid replay for a frozen Mondrian conformal recipe.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -import numpy as np -import pandas as pd - -from src.models.conformal import ( - build_mondrian_partition_labels, - create_pd_intervals_mondrian_from_predictions, -) -from src.models.conformal_tuning import apply_group_multipliers, build_group_temporal_segments - - -@dataclass(frozen=True) -class FrozenConformalRecipe: - """Selected interval design and holdout-learned widening policy.""" - - partition: str - partition_probability_source: str - n_score_bins: int - fallback_mode: str - score_scale_family: str - min_group_size: int - reference_target_alpha: float - reference_used_alpha: float - calibration_fraction: float - tuning_holdout_ratio: float - tuning_random_state: int - group_multipliers: dict[str, float] - temporal_segment_multipliers: dict[str, float] - temporal_segment_freq: str - global_rebalance_factor: float - - @classmethod - def from_results_payload(cls, payload: dict[str, Any]) -> FrozenConformalRecipe: - """Build a replay recipe from ``conformal_results_mondrian.pkl``.""" - selected = payload["tuning_90_best"] - split = payload["calibration_split"] - global_rebalance = payload.get("global_rebalance", {}) or {} - factor = ( - float(global_rebalance.get("factor", 1.0)) - if bool(global_rebalance.get("applied", False)) - else 1.0 - ) - recipe = cls( - partition=str(selected["partition"]), - partition_probability_source=str(selected["partition_probability_source"]), - n_score_bins=int(selected["n_score_bins"]), - fallback_mode=str(selected["fallback_mode"]), - score_scale_family=str(selected["score_scale_family"]), - min_group_size=int(selected["min_group_size"]), - reference_target_alpha=float(selected["alpha_target_90"]), - reference_used_alpha=float(selected["alpha_used_90"]), - calibration_fraction=float(split["calibration_fraction"]), - tuning_holdout_ratio=float(split["holdout_ratio"]), - tuning_random_state=int(split["random_state"]), - group_multipliers={ - str(key): float(value) - for key, value in (payload.get("group_coverage_multipliers", {}) or {}).items() - }, - temporal_segment_multipliers={ - str(key): float(value) - for key, value in (payload.get("temporal_segment_multipliers", {}) or {}).items() - }, - temporal_segment_freq=str(payload.get("temporal_segment_freq", "Q")), - global_rebalance_factor=factor, - ) - recipe.validate() - return recipe - - def validate(self) -> None: - """Reject recipe settings that could narrow a nominal interval silently.""" - if not 0.0 < self.reference_target_alpha < 1.0: - raise ValueError("reference_target_alpha must lie in (0, 1).") - if not 0.0 < self.reference_used_alpha <= self.reference_target_alpha: - raise ValueError( - "reference_used_alpha must be positive and no larger than the target alpha." - ) - multipliers = [ - *self.group_multipliers.values(), - *self.temporal_segment_multipliers.values(), - self.global_rebalance_factor, - ] - if any(value < 1.0 for value in multipliers): - raise ValueError("Frozen alpha-grid replay only supports widening adjustments.") - - def used_alpha(self, target_alpha: float) -> float: - """Apply the frozen conservative alpha ratio selected at the reference level.""" - target = float(target_alpha) - if not 0.0 < target < 1.0: - raise ValueError("target_alpha must lie in (0, 1).") - ratio = self.reference_used_alpha / self.reference_target_alpha - return target * ratio - - -@dataclass(frozen=True) -class ExactAlphaIntervals: - """One exact conformal interval vector and its replay metadata.""" - - target_alpha: float - used_alpha: float - point: np.ndarray - low: np.ndarray - high: np.ndarray - partition_labels: pd.Series - partition_metadata: dict[str, Any] - diagnostics: dict[str, Any] - - -def alpha_column_token(alpha: float) -> str: - """Return a stable column-safe token such as ``0p010``.""" - return f"{float(alpha):.3f}".replace(".", "p") - - -def alpha_interval_columns(alpha: float) -> tuple[str, str]: - """Return the low/high column names for an exact alpha-grid artifact.""" - token = alpha_column_token(alpha) - return f"pd_low_alpha_{token}", f"pd_high_alpha_{token}" - - -def _scale_around_prediction( - point: np.ndarray, - intervals: np.ndarray, - factor: float, -) -> np.ndarray: - radius = np.maximum(point - intervals[:, 0], intervals[:, 1] - point) - return np.column_stack( - [ - np.clip(point - radius * factor, 0.0, 1.0), - np.clip(point + radius * factor, 0.0, 1.0), - ] - ) - - -def compute_exact_alpha_intervals( - *, - recipe: FrozenConformalRecipe, - target_alpha: float, - y_cal: pd.Series | np.ndarray, - interval_probability_cal: np.ndarray, - interval_probability_eval: np.ndarray, - partition_probability_cal: np.ndarray, - partition_probability_eval: np.ndarray, - base_groups_cal: pd.Series | np.ndarray, - base_groups_eval: pd.Series | np.ndarray, - issue_dates_eval: pd.Series | np.ndarray | None = None, -) -> ExactAlphaIntervals: - """Recompute one alpha exactly under a frozen partition and widening recipe.""" - group_cal, group_eval, partition_metadata = build_mondrian_partition_labels( - y_prob_cal=partition_probability_cal, - y_prob_eval=partition_probability_eval, - partition=recipe.partition, - base_groups_cal=base_groups_cal, - base_groups_eval=base_groups_eval, - n_score_bins=recipe.n_score_bins, - min_group_size=recipe.min_group_size, - fallback_mode=recipe.fallback_mode, - ) - used_alpha = recipe.used_alpha(target_alpha) - point, intervals, diagnostics = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=interval_probability_cal, - y_test_pred=interval_probability_eval, - y_cal=y_cal, - group_cal=group_cal, - group_test=group_eval, - alpha=used_alpha, - min_group_size=recipe.min_group_size, - score_scale_family=recipe.score_scale_family, - log_summary=False, - ) - if recipe.group_multipliers: - intervals = apply_group_multipliers( - point, - intervals, - group_eval, - recipe.group_multipliers, - ) - if recipe.temporal_segment_multipliers: - if issue_dates_eval is None: - raise ValueError("issue_dates_eval is required by the frozen temporal multipliers.") - temporal_segments = build_group_temporal_segments( - group_eval, - issue_dates_eval, - freq=recipe.temporal_segment_freq, - ) - intervals = apply_group_multipliers( - point, - intervals, - temporal_segments, - recipe.temporal_segment_multipliers, - ) - if not np.isclose(recipe.global_rebalance_factor, 1.0): - intervals = _scale_around_prediction( - point, - intervals, - recipe.global_rebalance_factor, - ) - return ExactAlphaIntervals( - target_alpha=float(target_alpha), - used_alpha=used_alpha, - point=point, - low=intervals[:, 0], - high=intervals[:, 1], - partition_labels=group_eval, - partition_metadata=partition_metadata, - diagnostics=diagnostics, - ) diff --git a/src/models/conformal_artifacts.py b/src/models/conformal_artifacts.py deleted file mode 100644 index d375126..0000000 --- a/src/models/conformal_artifacts.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Utilities for loading conformal artifacts with canonical-path preference.""" - -from __future__ import annotations - -from pathlib import Path - -import pandas as pd -from loguru import logger - -CANONICAL_INTERVALS_PATH = Path("data/processed/conformal_intervals_mondrian.parquet") - - -def resolve_intervals_path( - override_path: str | Path | None = None, -) -> tuple[Path, bool]: - """Resolve conformal intervals artifact path. - - Returns: - path: selected artifact path - is_legacy: whether selected path is the legacy compatibility artifact - """ - if override_path is not None: - path = Path(override_path) - if not path.exists(): - raise FileNotFoundError(f"Conformal intervals override not found: {path}") - return path, False - - if CANONICAL_INTERVALS_PATH.exists(): - return CANONICAL_INTERVALS_PATH, False - - raise FileNotFoundError( - "Conformal intervals artifact not found. Expected canonical path " - f"'{CANONICAL_INTERVALS_PATH}'." - ) - - -def load_conformal_intervals( - override_path: str | Path | None = None, -) -> tuple[pd.DataFrame, Path, bool]: - """Load conformal interval artifact and return dataframe + selected path metadata.""" - path, is_legacy = resolve_intervals_path(override_path=override_path) - if override_path is not None: - logger.info(f"Using conformal intervals override artifact: {path}") - else: - logger.info(f"Using canonical conformal artifact: {path}") - - df = pd.read_parquet(path) - return df, path, is_legacy diff --git a/src/models/conformal_diagnostics.py b/src/models/conformal_diagnostics.py deleted file mode 100644 index 8b5bb02..0000000 --- a/src/models/conformal_diagnostics.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Conformal prediction diagnostics — first step of the conformal split. - -The functions here used to live inside ``src/models/conformal.py``. They are -**pure data summaries** — no MAPIE calls, no classifier state, no pickled -references — so moving them does not affect ``pd_canonical_calibrator.pkl`` -deserialization. - -The legacy import path keeps working: - - >>> from src.models.conformal import validate_coverage # still valid - -because ``conformal.py`` re-exports both names. New call sites should prefer: - - >>> from src.models.conformal_diagnostics import validate_coverage - -See ``docs/refactor/CONFORMAL_REFACTOR_PLAN.md`` for the full split plan. -""" - -from __future__ import annotations - -import numpy as np -from loguru import logger - - -def validate_coverage( - y_true: np.ndarray, - y_intervals: np.ndarray, - alpha: float, - log_summary: bool = True, -) -> dict[str, float]: - """Validate empirical coverage of a conformal interval set against its target. - - Args: - y_true: observed labels. - y_intervals: ``(n, 2)`` array of ``[low, high]`` per observation. - alpha: miscoverage rate. Target coverage is ``1 - alpha``. - log_summary: emit a one-line diagnostic summary. Large grid searches - set this to ``False`` to avoid logging becoming the bottleneck. - - Returns: - Dict with empirical_coverage, target_coverage, coverage_gap, - avg_interval_width and median_interval_width. - """ - low = y_intervals[:, 0] - high = y_intervals[:, 1] - covered = ((y_true >= low) & (y_true <= high)).mean() - target = 1 - alpha - - metrics = { - "empirical_coverage": float(covered), - "target_coverage": float(target), - "coverage_gap": float(abs(covered - target)), - "avg_interval_width": float((high - low).mean()), - "median_interval_width": float(np.median(high - low)), - } - if log_summary: - logger.info(f"Coverage validation: empirical={covered:.4f} vs target={target:.4f}") - return metrics - - -def summarize_prediction_sets( - y_true: np.ndarray, - y_pred: np.ndarray, - y_sets: np.ndarray, -) -> dict[str, float]: - """Summarize binary conformal prediction sets for abstention analysis. - - Reports singleton/ambiguous/empty rates, set coverage and conditional - default rates by set type. - """ - true_arr = np.asarray(y_true, dtype=int).reshape(-1) - pred_arr = np.asarray(y_pred, dtype=int).reshape(-1) - sets = np.asarray(y_sets, dtype=int) - if sets.ndim != 2: - raise ValueError(f"Expected y_sets to be 2D, got shape={sets.shape}") - if len(true_arr) != len(sets): - raise ValueError("y_true and y_sets must have the same length.") - - set_size = sets.sum(axis=1) - singleton_mask = set_size == 1 - ambiguous_mask = set_size > 1 - empty_mask = set_size == 0 - covered_mask = sets[np.arange(len(true_arr)), true_arr] == 1 - positive_singleton_mask = singleton_mask & (pred_arr == 1) - - return { - "n_obs": float(len(true_arr)), - "set_coverage": float(covered_mask.mean()) if len(true_arr) else float("nan"), - "singleton_rate": float(singleton_mask.mean()) if len(true_arr) else float("nan"), - "ambiguity_rate": float(ambiguous_mask.mean()) if len(true_arr) else float("nan"), - "empty_set_rate": float(empty_mask.mean()) if len(true_arr) else float("nan"), - "default_rate_ambiguous": float(true_arr[ambiguous_mask].mean()) - if ambiguous_mask.any() - else float("nan"), - "default_rate_singleton_positive": float(true_arr[positive_singleton_mask].mean()) - if positive_singleton_mask.any() - else float("nan"), - "default_rate_overall": float(true_arr.mean()) if len(true_arr) else float("nan"), - } diff --git a/src/models/conformal_tuning.py b/src/models/conformal_tuning.py deleted file mode 100644 index e32226f..0000000 --- a/src/models/conformal_tuning.py +++ /dev/null @@ -1,883 +0,0 @@ -"""Conformal interval tuning utilities. - -Extracted from scripts/generate_conformal_intervals.py to keep the script -under the 400-line guideline. Contains: -- Calibration split logic for leakage-free hyperparameter tuning. -- Pareto front identification for multi-objective config selection. -- Hierarchical config selection with guardbands. -- Group coverage floor enforcement via interval widening. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, cast - -import numpy as np -import pandas as pd -from loguru import logger -from sklearn.model_selection import train_test_split - - -def split_calibration_for_tuning( - y_cal: pd.Series, - group_cal: pd.Series, - issue_dates: pd.Series | None = None, - holdout_ratio: float = 0.20, - random_state: int = 42, -) -> tuple[np.ndarray, np.ndarray]: - """Split calibration rows into fit/tuning partitions without touching test labels. - - Prefers a temporal split using ``issue_dates`` (latest tail as tuning holdout). - Falls back to stratified random split when temporal metadata is unavailable or - would create degenerate class partitions. - """ - n = int(len(y_cal)) - if n <= 1: - idx = np.arange(n, dtype=int) - return idx, np.array([], dtype=int) - - holdout_ratio = float(np.clip(holdout_ratio, 0.05, 0.50)) - idx = np.arange(n, dtype=int) - - n_tune = max(1, int(round(n * holdout_ratio))) - n_tune = min(n - 1, n_tune) - y_arr = np.asarray(y_cal, dtype=float) - - if issue_dates is not None: - issue_dt = pd.to_datetime(issue_dates, errors="coerce") - valid_dates = int(issue_dt.notna().sum()) - if valid_dates >= max(100, int(0.70 * n)): - ordered = pd.DataFrame({"idx": idx, "issue_d": issue_dt}) - ordered["issue_d_filled"] = ordered["issue_d"].fillna(pd.Timestamp("1900-01-01")) - ordered = ordered.sort_values(["issue_d_filled", "idx"]).reset_index(drop=True) - - idx_sorted = ordered["idx"].to_numpy(dtype=int) - idx_fit = idx_sorted[:-n_tune] - idx_tune = idx_sorted[-n_tune:] - - fit_classes = np.unique(y_arr[idx_fit].astype(int)) - tune_classes = np.unique(y_arr[idx_tune].astype(int)) - if len(fit_classes) >= 2 and len(tune_classes) >= 2: - logger.info( - "Using temporal calibration holdout by issue_d: " - f"valid_dates={valid_dates:,}/{n:,}, holdout_ratio={holdout_ratio:.2%}" - ) - return np.sort(idx_fit), np.sort(idx_tune) - - logger.warning( - "Temporal calibration split produced single-class partition; " - "falling back to stratified random split." - ) - - stratify = ( - pd.Series(group_cal).fillna("UNKNOWN").astype(str) - + "|" - + pd.Series(y_cal).astype(int).astype(str) - ) - try: - idx_fit, idx_tune = train_test_split( - idx, - test_size=holdout_ratio, - random_state=random_state, - stratify=stratify, - ) - except ValueError: - logger.warning( - "Stratified split failed for calibration holdout; using deterministic random split." - ) - rng = np.random.default_rng(random_state) - shuffled = idx.copy() - rng.shuffle(shuffled) - idx_tune = shuffled[:n_tune] - idx_fit = shuffled[n_tune:] - - return np.sort(np.asarray(idx_fit, dtype=int)), np.sort(np.asarray(idx_tune, dtype=int)) - - -def mark_pareto_front(results_df: pd.DataFrame) -> pd.Series: - """Pareto front for (maximize coverage, maximize min group coverage, minimize width).""" - n = len(results_df) - dominated = np.zeros(n, dtype=bool) - arr_cov = results_df["empirical_coverage"].to_numpy(dtype=float) - arr_grp = results_df["min_group_coverage"].to_numpy(dtype=float) - arr_wid = results_df["avg_interval_width"].to_numpy(dtype=float) - - for i in range(n): - if dominated[i]: - continue - for j in range(n): - if i == j: - continue - better_or_equal = ( - arr_cov[j] >= arr_cov[i] and arr_grp[j] >= arr_grp[i] and arr_wid[j] <= arr_wid[i] - ) - strictly_better = ( - arr_cov[j] > arr_cov[i] or arr_grp[j] > arr_grp[i] or arr_wid[j] < arr_wid[i] - ) - if better_or_equal and strictly_better: - dominated[i] = True - break - return pd.Series(~dominated, index=results_df.index, dtype=bool) - - -def choose_best_tuning_row( - results_df: pd.DataFrame, - target_coverage: float, - min_group_coverage_target: float, - max_width_budget: float | None = None, - coverage_guardband: float = 0.015, - min_group_guardband: float = 0.0, -) -> tuple[pd.Series, str]: - """Select config with hierarchical multi-objective constraints.""" - df = results_df.copy() - df["global_ok"] = df["empirical_coverage"] >= target_coverage - df["group_ok"] = df["min_group_coverage"] >= min_group_coverage_target - strong_cov_target = target_coverage + max(0.0, float(coverage_guardband)) - strong_group_target = min_group_coverage_target + max(0.0, float(min_group_guardband)) - df["global_strong"] = df["empirical_coverage"] >= strong_cov_target - df["group_strong"] = df["min_group_coverage"] >= strong_group_target - df["coverage_guard_shortfall"] = (strong_cov_target - df["empirical_coverage"]).clip(lower=0.0) - df["group_guard_shortfall"] = (strong_group_target - df["min_group_coverage"]).clip(lower=0.0) - - if max_width_budget is None: - df["width_ok"] = True - else: - df["width_ok"] = df["avg_interval_width"] <= max_width_budget - - tiers = [ - ( - "strong_global+strong_group+width", - df["global_strong"] & df["group_strong"] & df["width_ok"], - ), - ("strong_global+strong_group", df["global_strong"] & df["group_strong"]), - ("strong_global+width", df["global_strong"] & df["width_ok"]), - ("strong_global_only", df["global_strong"]), - ("global+group+width", df["global_ok"] & df["group_ok"] & df["width_ok"]), - ("global+group", df["global_ok"] & df["group_ok"]), - ("global+width", df["global_ok"] & df["width_ok"]), - ("global_only", df["global_ok"]), - ] - for tier_name, mask in tiers: - candidate = df[mask].copy() - if not candidate.empty: - sort_cols = [ - col - for col in [ - "coverage_guard_shortfall", - "group_guard_shortfall", - "coverage_gap", - "avg_interval_width", - "winkler_90", - "max_monthly_gap", - "stability_over_time", - "min_group_coverage", - ] - if col in candidate.columns - ] - ascending = [col not in {"min_group_coverage"} for col in sort_cols] - candidate = candidate.sort_values( - by=sort_cols, - ascending=ascending, - ) - return candidate.iloc[0], tier_name - - # Fallback: penalty score - fallback = df.copy() - fallback["coverage_shortfall"] = (target_coverage - fallback["empirical_coverage"]).clip( - lower=0.0 - ) - fallback["group_shortfall"] = (min_group_coverage_target - fallback["min_group_coverage"]).clip( - lower=0.0 - ) - if max_width_budget is None: - fallback["width_excess"] = 0.0 - else: - fallback["width_excess"] = (fallback["avg_interval_width"] - max_width_budget).clip( - lower=0.0 - ) - fallback["winkler_penalty"] = fallback.get("winkler_90", pd.Series(0.0, index=fallback.index)) - fallback["monthly_gap_penalty"] = fallback.get( - "max_monthly_gap", pd.Series(0.0, index=fallback.index) - ) - fallback["stability_penalty"] = fallback.get( - "stability_over_time", pd.Series(0.0, index=fallback.index) - ) - fallback["score"] = ( - 120.0 * fallback["coverage_guard_shortfall"] - + 80.0 * fallback["group_guard_shortfall"] - + 40.0 * fallback["coverage_shortfall"] - + 20.0 * fallback["group_shortfall"] - + 10.0 * fallback["width_excess"] - + 8.0 * fallback["winkler_penalty"] - + 6.0 * fallback["monthly_gap_penalty"] - + 4.0 * fallback["stability_penalty"] - + fallback["avg_interval_width"] - ) - fallback = fallback.sort_values( - by=[ - "score", - "coverage_shortfall", - "group_shortfall", - "winkler_penalty", - "avg_interval_width", - ], - ascending=[True, True, True, True, True], - ) - return fallback.iloc[0], "fallback_penalty" - - -def apply_group_multipliers( - y_pred: np.ndarray, - y_intervals: np.ndarray, - groups: pd.Series | np.ndarray, - multipliers: dict[str, float], -) -> np.ndarray: - """Apply group-specific interval multipliers around point predictions.""" - g = pd.Series(groups).fillna("UNKNOWN").astype(str).to_numpy() - low = y_intervals[:, 0].astype(float).copy() - high = y_intervals[:, 1].astype(float).copy() - radius = np.maximum(y_pred - low, high - y_pred) - out_low = low.copy() - out_high = high.copy() - for group, factor in multipliers.items(): - if factor <= 1.0: - continue - mask = g == str(group) - if not mask.any(): - continue - out_low[mask] = np.clip(y_pred[mask] - radius[mask] * factor, 0.0, 1.0) - out_high[mask] = np.clip(y_pred[mask] + radius[mask] * factor, 0.0, 1.0) - return np.column_stack([out_low, out_high]) - - -def enforce_group_coverage_floor( - y_true: np.ndarray, - y_pred: np.ndarray, - y_intervals: np.ndarray, - groups: pd.Series | np.ndarray, - target_coverage: float, - multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), -) -> tuple[np.ndarray, dict[str, float], pd.DataFrame]: - """Increase interval radii for undercovered groups to meet coverage floor.""" - g = pd.Series(groups).fillna("UNKNOWN").astype(str).to_numpy() - y_true_arr = np.asarray(y_true, dtype=float) - base = y_intervals.astype(float).copy() - current = base.copy() - - def _group_cov(intervals: np.ndarray, group: str) -> float: - mask = g == group - if not mask.any(): - return float("nan") - return float( - ( - (y_true_arr[mask] >= intervals[mask, 0]) & (y_true_arr[mask] <= intervals[mask, 1]) - ).mean() - ) - - group_factors: dict[str, float] = {} - report_rows: list[dict[str, Any]] = [] - group_list = sorted(set(g)) - - for group in group_list: - before_cov = _group_cov(current, group) - factor = 1.0 - after_cov = before_cov - if np.isfinite(before_cov) and before_cov < target_coverage: - mask = g == group - candidate = current.copy() - for m in multiplier_grid: - if m < 1.0: - continue - trial = current.copy() - trial_group = apply_group_multipliers( - y_pred=y_pred[mask], - y_intervals=current[mask], - groups=np.array([group] * int(mask.sum())), - multipliers={group: float(m)}, - ) - trial[mask] = trial_group - cov = _group_cov(trial, group) - if cov >= target_coverage: - candidate = trial - factor = float(m) - after_cov = cov - break - candidate = trial - factor = float(m) - after_cov = cov - current = candidate - - if factor > 1.0: - group_factors[group] = factor - report_rows.append( - { - "group": group, - "coverage_before": float(before_cov), - "coverage_after": float(after_cov), - "target_coverage": float(target_coverage), - "multiplier": float(factor), - "adjusted": bool(factor > 1.0), - } - ) - - report = pd.DataFrame(report_rows).sort_values("group") - return current, group_factors, report - - -def build_group_temporal_segments( - groups: pd.Series | np.ndarray, - issue_dates: pd.Series | np.ndarray, - *, - freq: str = "Q", - missing_bucket: str = "UNKNOWN", -) -> pd.Series: - """Build stable segment keys for group-vintage adjustments.""" - group_series = pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True) - issue_dt = pd.to_datetime(pd.Series(issue_dates), errors="coerce").reset_index(drop=True) - vintage = issue_dt.dt.to_period(freq).astype(str) - vintage = vintage.where(issue_dt.notna(), missing_bucket) - return (group_series + "|vintage=" + vintage.astype(str)).astype(str) - - -def enforce_segment_coverage_floor( - y_true: np.ndarray, - y_pred: np.ndarray, - y_intervals: np.ndarray, - segments: pd.Series | np.ndarray, - target_coverage: float, - min_segment_size: int = 250, - multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), -) -> tuple[np.ndarray, dict[str, float], pd.DataFrame]: - """Increase interval radii for undercovered temporal segments. - - Segment adjustments are learned only for segments with enough support to avoid - overfitting tiny slices. Output report includes per-segment support. - """ - seg = pd.Series(segments).fillna("UNKNOWN").astype(str).to_numpy() - y_true_arr = np.asarray(y_true, dtype=float) - current = np.asarray(y_intervals, dtype=float).copy() - min_segment_size = max(1, int(min_segment_size)) - - def _mask_for(segment: str) -> np.ndarray: - return cast(np.ndarray, seg == segment) - - def _segment_cov(intervals: np.ndarray, segment: str) -> float: - mask = _mask_for(segment) - if not mask.any(): - return float("nan") - return float( - ( - (y_true_arr[mask] >= intervals[mask, 0]) & (y_true_arr[mask] <= intervals[mask, 1]) - ).mean() - ) - - segment_factors: dict[str, float] = {} - rows: list[dict[str, Any]] = [] - segment_list = sorted(set(seg)) - - for segment in segment_list: - mask = _mask_for(segment) - support = int(mask.sum()) - before_cov = _segment_cov(current, segment) - factor = 1.0 - after_cov = before_cov - adjusted = False - - if support >= min_segment_size and np.isfinite(before_cov) and before_cov < target_coverage: - candidate = current.copy() - for m in multiplier_grid: - if m < 1.0: - continue - trial = current.copy() - trial_segment = apply_group_multipliers( - y_pred=y_pred[mask], - y_intervals=current[mask], - groups=np.array([segment] * support), - multipliers={segment: float(m)}, - ) - trial[mask] = trial_segment - cov = _segment_cov(trial, segment) - candidate = trial - factor = float(m) - after_cov = cov - if cov >= target_coverage: - break - current = candidate - adjusted = factor > 1.0 - - if factor > 1.0: - segment_factors[str(segment)] = factor - rows.append( - { - "segment": str(segment), - "support_n": support, - "coverage_before": float(before_cov), - "coverage_after": float(after_cov), - "target_coverage": float(target_coverage), - "min_segment_size": int(min_segment_size), - "multiplier": float(factor), - "adjusted": bool(adjusted), - } - ) - - report = pd.DataFrame(rows).sort_values("segment").reset_index(drop=True) - return current, segment_factors, report - - -def to_python_scalar(value: Any) -> Any: - """Convert numpy/pandas scalar values to Python primitives.""" - if isinstance(value, np.floating | np.integer | np.bool_): - return value.item() - return value - - -def empirical_interval_coverage(y_true: np.ndarray, y_intervals: np.ndarray) -> float: - y_true_arr = np.asarray(y_true, dtype=float) - intervals = np.asarray(y_intervals, dtype=float) - if len(y_true_arr) == 0 or len(intervals) == 0: - return float("nan") - inside = (y_true_arr >= intervals[:, 0]) & (y_true_arr <= intervals[:, 1]) - return float(np.mean(inside)) - - -def min_group_interval_coverage( - y_true: np.ndarray, - y_intervals: np.ndarray, - groups: pd.Series | np.ndarray, -) -> float: - g = pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_true_arr = np.asarray(y_true, dtype=float) - intervals = np.asarray(y_intervals, dtype=float) - covs: list[float] = [] - for group in sorted(g.unique()): - mask = g == group - if not mask.any(): - continue - inside = (y_true_arr[mask] >= intervals[mask, 0]) & (y_true_arr[mask] <= intervals[mask, 1]) - covs.append(float(np.mean(inside))) - return float(min(covs)) if covs else float("nan") - - -def average_interval_width(y_intervals: np.ndarray) -> float: - intervals = np.asarray(y_intervals, dtype=float) - if len(intervals) == 0: - return float("nan") - return float(np.mean(intervals[:, 1] - intervals[:, 0])) - - -def mean_winkler_score( - y_true: np.ndarray, - y_intervals: np.ndarray, - *, - alpha: float, -) -> float: - y_true_arr = np.asarray(y_true, dtype=float) - intervals = np.asarray(y_intervals, dtype=float) - if len(y_true_arr) == 0 or len(intervals) == 0: - return float("inf") - low = intervals[:, 0] - high = intervals[:, 1] - width = np.maximum(high - low, 0.0) - below = np.maximum(low - y_true_arr, 0.0) - above = np.maximum(y_true_arr - high, 0.0) - penalty = (2.0 / max(float(alpha), 1e-12)) * (below + above) - return float(np.mean(width + penalty)) - - -def temporal_stability_summary( - y_true: np.ndarray, - y_intervals: np.ndarray, - issue_dates: pd.Series | np.ndarray | None, - *, - target_coverage: float, - freq: str = "M", -) -> dict[str, float]: - if issue_dates is None: - return { - "min_monthly_coverage": float("nan"), - "last_monthly_coverage": float("nan"), - "max_monthly_gap": float("nan"), - "stability_over_time": float("nan"), - } - dates = pd.to_datetime(pd.Series(issue_dates), errors="coerce") - y_true_arr = np.asarray(y_true, dtype=float) - intervals = np.asarray(y_intervals, dtype=float) - if len(dates) == 0 or len(y_true_arr) == 0 or len(intervals) == 0: - return { - "min_monthly_coverage": float("nan"), - "last_monthly_coverage": float("nan"), - "max_monthly_gap": float("nan"), - "stability_over_time": float("nan"), - } - frame = pd.DataFrame( - { - "month": dates.dt.to_period(freq).dt.to_timestamp(), - "y_true": y_true_arr, - "low": intervals[:, 0], - "high": intervals[:, 1], - } - ).dropna(subset=["month"]) - if frame.empty: - return { - "min_monthly_coverage": float("nan"), - "last_monthly_coverage": float("nan"), - "max_monthly_gap": float("nan"), - "stability_over_time": float("nan"), - } - frame["covered"] = ( - (frame["y_true"] >= frame["low"]) & (frame["y_true"] <= frame["high"]) - ).astype(float) - monthly = ( - frame.groupby("month", observed=True) - .agg(coverage=("covered", "mean")) - .reset_index() - .sort_values("month") - ) - monthly["gap"] = (monthly["coverage"] - float(target_coverage)).abs() - return { - "min_monthly_coverage": float(monthly["coverage"].min()), - "last_monthly_coverage": float(monthly["coverage"].iloc[-1]), - "max_monthly_gap": float(monthly["gap"].max()), - "stability_over_time": float(monthly["gap"].mean()), - } - - -@dataclass(frozen=True) -class _ShrinkContext: - y_true: np.ndarray - y_pred: np.ndarray - base_intervals: np.ndarray - groups: pd.Series - temporal_segments: pd.Series | None - issue_dates: pd.Series | np.ndarray | None - target_coverage: float - min_group_coverage_target: float - max_monthly_gap_target: float | None - alpha: float - - -@dataclass(frozen=True) -class _ShrinkCandidate: - scope: str - key: str - factor: float - accepted: bool - intervals: np.ndarray - group_factors: dict[str, float] - temporal_factors: dict[str, float] - metrics: dict[str, float] - - -def _active_widening_factors(factors: dict[str, float] | None) -> dict[str, float]: - return {str(k): float(v) for k, v in (factors or {}).items() if float(v) > 1.0} - - -def _apply_shrink_factors( - context: _ShrinkContext, - group_factors: dict[str, float], - temporal_factors: dict[str, float], -) -> np.ndarray: - intervals = context.base_intervals.copy() - if group_factors: - intervals = apply_group_multipliers( - context.y_pred, - intervals, - context.groups, - group_factors, - ) - if temporal_factors and context.temporal_segments is not None: - intervals = apply_group_multipliers( - context.y_pred, - intervals, - context.temporal_segments, - temporal_factors, - ) - return intervals - - -def _shrink_metrics(context: _ShrinkContext, intervals: np.ndarray) -> dict[str, float]: - temporal = temporal_stability_summary( - context.y_true, - intervals, - context.issue_dates, - target_coverage=context.target_coverage, - freq="M", - ) - return { - "coverage": empirical_interval_coverage(context.y_true, intervals), - "min_group_coverage": min_group_interval_coverage( - context.y_true, - intervals, - context.groups, - ), - "avg_width": average_interval_width(intervals), - "winkler_90": mean_winkler_score(context.y_true, intervals, alpha=context.alpha), - "max_monthly_gap": float(temporal["max_monthly_gap"]), - "stability_over_time": float(temporal["stability_over_time"]), - } - - -def _shrink_constraints_ok(context: _ShrinkContext, metrics: dict[str, float]) -> bool: - if float(metrics["coverage"]) < context.target_coverage: - return False - if float(metrics["min_group_coverage"]) < context.min_group_coverage_target: - return False - return not ( - context.max_monthly_gap_target is not None - and np.isfinite(context.max_monthly_gap_target) - and float(metrics["max_monthly_gap"]) > context.max_monthly_gap_target - ) - - -def _next_lower_factor(value: float, grid: tuple[float, ...]) -> float | None: - ordered = sorted({round(float(x), 6) for x in grid if float(x) <= float(value) + 1e-9}) - current = round(float(value), 6) - if current not in ordered: - ordered.append(current) - ordered = sorted(set(ordered)) - idx = ordered.index(current) - if idx == 0: - return None - return float(ordered[idx - 1]) - - -def _with_reduced_factor( - factors: dict[str, float], - key: str, - next_factor: float, -) -> dict[str, float]: - reduced = dict(factors) - if next_factor <= 1.0: - reduced.pop(key, None) - else: - reduced[key] = float(next_factor) - return reduced - - -def _evaluate_shrink_candidate( - context: _ShrinkContext, - *, - scope: str, - key: str, - factor: float, - group_factors: dict[str, float], - temporal_factors: dict[str, float], -) -> _ShrinkCandidate: - intervals = _apply_shrink_factors(context, group_factors, temporal_factors) - metrics = _shrink_metrics(context, intervals) - return _ShrinkCandidate( - scope=scope, - key=key, - factor=float(factor), - accepted=_shrink_constraints_ok(context, metrics), - intervals=intervals, - group_factors=dict(group_factors), - temporal_factors=dict(temporal_factors), - metrics=metrics, - ) - - -def _build_shrink_attempts( - context: _ShrinkContext, - *, - scope: str, - group_factors: dict[str, float], - temporal_factors: dict[str, float], - multiplier_grid: tuple[float, ...], -) -> list[_ShrinkCandidate]: - active_factors = group_factors if scope == "group" else temporal_factors - candidates: list[_ShrinkCandidate] = [] - for key, value in list(active_factors.items()): - next_factor = _next_lower_factor(value, multiplier_grid) - if next_factor is None: - continue - trial_group = ( - _with_reduced_factor(group_factors, key, next_factor) - if scope == "group" - else dict(group_factors) - ) - trial_temporal = ( - _with_reduced_factor(temporal_factors, key, next_factor) - if scope == "temporal" - else dict(temporal_factors) - ) - candidates.append( - _evaluate_shrink_candidate( - context, - scope=scope, - key=key, - factor=next_factor, - group_factors=trial_group, - temporal_factors=trial_temporal, - ) - ) - return candidates - - -def _shrink_candidate_is_better( - candidate: _ShrinkCandidate, - best_candidate: _ShrinkCandidate | None, -) -> bool: - if not candidate.accepted: - return False - if best_candidate is None: - return True - candidate_width = float(candidate.metrics["avg_width"]) - best_width = float(best_candidate.metrics["avg_width"]) - if candidate_width < best_width: - return True - return bool( - np.isclose(candidate_width, best_width) - and float(candidate.metrics["winkler_90"]) < float(best_candidate.metrics["winkler_90"]) - ) - - -def _shrink_report_row( - *, - stage: str, - factor_scope: str, - factor_key: str, - candidate_factor: float, - accepted: bool, - metrics: dict[str, float], -) -> dict[str, Any]: - return { - "stage": stage, - "factor_scope": factor_scope, - "factor_key": factor_key, - "candidate_factor": candidate_factor, - "accepted": accepted, - **metrics, - } - - -def shrink_group_multipliers( - *, - y_true: np.ndarray, - y_pred: np.ndarray, - base_intervals: np.ndarray, - groups: pd.Series | np.ndarray, - issue_dates: pd.Series | np.ndarray | None, - group_factors: dict[str, float] | None = None, - temporal_segments: pd.Series | np.ndarray | None = None, - temporal_factors: dict[str, float] | None = None, - target_coverage: float = 0.90, - min_group_coverage_target: float = 0.88, - max_monthly_gap_target: float | None = None, - alpha: float = 0.10, - group_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), -) -> tuple[np.ndarray, dict[str, float], dict[str, float], pd.DataFrame]: - """Greedily shrink learned widening factors while preserving constraints.""" - group_factors_cur = _active_widening_factors(group_factors) - temporal_factors_cur = _active_widening_factors(temporal_factors) - temporal_series = ( - pd.Series(temporal_segments).fillna("UNKNOWN").astype(str).reset_index(drop=True) - if temporal_segments is not None - else None - ) - context = _ShrinkContext( - y_true=np.asarray(y_true, dtype=float), - y_pred=np.asarray(y_pred, dtype=float), - base_intervals=np.asarray(base_intervals, dtype=float), - groups=pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True), - temporal_segments=temporal_series, - issue_dates=issue_dates, - target_coverage=float(target_coverage), - min_group_coverage_target=float(min_group_coverage_target), - max_monthly_gap_target=max_monthly_gap_target, - alpha=float(alpha), - ) - - current_intervals = _apply_shrink_factors( - context, - group_factors_cur, - temporal_factors_cur, - ) - current_metrics = _shrink_metrics(context, current_intervals) - report_rows: list[dict[str, Any]] = [ - _shrink_report_row( - stage="initial", - factor_scope="all", - factor_key="all", - candidate_factor=np.nan, - accepted=True, - metrics=current_metrics, - ) - ] - - if not _shrink_constraints_ok(context, current_metrics): - report_rows.append( - _shrink_report_row( - stage="initial_infeasible", - factor_scope="all", - factor_key="all", - candidate_factor=np.nan, - accepted=False, - metrics=current_metrics, - ) - ) - return current_intervals, group_factors_cur, temporal_factors_cur, pd.DataFrame(report_rows) - - while True: - attempts = _build_shrink_attempts( - context, - scope="group", - group_factors=group_factors_cur, - temporal_factors=temporal_factors_cur, - multiplier_grid=group_multiplier_grid, - ) - attempts.extend( - _build_shrink_attempts( - context, - scope="temporal", - group_factors=group_factors_cur, - temporal_factors=temporal_factors_cur, - multiplier_grid=temporal_multiplier_grid, - ) - ) - best_candidate: _ShrinkCandidate | None = None - for candidate in attempts: - if _shrink_candidate_is_better(candidate, best_candidate): - best_candidate = candidate - report_rows.append( - _shrink_report_row( - stage="attempt", - factor_scope=candidate.scope, - factor_key=candidate.key, - candidate_factor=float(candidate.factor), - accepted=bool(candidate.accepted), - metrics=candidate.metrics, - ) - ) - if best_candidate is None: - break - - current_intervals = np.asarray(best_candidate.intervals, dtype=float) - group_factors_cur = dict(best_candidate.group_factors) - temporal_factors_cur = dict(best_candidate.temporal_factors) - current_metrics = dict(best_candidate.metrics) - report_rows.append( - _shrink_report_row( - stage="accepted", - factor_scope=best_candidate.scope, - factor_key=best_candidate.key, - candidate_factor=float(best_candidate.factor), - accepted=True, - metrics=current_metrics, - ) - ) - - report_rows.append( - _shrink_report_row( - stage="final", - factor_scope="all", - factor_key="all", - candidate_factor=np.nan, - accepted=True, - metrics=current_metrics, - ) - ) - report = pd.DataFrame(report_rows) - return current_intervals, group_factors_cur, temporal_factors_cur, report diff --git a/src/models/maturity_safe_pd.py b/src/models/maturity_safe_pd.py new file mode 100644 index 0000000..3ff697b --- /dev/null +++ b/src/models/maturity_safe_pd.py @@ -0,0 +1,129 @@ +"""Fixed temporal PD model and probability-calibration helpers.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +import numpy as np +import pandas as pd +from catboost import CatBoostClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score + +from src.features.feature_engineering import run_feature_pipeline + +OUTCOME_COLUMNS = frozenset( + { + "default_flag", + "loan_status", + "outcome", + "payoff", + "realized_payoff", + "snapshot_default", + "snapshot_resolution", + "weighted_default", + "weighted_miscoverage", + "y_true", + } +) + + +def validate_model_feature_contract(model_config: Mapping[str, Any]) -> tuple[list[str], list[str]]: + """Return fixed feature lists after rejecting outcome-derived names.""" + numeric = [str(value) for value in model_config["numeric_features"]] + categorical = [str(value) for value in model_config["categorical_features"]] + features = numeric + categorical + duplicates = sorted({name for name in features if features.count(name) > 1}) + if duplicates: + raise ValueError(f"Model feature config contains duplicates: {duplicates}") + forbidden = sorted(OUTCOME_COLUMNS.intersection(features)) + if forbidden: + raise ValueError(f"Model feature config contains outcome fields: {forbidden}") + return numeric, categorical + + +def engineer_model_matrix( + frame: pd.DataFrame, + *, + numeric_features: Sequence[str], + categorical_features: Sequence[str], +) -> pd.DataFrame: + """Build target-free canonical features with CatBoost-safe dtypes.""" + source = frame.drop( + columns=["loan_status", "snapshot_default", "snapshot_resolution"], + errors="ignore", + ) + engineered = run_feature_pipeline(source) + output = pd.DataFrame(index=frame.index) + for feature in numeric_features: + values = engineered.get(feature, pd.Series(np.nan, index=frame.index)) + output[feature] = pd.to_numeric(values, errors="coerce").replace([np.inf, -np.inf], np.nan) + for feature in categorical_features: + values = engineered.get(feature, pd.Series("__MISSING__", index=frame.index)) + output[feature] = values.astype("string").fillna("__MISSING__").astype(str) + if OUTCOME_COLUMNS.intersection(output.columns): + raise AssertionError("Engineered model matrix contains an outcome field.") + return output + + +def require_binary_labels(frame: pd.DataFrame, *, block: str) -> np.ndarray: + """Return observed binary labels or fail instead of dropping rows.""" + labels = frame["snapshot_default"] + if bool(labels.isna().any()): + raise RuntimeError(f"{block} contains unresolved snapshot outcomes.") + values = labels.astype(int).to_numpy(dtype=int) + if set(np.unique(values)) != {0, 1}: + raise RuntimeError(f"{block} must contain both binary outcome classes.") + return values + + +def classification_metrics( + y_true: np.ndarray, + probabilities: np.ndarray, +) -> dict[str, float | int]: + """Compute fixed binary probability metrics.""" + clipped = np.clip(np.asarray(probabilities, dtype=float), 1e-12, 1.0 - 1e-12) + return { + "rows": int(len(y_true)), + "default_rate": float(np.mean(y_true)), + "roc_auc": float(roc_auc_score(y_true, clipped)), + "brier": float(brier_score_loss(y_true, clipped)), + "log_loss": float(log_loss(y_true, clipped, labels=[0, 1])), + } + + +def catboost_raw_margin(model: CatBoostClassifier, features: pd.DataFrame) -> np.ndarray: + """Return the one-dimensional CatBoost raw margin.""" + prediction = np.asarray( + model.predict(features, prediction_type="RawFormulaVal"), + dtype=float, + ).reshape(-1) + if len(prediction) != len(features) or not bool(np.isfinite(prediction).all()): + raise RuntimeError("CatBoost returned invalid raw margins.") + return prediction + + +def fit_platt_calibrator( + raw_margin: np.ndarray, + labels: np.ndarray, + config: Mapping[str, Any], +) -> LogisticRegression: + """Fit the predeclared Platt calibrator on CatBoost raw margins.""" + model = LogisticRegression(**dict(config["logistic_regression"])) + model.fit(np.asarray(raw_margin, dtype=float).reshape(-1, 1), labels) + return model + + +def apply_platt_calibrator( + calibrator: LogisticRegression, + raw_margin: np.ndarray, +) -> np.ndarray: + """Apply a fitted raw-margin Platt calibrator.""" + probabilities = np.asarray( + calibrator.predict_proba(np.asarray(raw_margin, dtype=float).reshape(-1, 1))[:, 1], + dtype=float, + ) + if not bool(np.isfinite(probabilities).all()): + raise RuntimeError("Probability calibrator returned non-finite values.") + return np.clip(probabilities, 0.0, 1.0) diff --git a/src/models/optuna_tuning.py b/src/models/optuna_tuning.py deleted file mode 100644 index 2c5a724..0000000 --- a/src/models/optuna_tuning.py +++ /dev/null @@ -1,1188 +0,0 @@ -"""Optuna-based CatBoost hyperparameter tuning for PD models.""" - -from __future__ import annotations - -import gc -from dataclasses import dataclass -from functools import partial -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from catboost import CatBoostClassifier, Pool -from loguru import logger -from sklearn.metrics import brier_score_loss, roc_auc_score - -from src.models.calibration import expected_calibration_error -from src.models.pd_model import CATEGORICAL_FEATURES, _catboost_base_params - -SEARCH_SPACE_VERSION = "cb_space_v2" -_JOURNAL_STORAGE_PREFIXES = ("journal+file:", "journalfile:", "journal:") -CategoricalChoice = None | bool | int | float | str - - -@dataclass(frozen=True) -class _SelectedModelFit: - model: CatBoostClassifier - selected_params: dict[str, Any] - resolved_params: dict[str, Any] - validation_auc: float - best_iteration: int - - -def _is_journal_storage_url(url: str) -> bool: - return url.lower().startswith(_JOURNAL_STORAGE_PREFIXES) - - -def _journal_path_from_storage_url(url: str) -> str: - for prefix in _JOURNAL_STORAGE_PREFIXES: - if url.lower().startswith(prefix): - value = url[len(prefix) :] - if value.startswith("///"): - return "/" + value[3:] - return value - return url - - -def _categorical_choices(raw: Any) -> list[CategoricalChoice]: - choices: list[CategoricalChoice] = [] - for value in list(raw): - if value is None or isinstance(value, bool | int | float | str): - choices.append(value) - else: - raise TypeError(f"Unsupported Optuna categorical choice {value!r}") - if not choices: - raise ValueError("Optuna categorical choices must not be empty") - return choices - - -def resolve_optuna_study_name( - study_name: str | None, - *, - search_space_version: str = SEARCH_SPACE_VERSION, -) -> str: - """Append a stable search-space version to persistent Optuna study names. - - This prevents historical studies with incompatible distributions from being - reused after search-space changes, which otherwise can fail mid-run with - dynamic distribution compatibility errors. - """ - base_name = str(study_name or "pd_catboost_optuna").strip() or "pd_catboost_optuna" - suffix = f"__{search_space_version.strip()}" - return base_name if base_name.endswith(suffix) else f"{base_name}{suffix}" - - -def _build_optuna_sampler_pruner( - optuna_module: Any, - *, - sampler: str, - pruner: str, - n_startup_trials: int, - multivariate_tpe: bool, - group_tpe: bool, - constant_liar: bool, - pruner_n_startup_trials: int, - pruner_n_warmup_steps: int, - constraints_func: Any | None = None, -) -> tuple[Any, Any]: - use_multivariate = bool(multivariate_tpe) - use_group_tpe = bool(group_tpe and use_multivariate) - - if sampler == "tpe": - sampler_obj = optuna_module.samplers.TPESampler( - seed=42, - n_startup_trials=max(10, int(n_startup_trials)), - multivariate=use_multivariate, - group=use_group_tpe, - constant_liar=bool(constant_liar), - constraints_func=constraints_func, - ) - elif sampler == "random": - sampler_obj = optuna_module.samplers.RandomSampler(seed=42) - else: - sampler_obj = optuna_module.samplers.TPESampler( - seed=42, - n_startup_trials=max(10, int(n_startup_trials)), - multivariate=use_multivariate, - group=use_group_tpe, - constant_liar=bool(constant_liar), - ) - - if pruner == "median": - pruner_obj = optuna_module.pruners.MedianPruner( - n_startup_trials=max(5, int(pruner_n_startup_trials)), - n_warmup_steps=max(1, int(pruner_n_warmup_steps)), - interval_steps=25, - ) - elif pruner == "none": - pruner_obj = optuna_module.pruners.NopPruner() - else: - pruner_obj = optuna_module.pruners.MedianPruner( - n_startup_trials=max(5, int(pruner_n_startup_trials)), - n_warmup_steps=max(1, int(pruner_n_warmup_steps)), - interval_steps=25, - ) - - return sampler_obj, pruner_obj - - -def _align_feature_vector( - raw: Any, - *, - feature_order: list[str], - default: float, -) -> list[float] | Any: - if not isinstance(raw, dict): - return raw - raw_by_feature = {str(feature): float(value) for feature, value in raw.items()} - return [float(raw_by_feature.get(feature, default)) for feature in feature_order] - - -def _normalize_feature_penalty_params( - params: dict[str, Any], - *, - feature_order: list[str], -) -> dict[str, Any]: - normalized = dict(params) - if "feature_weights" in normalized: - normalized["feature_weights"] = _align_feature_vector( - normalized["feature_weights"], - feature_order=feature_order, - default=1.0, - ) - if "first_feature_use_penalties" in normalized: - normalized["first_feature_use_penalties"] = _align_feature_vector( - normalized["first_feature_use_penalties"], - feature_order=feature_order, - default=0.0, - ) - return normalized - - -def _local_choice(trial: Any, name: str, spec: Any, default: Any) -> Any: - if spec is None: - return default - if isinstance(spec, dict): - if spec.get("choices") is not None: - return trial.suggest_categorical(name, _categorical_choices(spec["choices"])) - low = spec.get("low") - high = spec.get("high") - step = spec.get("step") - log = bool(spec.get("log", False)) - if low is None or high is None: - return default - if isinstance(low, int) and isinstance(high, int) and not log: - return trial.suggest_int(name, int(low), int(high), step=int(step or 1)) - return trial.suggest_float( - name, - float(low), - float(high), - step=None if log else (float(step) if step is not None else None), - log=log, - ) - if isinstance(spec, list): - return trial.suggest_categorical(name, _categorical_choices(spec)) - return spec - - -def _apply_local_feature_priors( - trial: Any, - params: dict[str, Any], - *, - local_refine_space: dict[str, Any], -) -> None: - feature_weights_cfg = dict(local_refine_space.get("feature_weights", {}) or {}) - if feature_weights_cfg: - weights: dict[str, float] = {} - for feature, spec in feature_weights_cfg.items(): - value = float(_local_choice(trial, f"feature_weight__{feature}", spec, 1.0)) - weights[str(feature)] = value - if any(abs(value - 1.0) > 1e-12 for value in weights.values()): - params["feature_weights"] = weights - penalties_cfg = dict(local_refine_space.get("first_feature_use_penalties", {}) or {}) - if penalties_cfg: - penalties: dict[str, float] = {} - for feature, spec in penalties_cfg.items(): - value = float(_local_choice(trial, f"first_use_penalty__{feature}", spec, 0.0)) - penalties[str(feature)] = value - if any(abs(value) > 1e-12 for value in penalties.values()): - params["first_feature_use_penalties"] = penalties - penalties_coeff_spec = local_refine_space.get("penalties_coefficient") - if penalties_coeff_spec is not None: - params["penalties_coefficient"] = float( - _local_choice(trial, "penalties_coefficient", penalties_coeff_spec, 1.0) - ) - - -def _materialize_study_params( - sampled_params: dict[str, Any], - *, - base: dict[str, Any], - has_monotone_constraints: bool, -) -> dict[str, Any]: - params = {**base} - feature_weights: dict[str, float] = {} - penalties: dict[str, float] = {} - - for key, value in dict(sampled_params or {}).items(): - key_str = str(key) - if key_str.startswith("feature_weight__"): - feature_name = key_str.split("__", 1)[1] - feature_weights[feature_name] = float(value) - continue - if key_str.startswith("first_use_penalty__"): - feature_name = key_str.split("__", 1)[1] - penalties[feature_name] = float(value) - continue - params[key_str] = value - - if feature_weights and any(abs(weight - 1.0) > 1e-12 for weight in feature_weights.values()): - params["feature_weights"] = feature_weights - else: - params.pop("feature_weights", None) - if penalties and any(abs(weight) > 1e-12 for weight in penalties.values()): - params["first_feature_use_penalties"] = penalties - else: - params.pop("first_feature_use_penalties", None) - - if str(params.get("bootstrap_type", "")).strip() == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) - - if str(params.get("grow_policy", "")).strip() == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) - - if str(params.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) - - return {key: value for key, value in params.items() if value is not None} - - -def _sanitize_enqueued_trial( - raw_params: dict[str, Any], - *, - base: dict[str, Any], - search_space_mode_resolved: str, - has_monotone_constraints: bool, -) -> dict[str, Any]: - """Keep only parameters that are actually sampled by the active Optuna space.""" - allowed = { - "bootstrap_type", - "grow_policy", - "learning_rate", - "l2_leaf_reg", - "min_data_in_leaf", - "random_strength", - "border_count", - "leaf_estimation_iterations", - "rsm", - "depth", - "max_leaves", - "subsample", - "bagging_temperature", - } - if search_space_mode_resolved == "local_refine": - allowed.add("iterations") - params: dict[str, Any] = {} - for key, value in dict(raw_params or {}).items(): - key_str = str(key) - if key_str in allowed or key_str.startswith(("feature_weight__", "first_use_penalty__")): - params[key_str] = value - - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) - if str(params.get("grow_policy", base.get("grow_policy", "SymmetricTree"))) == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", base.get("bootstrap_type", "MVS"))) == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) - if str(base.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) - return {key: value for key, value in params.items() if value is not None} - - -def _trial_params_match(left: dict[str, Any], right: dict[str, Any]) -> bool: - if set(left) != set(right): - return False - for key, left_value in left.items(): - right_value = right.get(key) - if right_value is None: - if left_value is not None: - return False - continue - try: - if abs(float(left_value) - float(right_value)) > 1e-12: - return False - except (TypeError, ValueError): - if str(left_value) != str(right_value): - return False - return True - - -def _enqueue_prior_trials( - study: Any, - *, - enqueue_trials: list[dict[str, Any]], - base: dict[str, Any], - search_space_mode_resolved: str, - has_monotone_constraints: bool, -) -> int: - enqueued = 0 - existing = [dict(trial.params) for trial in study.trials] - for raw_params in enqueue_trials: - params = _sanitize_enqueued_trial( - raw_params, - base=base, - search_space_mode_resolved=search_space_mode_resolved, - has_monotone_constraints=has_monotone_constraints, - ) - if not params: - continue - if any(_trial_params_match(params, trial_params) for trial_params in existing): - continue - try: - study.enqueue_trial(params, skip_if_exists=True) - except TypeError: - study.enqueue_trial(params) - existing.append(params) - enqueued += 1 - return enqueued - - -def _local_refine_params( - trial: Any, - *, - base: dict[str, Any], - local_refine_space: dict[str, Any], - feature_order: list[str], - is_gpu: bool, -) -> dict[str, Any]: - params = {**base} - fixed_params = dict(local_refine_space.get("fixed_params", {}) or {}) - params.update(fixed_params) - - params["iterations"] = int( - _local_choice( - trial, - "iterations", - local_refine_space.get("iterations"), - base.get("iterations", 3000), - ) - ) - params["learning_rate"] = float( - _local_choice( - trial, - "learning_rate", - local_refine_space.get("learning_rate"), - base.get("learning_rate", 0.03), - ) - ) - params["l2_leaf_reg"] = float( - _local_choice( - trial, - "l2_leaf_reg", - local_refine_space.get("l2_leaf_reg"), - base.get("l2_leaf_reg", 3.0), - ) - ) - params["min_data_in_leaf"] = int( - _local_choice( - trial, - "min_data_in_leaf", - local_refine_space.get("min_data_in_leaf"), - base.get("min_data_in_leaf", 64), - ) - ) - params["random_strength"] = float( - _local_choice( - trial, - "random_strength", - local_refine_space.get("random_strength"), - base.get("random_strength", 1e-6), - ) - ) - params["border_count"] = int( - _local_choice( - trial, - "border_count", - local_refine_space.get("border_count"), - base.get("border_count", 128), - ) - ) - params["leaf_estimation_iterations"] = int( - _local_choice( - trial, - "leaf_estimation_iterations", - local_refine_space.get("leaf_estimation_iterations"), - base.get("leaf_estimation_iterations", 3), - ) - ) - params["bootstrap_type"] = _local_choice( - trial, - "bootstrap_type", - local_refine_space.get("bootstrap_type"), - base.get("bootstrap_type", "MVS"), - ) - params["grow_policy"] = _local_choice( - trial, - "grow_policy", - local_refine_space.get("grow_policy"), - base.get("grow_policy", "SymmetricTree"), - ) - - if is_gpu: - params.pop("rsm", None) - else: - params["rsm"] = float( - _local_choice(trial, "rsm", local_refine_space.get("rsm"), base.get("rsm", 1.0)) - ) - if str(params.get("grow_policy", "SymmetricTree")) == "Lossguide": - params.pop("depth", None) - params["max_leaves"] = int( - _local_choice(trial, "max_leaves", local_refine_space.get("max_leaves"), 32) - ) - else: - params["depth"] = int( - _local_choice(trial, "depth", local_refine_space.get("depth"), base.get("depth", 8)) - ) - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", "MVS")) == "Bayesian": - params.pop("subsample", None) - bagging_spec = local_refine_space.get("bagging_temperature", {"low": 0.0, "high": 10.0}) - params["bagging_temperature"] = float( - _local_choice( - trial, - "bagging_temperature", - bagging_spec, - base.get("bagging_temperature", 1.0), - ) - ) - else: - params.pop("bagging_temperature", None) - params["subsample"] = float( - _local_choice( - trial, - "subsample", - local_refine_space.get("subsample"), - base.get("subsample", 0.8), - ) - ) - _apply_local_feature_priors(trial, params, local_refine_space=local_refine_space) - return _normalize_feature_penalty_params(params, feature_order=feature_order) - - -def _global_search_params( - trial: Any, - *, - base: dict[str, Any], - has_monotone_constraints: bool, -) -> dict[str, Any]: - bootstrap_type = trial.suggest_categorical("bootstrap_type", ["Bayesian", "Bernoulli", "MVS"]) - grow_policy_choices = ( - ["SymmetricTree"] - if has_monotone_constraints - else ["SymmetricTree", "Depthwise", "Lossguide"] - ) - grow_policy = trial.suggest_categorical("grow_policy", grow_policy_choices) - params = { - **base, - "learning_rate": trial.suggest_float("learning_rate", 0.005, 0.20, log=True), - "l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 0.5, 100.0, log=True), - "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 20, 500), - "random_strength": trial.suggest_float("random_strength", 1e-9, 10.0, log=True), - "border_count": trial.suggest_int("border_count", 64, 254), - "bootstrap_type": bootstrap_type, - "grow_policy": grow_policy, - "leaf_estimation_iterations": trial.suggest_int("leaf_estimation_iterations", 1, 10), - "random_seed": int(base.get("random_seed", 42)), - } - if str(base.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) - else: - params["rsm"] = trial.suggest_float("rsm", 0.5, 1.0) - if grow_policy == "Lossguide": - params["max_leaves"] = trial.suggest_int("max_leaves", 16, 64) - params.pop("depth", None) - else: - params["depth"] = trial.suggest_int("depth", 4, 10) - if bootstrap_type == "Bayesian": - params.pop("subsample", None) - params["bagging_temperature"] = trial.suggest_float("bagging_temperature", 0.0, 10.0) - else: - params.pop("bagging_temperature", None) - params["subsample"] = trial.suggest_float("subsample", 0.5, 0.95) - return params - - -def _constraints_violations( - frozen_trial: Any, - *, - incumbent_metrics: dict[str, float], - constraints_policy: dict[str, Any], -) -> list[float]: - attrs = frozen_trial.user_attrs - violations: list[float] = [] - max_brier_delta = constraints_policy.get("max_brier_delta") - max_ece_delta = constraints_policy.get("max_ece_delta") - min_auc_delta = constraints_policy.get("min_auc_delta") - if max_brier_delta is not None: - ceiling = float(incumbent_metrics.get("validation_brier", 0.0)) + float(max_brier_delta) - violations.append(float(attrs.get("validation_brier", float("inf"))) - ceiling) - if max_ece_delta is not None: - ceiling = float(incumbent_metrics.get("validation_ece", 0.0)) + float(max_ece_delta) - violations.append(float(attrs.get("validation_ece", float("inf"))) - ceiling) - if min_auc_delta is not None: - floor = float(incumbent_metrics.get("validation_auc", 0.0)) + float(min_auc_delta) - violations.append(floor - float(attrs.get("validation_auc", float("-inf")))) - return violations - - -def _incumbent_validation_metrics( - *, - base: dict[str, Any], - train_pool: Pool, - val_pool: Pool, - X_val: pd.DataFrame, - y_val: pd.Series, -) -> dict[str, float]: - incumbent_model = CatBoostClassifier(**base) - incumbent_model.fit(train_pool, eval_set=val_pool, use_best_model=True) - incumbent_y_val_prob = incumbent_model.predict_proba(X_val)[:, 1] - y_val_array = np.asarray(y_val, dtype=int) - return { - "validation_auc": float(roc_auc_score(y_val, incumbent_y_val_prob)), - "validation_brier": float(brier_score_loss(y_val, incumbent_y_val_prob)), - "validation_ece": float(expected_calibration_error(y_val_array, incumbent_y_val_prob)), - } - - -def _catboost_pruning_callbacks( - trial: Any, *, use_pruning_callback: bool -) -> tuple[Any | None, list[Any]]: - if not use_pruning_callback: - return None, [] - try: - from optuna_integration.catboost import CatBoostPruningCallback - - pruning_callback = CatBoostPruningCallback(trial, "AUC") - return pruning_callback, [pruning_callback] - except Exception as exc: # pragma: no cover - optional integration path - if trial.number == 0: - logger.warning( - "CatBoost pruning callback unavailable; disabling pruning callback: {}", exc - ) - return None, [] - - -def _catboost_objective( - trial: Any, - *, - base: dict[str, Any], - search_space_mode_resolved: str, - local_refine_space: dict[str, Any], - feature_order: list[str], - has_monotone_constraints: bool, - train_pool: Pool, - val_pool: Pool, - X_val: pd.DataFrame, - y_val: pd.Series, - use_pruning_callback: bool, -) -> float: - is_gpu = str(base.get("task_type", "")).strip().upper() == "GPU" - if search_space_mode_resolved == "local_refine": - params = _local_refine_params( - trial, - base=base, - local_refine_space=local_refine_space, - feature_order=feature_order, - is_gpu=is_gpu, - ) - else: - params = _global_search_params( - trial, - base=base, - has_monotone_constraints=has_monotone_constraints, - ) - - params = _normalize_feature_penalty_params(params, feature_order=feature_order) - model = CatBoostClassifier(**params) - pruning_callback, callbacks = _catboost_pruning_callbacks( - trial, - use_pruning_callback=use_pruning_callback, - ) - model.fit( - train_pool, - eval_set=val_pool, - use_best_model=True, - callbacks=callbacks or None, - ) - - if pruning_callback is not None: - pruning_callback.check_pruned() - - val_auc = model.get_best_score().get("validation", {}).get("AUC") - y_val_prob = model.predict_proba(X_val)[:, 1] - if val_auc is None: - val_auc = roc_auc_score(y_val, y_val_prob) - y_val_array = np.asarray(y_val, dtype=int) - trial.set_user_attr("best_iteration", int(model.get_best_iteration())) - trial.set_user_attr("validation_auc", float(val_auc)) - trial.set_user_attr("validation_brier", float(brier_score_loss(y_val_array, y_val_prob))) - trial.set_user_attr( - "validation_ece", - float(expected_calibration_error(y_val_array, y_val_prob)), - ) - return float(val_auc) - - -def _create_optuna_study( - optuna_module: Any, - *, - sampler_obj: Any, - pruner_obj: Any, - study_storage: str | None, - study_name: str | None, - load_if_exists: bool, - search_space_version: str, - storage_heartbeat_interval: int, - storage_grace_period: int, - sqlite_timeout_seconds: int, - retry_failed_trials: int, -) -> tuple[Any, Any | None]: - create_study_kwargs: dict[str, Any] = { - "direction": "maximize", - "sampler": sampler_obj, - "pruner": pruner_obj, - } - retry_callback = None - if not study_storage: - return optuna_module.create_study(**create_study_kwargs), retry_callback - - storage_obj: Any = study_storage - hb_interval = max(0, int(storage_heartbeat_interval)) - hb_grace = max(0, int(storage_grace_period)) - storage_text = str(study_storage) - if _is_journal_storage_url(storage_text): - journal_path = _journal_path_from_storage_url(storage_text) - Path(journal_path).parent.mkdir(parents=True, exist_ok=True) - from src.utils.optuna_storage import _make_journal_storage - - storage_obj = _make_journal_storage( - Path(journal_path), - study_name=resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, - ), - ) - else: - engine_kwargs = ( - {"connect_args": {"timeout": max(1, int(sqlite_timeout_seconds))}} - if storage_text.startswith(("sqlite:///", "sqlite+pysqlite:///")) - else None - ) - if hb_interval > 0 or hb_grace > 0: - try: - heartbeat_cb = None - failed_cb = None - if int(retry_failed_trials) > 0: - retry_factory = getattr( - optuna_module.storages, - "RetryHeartbeatStaleTrialCallback", - None, - ) - if retry_factory is not None: - heartbeat_cb = retry_factory(max_retry=int(retry_failed_trials)) - retry_callback = heartbeat_cb - else: # pragma: no cover - compatibility with older Optuna. - failed_cb = optuna_module.storages.RetryFailedTrialCallback( - max_retry=int(retry_failed_trials) - ) - retry_callback = failed_cb - storage_kwargs: dict[str, Any] = { - "url": storage_text, - "engine_kwargs": engine_kwargs, - "heartbeat_interval": hb_interval or None, - "grace_period": hb_grace or None, - } - if heartbeat_cb is not None: - storage_kwargs["heartbeat_stale_trial_callback"] = heartbeat_cb - elif failed_cb is not None: - storage_kwargs["failed_trial_callback"] = failed_cb - storage_obj = optuna_module.storages.RDBStorage(**storage_kwargs) - except Exception as exc: - logger.warning( - "Optuna RDBStorage heartbeat/retry setup failed; falling back to storage " - "URL. reason={}", - exc, - ) - - create_study_kwargs["storage"] = storage_obj - create_study_kwargs["study_name"] = resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, - ) - create_study_kwargs["load_if_exists"] = bool(load_if_exists) - return optuna_module.create_study(**create_study_kwargs), retry_callback - - -def _local_refine_base_trial_params(base: dict[str, Any]) -> dict[str, Any]: - return { - key: value - for key, value in { - "iterations": int(base.get("iterations", 3000)), - "learning_rate": float(base.get("learning_rate", 0.03)), - "l2_leaf_reg": float(base.get("l2_leaf_reg", 3.0)), - "min_data_in_leaf": int(base.get("min_data_in_leaf", 64)), - "random_strength": float(base.get("random_strength", 1e-6)), - "border_count": int(base.get("border_count", 128)), - "leaf_estimation_iterations": int(base.get("leaf_estimation_iterations", 3)), - "bootstrap_type": str(base.get("bootstrap_type", "MVS")), - "grow_policy": str(base.get("grow_policy", "SymmetricTree")), - "rsm": None - if str(base.get("task_type", "")).strip().upper() == "GPU" - else float(base.get("rsm", 1.0)), - "depth": None - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else int(base.get("depth", 8)), - "max_leaves": int(base.get("max_leaves", 32)) - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else None, - "subsample": None - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else float(base.get("subsample", 0.8)), - "bagging_temperature": float(base.get("bagging_temperature", 1.0)) - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else None, - }.items() - if value is not None - } - - -def _enqueue_local_refine_base_trial( - study: Any, - *, - base: dict[str, Any], - local_refine_space: dict[str, Any], - search_space_mode_resolved: str, -) -> None: - if search_space_mode_resolved != "local_refine" or not bool( - local_refine_space.get("enqueue_base_trial", True) - ): - return - try: - study.enqueue_trial(_local_refine_base_trial_params(base)) - except Exception as exc: - logger.warning("Optuna enqueue_trial for local_refine skipped: {}", exc) - - -def _complete_trial_value(trial: Any) -> float: - value = trial.value - if value is None: - raise ValueError("COMPLETE Optuna trial has no objective value.") - return float(value) - - -def _complete_trials(study: Any, optuna_module: Any) -> list[Any]: - return [ - trial - for trial in study.trials - if trial.state == optuna_module.trial.TrialState.COMPLETE and trial.value is not None - ] - - -def _selected_optuna_trial(study: Any, optuna_module: Any) -> tuple[Any, float, bool]: - complete_trials = _complete_trials(study, optuna_module) - if not complete_trials: - raise ValueError("Optuna study has no COMPLETE trials available for model selection.") - try: - selected_trial = study.best_trial - constrained_best_trial = True - except ValueError as exc: - selected_trial = max(complete_trials, key=_complete_trial_value) - constrained_best_trial = False - logger.warning( - "Optuna study has no feasible trial under constraints; using best COMPLETE trial " - "by validation AUC as fallback. reason={}", - exc, - ) - return selected_trial, _complete_trial_value(selected_trial), constrained_best_trial - - -def _run_or_reuse_study( - study: Any, - *, - objective: Any, - n_trials: int, - timeout_minutes: int, - gc_after_trial: bool, - n_jobs: int, -) -> None: - timeout = None if timeout_minutes <= 0 else int(timeout_minutes * 60) - requested_trials = int(n_trials) - if requested_trials > 0: - study.optimize( - objective, - n_trials=requested_trials, - timeout=timeout, - show_progress_bar=False, - gc_after_trial=bool(gc_after_trial), - n_jobs=max(1, int(n_jobs)), - ) - else: - complete_trials = [ - trial - for trial in study.trials - if trial.state.name == "COMPLETE" and trial.value is not None - ] - if not complete_trials: - raise ValueError( - "n_trials=0 requested, but the Optuna study has no COMPLETE trials to reuse." - ) - logger.info( - "Optuna reuse mode enabled (n_trials=0): skipping optimization and reusing " - "{} existing trials from study '{}'.", - len(study.trials), - study.study_name, - ) - gc.collect() - - -def _fit_selected_catboost_model( - *, - selected_trial: Any, - base: dict[str, Any], - has_monotone_constraints: bool, - feature_order: list[str], - train_pool: Pool, - val_pool: Pool, - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - cat_features: list[str], - sample_weight: np.ndarray | None, - eval_sample_weight: np.ndarray | None, - refit_full_train: bool, -) -> _SelectedModelFit: - selected_params = dict(selected_trial.params) - best_params = _materialize_study_params( - selected_params, - base=base, - has_monotone_constraints=has_monotone_constraints, - ) - best_params["verbose"] = 100 - catboost_best_params = _normalize_feature_penalty_params( - best_params, - feature_order=feature_order, - ) - selection_model = CatBoostClassifier(**catboost_best_params) - selection_model.fit(train_pool, eval_set=val_pool, use_best_model=True) - y_val_prob = selection_model.predict_proba(X_val)[:, 1] - val_auc = float(roc_auc_score(y_val, y_val_prob)) - best_iteration = int(selection_model.get_best_iteration()) - - if not refit_full_train: - return _SelectedModelFit( - model=selection_model, - selected_params=selected_params, - resolved_params=best_params, - validation_auc=val_auc, - best_iteration=best_iteration, - ) - - full_X = pd.concat([X_train, X_val], axis=0).reset_index(drop=True) - full_y = pd.concat([y_train, y_val], axis=0).reset_index(drop=True) - full_weight = None - if sample_weight is not None and eval_sample_weight is not None: - full_weight = np.concatenate( - [ - np.asarray(sample_weight, dtype=float), - np.asarray(eval_sample_weight, dtype=float), - ] - ) - full_pool = Pool(full_X, full_y, cat_features=cat_features, weight=full_weight) - refit_params = { - key: value for key, value in catboost_best_params.items() if key != "early_stopping_rounds" - } - if best_iteration > 0: - refit_params["iterations"] = best_iteration + 1 - best_model = CatBoostClassifier(**refit_params) - best_model.fit(full_pool) - return _SelectedModelFit( - model=best_model, - selected_params=selected_params, - resolved_params=best_params, - validation_auc=val_auc, - best_iteration=best_iteration, - ) - - -def train_catboost_tuned_optuna( - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - X_test: pd.DataFrame | None = None, - y_test: pd.Series | None = None, - *, - cat_features: list[str] | None = None, - base_params: dict[str, Any] | None = None, - n_trials: int = 100, - sampler: str = "tpe", - pruner: str = "median", - timeout_minutes: int = 0, - n_startup_trials: int = 40, - multivariate_tpe: bool = True, - group_tpe: bool = True, - warn_independent_sampling: bool = True, - constant_liar: bool = False, - pruner_n_startup_trials: int = 20, - pruner_n_warmup_steps: int = 50, - use_pruning_callback: bool = True, - study_storage: str | None = None, - study_name: str | None = None, - load_if_exists: bool = True, - refit_full_train: bool = True, - gc_after_trial: bool = True, - storage_heartbeat_interval: int = 0, - storage_grace_period: int = 0, - sqlite_timeout_seconds: int = 60, - retry_failed_trials: int = 0, - n_jobs: int = 1, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, - search_space_mode: str = "global", - local_refine_space: dict[str, Any] | None = None, - constraints_policy: dict[str, Any] | None = None, - search_space_version: str = SEARCH_SPACE_VERSION, - enqueue_trials: list[dict[str, Any]] | None = None, -) -> tuple[CatBoostClassifier, dict[str, Any]]: - """Tune CatBoost with Optuna and return best fitted model and metadata.""" - import optuna - - if cat_features is None: - cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] - - base = _catboost_base_params(base_params) - base["verbose"] = 0 - has_monotone_constraints = bool(str(base.get("monotone_constraints", "")).strip()) - if has_monotone_constraints: - base["grow_policy"] = "SymmetricTree" - search_space_mode_resolved = str(search_space_mode or "global").strip().lower() or "global" - local_refine_space = dict(local_refine_space or {}) - if has_monotone_constraints: - local_refine_space["grow_policy"] = ["SymmetricTree"] - constraints_policy = dict(constraints_policy or {}) - enqueue_trials = list(enqueue_trials or []) - feature_order = [str(col) for col in X_train.columns] - - train_pool = Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight) - val_pool = Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight) - incumbent_metrics: dict[str, float] = {} - if constraints_policy: - incumbent_metrics.update( - _incumbent_validation_metrics( - base=base, - train_pool=train_pool, - val_pool=val_pool, - X_val=X_val, - y_val=y_val, - ) - ) - - use_multivariate = bool(multivariate_tpe) - use_group_tpe = bool(group_tpe and use_multivariate) - constraints_func = ( - partial( - _constraints_violations, - incumbent_metrics=incumbent_metrics, - constraints_policy=constraints_policy, - ) - if constraints_policy - else None - ) - - sampler_obj, pruner_obj = _build_optuna_sampler_pruner( - optuna, - sampler=sampler, - pruner=pruner, - n_startup_trials=n_startup_trials, - multivariate_tpe=use_multivariate, - group_tpe=use_group_tpe, - constant_liar=constant_liar, - pruner_n_startup_trials=pruner_n_startup_trials, - pruner_n_warmup_steps=pruner_n_warmup_steps, - constraints_func=constraints_func, - ) - - objective = partial( - _catboost_objective, - base=base, - search_space_mode_resolved=search_space_mode_resolved, - local_refine_space=local_refine_space, - feature_order=feature_order, - has_monotone_constraints=has_monotone_constraints, - train_pool=train_pool, - val_pool=val_pool, - X_val=X_val, - y_val=y_val, - use_pruning_callback=use_pruning_callback, - ) - - study, retry_callback = _create_optuna_study( - optuna, - sampler_obj=sampler_obj, - pruner_obj=pruner_obj, - study_storage=study_storage, - study_name=study_name, - load_if_exists=load_if_exists, - search_space_version=search_space_version, - storage_heartbeat_interval=storage_heartbeat_interval, - storage_grace_period=storage_grace_period, - sqlite_timeout_seconds=sqlite_timeout_seconds, - retry_failed_trials=retry_failed_trials, - ) - n_enqueued_prior_trials = _enqueue_prior_trials( - study, - enqueue_trials=enqueue_trials, - base=base, - search_space_mode_resolved=search_space_mode_resolved, - has_monotone_constraints=has_monotone_constraints, - ) - _enqueue_local_refine_base_trial( - study, - base=base, - local_refine_space=local_refine_space, - search_space_mode_resolved=search_space_mode_resolved, - ) - if retry_callback is not None and hasattr(optuna.storages, "fail_stale_trials"): - try: - optuna.storages.fail_stale_trials(study) - except Exception as exc: - logger.warning("Optuna stale-trial recovery skipped: {}", exc) - _run_or_reuse_study( - study, - objective=objective, - n_trials=n_trials, - timeout_minutes=timeout_minutes, - gc_after_trial=gc_after_trial, - n_jobs=n_jobs, - ) - - selected_trial, selected_value, constrained_best_trial = _selected_optuna_trial(study, optuna) - selected_fit = _fit_selected_catboost_model( - selected_trial=selected_trial, - base=base, - has_monotone_constraints=has_monotone_constraints, - feature_order=feature_order, - train_pool=train_pool, - val_pool=val_pool, - X_train=X_train, - y_train=y_train, - X_val=X_val, - y_val=y_val, - cat_features=cat_features, - sample_weight=sample_weight, - eval_sample_weight=eval_sample_weight, - refit_full_train=refit_full_train, - ) - - metrics: dict[str, Any] = { - "validation_auc": float(selected_fit.validation_auc), - "best_iteration": selected_fit.best_iteration, - "best_params": selected_fit.selected_params, - "best_params_resolved": selected_fit.resolved_params, - "hpo_trials_executed": len(study.trials), - "hpo_best_validation_auc": selected_value, - "hpo_selected_trial_number": int(selected_trial.number), - "hpo_selected_trial_constrained_feasible": bool(constrained_best_trial), - "study_name_resolved": study.study_name, - "refit_full_train": bool(refit_full_train), - "model_type": "catboost_tuned", - "search_space_mode": search_space_mode_resolved, - "constraint_baseline_metrics": incumbent_metrics, - "enqueued_prior_trials": n_enqueued_prior_trials, - } - if X_test is not None and y_test is not None: - y_test_prob = selected_fit.model.predict_proba(X_test)[:, 1] - metrics["auc_roc"] = float(roc_auc_score(y_test, y_test_prob)) - - logger.info( - "CatBoost tuned — val_AUC: " - f"{selected_fit.validation_auc:.4f}, best_trial_val_AUC: {selected_value:.4f}, " - f"trials={len(study.trials)}, multivariate_tpe={use_multivariate}, group_tpe={use_group_tpe}" - ) - return selected_fit.model, metrics - - -def export_hpo_visualizations( - study_storage: str, - study_name: str | None = None, - output_dir: str = "reports/figures/hpo", -) -> list[str]: - """Load an Optuna study and export visualization plots as HTML + PNG. - - Args: - study_storage: SQLite URL (e.g., ``sqlite:///models/optuna_pd_catboost.db``). - study_name: Study name (auto-resolved with search-space version if None). - output_dir: Directory to write plots to. - - Returns: - List of paths to generated plot files. - """ - from pathlib import Path as _Path - - import optuna - - out = _Path(output_dir) - out.mkdir(parents=True, exist_ok=True) - resolved_name = resolve_optuna_study_name(study_name) - study = optuna.load_study(study_name=resolved_name, storage=study_storage) - complete = [t for t in study.trials if t.state.name == "COMPLETE"] - if len(complete) < 2: - logger.warning("Study has <2 complete trials — skipping visualization export.") - return [] - - try: - from optuna.visualization import ( - plot_optimization_history, - plot_parallel_coordinate, - plot_param_importances, - plot_slice, - ) - except ImportError: - logger.warning("optuna.visualization requires plotly — install plotly to export HPO plots.") - return [] - - plots = { - "optimization_history": plot_optimization_history(study), - "param_importances": plot_param_importances(study), - "parallel_coordinate": plot_parallel_coordinate(study), - "slice": plot_slice(study), - } - - saved: list[str] = [] - for name, fig in plots.items(): - html_path = out / f"{name}.html" - fig.write_html(str(html_path)) - saved.append(str(html_path)) - try: - png_path = out / f"{name}.png" - fig.write_image(str(png_path), width=1200, height=700, scale=2) - saved.append(str(png_path)) - except Exception: - logger.debug(f"PNG export skipped for {name} (kaleido not installed)") - - logger.info(f"Exported {len(saved)} HPO visualization files to {out}") - return saved diff --git a/src/models/pd_contract.py b/src/models/pd_contract.py deleted file mode 100644 index ae90d96..0000000 --- a/src/models/pd_contract.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Canonical PD model contract helpers. - -Defines a single source of truth for: -- PD model artifact path -- calibrator artifact path -- feature contract (names, categorical subset) -""" - -from __future__ import annotations - -import json -import os -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, cast - -import pandas as pd -from catboost import CatBoostClassifier - -CANONICAL_MODEL_PATH = Path("models/pd_canonical.cbm") -CANONICAL_CALIBRATOR_PATH = Path("models/pd_canonical_calibrator.pkl") -CONTRACT_PATH = Path("models/pd_model_contract.json") - - -def _upstream_search_pd_candidates() -> list[Path]: - run_tag = str(os.environ.get("UPSTREAM_CANONICAL_RUN_TAG", "") or "").strip() - if not run_tag: - return [] - base_dir = Path("models/search_pd") / run_tag - return [ - base_dir / "pd_candidate_model.cbm", - base_dir / "pd_local_hpo_tuned.cbm", - base_dir / "pd_candidate_calibrator.pkl", - base_dir / "pd_local_hpo_calibrator.pkl", - ] - - -def resolve_model_path() -> Path: - """Resolve canonical PD model path with fallback candidates.""" - upstream_candidates = _upstream_search_pd_candidates() - candidates = [ - upstream_candidates[0] if len(upstream_candidates) > 0 else None, - upstream_candidates[1] if len(upstream_candidates) > 1 else None, - CANONICAL_MODEL_PATH, - Path("models/pd_catboost.cbm"), - Path("models/pd_catboost_tuned.cbm"), - ] - existing: list[Path] = [candidate for candidate in candidates if candidate is not None] - path = next((p for p in existing if p.exists()), None) - if path is None: - raise FileNotFoundError("No PD model artifact found in models/.") - return path - - -def resolve_calibrator_path() -> Path | None: - """Resolve canonical calibrator path with fallback candidates.""" - upstream_candidates = _upstream_search_pd_candidates() - candidates = [ - upstream_candidates[2] if len(upstream_candidates) > 2 else None, - upstream_candidates[3] if len(upstream_candidates) > 3 else None, - CANONICAL_CALIBRATOR_PATH, - Path("models/pd_calibrator.pkl"), - ] - existing: list[Path] = [candidate for candidate in candidates if candidate is not None] - return next((p for p in existing if p.exists()), None) - - -def load_contract(path: Path = CONTRACT_PATH) -> dict[str, Any] | None: - """Load persisted contract if present.""" - if not path.exists(): - return None - with open(path, encoding="utf-8") as f: - return cast(dict[str, Any], json.load(f)) - - -def build_contract_payload( - model_path: Path, - calibrator_path: Path | None, - feature_names: list[str], - categorical_features: list[str], - split_shapes: dict[str, tuple[int, int]] | None = None, - split_missing_features: dict[str, list[str]] | None = None, -) -> dict[str, Any]: - """Build serializable contract payload.""" - payload: dict[str, Any] = { - "created_at_utc": datetime.now(UTC).isoformat(), - "model_path": model_path.as_posix(), - "calibrator_path": calibrator_path.as_posix() if calibrator_path else None, - "feature_names": feature_names, - "categorical_features": categorical_features, - "n_features": len(feature_names), - } - if split_shapes is not None: - payload["split_shapes"] = {k: [int(v[0]), int(v[1])] for k, v in split_shapes.items()} - if split_missing_features is not None: - payload["split_missing_features"] = split_missing_features - return payload - - -def infer_model_feature_contract(model: CatBoostClassifier) -> tuple[list[str], list[str]]: - """Extract model feature names and categorical feature names.""" - feature_names = list(getattr(model, "feature_names_", []) or []) - cat_idx = set(model.get_cat_feature_indices()) - categorical = [f for i, f in enumerate(feature_names) if i in cat_idx] - return feature_names, categorical - - -def validate_features_in_splits( - feature_names: list[str], - splits: dict[str, pd.DataFrame], -) -> tuple[dict[str, tuple[int, int]], dict[str, list[str]]]: - """Validate that required features exist in each split.""" - shapes: dict[str, tuple[int, int]] = {} - missing: dict[str, list[str]] = {} - for split_name, df in splits.items(): - shapes[split_name] = tuple(df.shape) - missing[split_name] = [f for f in feature_names if f not in df.columns] - return shapes, missing - - -def save_contract(payload: dict[str, Any], path: Path = CONTRACT_PATH) -> None: - """Persist contract as JSON.""" - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: - json.dump(payload, f, indent=2) diff --git a/src/models/pd_model.py b/src/models/pd_model.py deleted file mode 100644 index 0afaebf..0000000 --- a/src/models/pd_model.py +++ /dev/null @@ -1,306 +0,0 @@ -"""Probability of Default (PD) modeling utilities.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from catboost import CatBoostClassifier, Pool -from loguru import logger -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import roc_auc_score - -from src.features.feature_config_io import load_feature_config as load_feature_config_artifact -from src.features.feature_engineering import ( - CATBOOST_FEATURES as CANONICAL_CATBOOST_FEATURES, - CATEGORICAL_FEATURES as CANONICAL_CATEGORICAL_FEATURES, - LOGREG_FEATURES as CANONICAL_LOGREG_FEATURES, - NUMERIC_FEATURES as CANONICAL_NUMERIC_FEATURES, - WOE_FEATURES as CANONICAL_WOE_FEATURES, -) - -# ── Backward-compatible default feature configuration ── -NUMERIC_FEATURES = list(CANONICAL_NUMERIC_FEATURES) -WOE_FEATURES = list(CANONICAL_WOE_FEATURES) -CATEGORICAL_FEATURES = list(CANONICAL_CATEGORICAL_FEATURES) -ALL_FEATURES = list(CANONICAL_CATBOOST_FEATURES) + [ - f for f in CANONICAL_LOGREG_FEATURES if f not in CANONICAL_CATBOOST_FEATURES -] -TARGET = "default_flag" - - -def get_available_features(df: pd.DataFrame) -> list[str]: - """Return legacy fallback features that exist in the DataFrame.""" - return [f for f in ALL_FEATURES if f in df.columns] - - -def load_feature_config(feature_config_path: str | Path) -> dict[str, Any]: - """Load persisted feature config artifact if available. - - YAML is the live pipeline contract. The legacy pickle can still be forced - through ``src.features.feature_config_io`` for old artifact audits. - """ - path = Path(feature_config_path) - yaml_path = path if path.suffix.lower() in {".yml", ".yaml"} else path.with_suffix(".yml") - if not path.exists() and not yaml_path.exists(): - return {} - try: - if path.suffix.lower() in {".yml", ".yaml"}: - return load_feature_config_artifact( - pickle_path=path.with_suffix(".pkl"), - yaml_path=path, - prefer="yaml", - ) - return load_feature_config_artifact( - pickle_path=path, - yaml_path=yaml_path, - prefer="yaml", - ) - except (FileNotFoundError, TypeError) as exc: - logger.warning(f"Unable to load feature_config from {path}: {exc}") - return {} - - -def resolve_feature_sets( - df: pd.DataFrame, - feature_source: str = "auto", - feature_config_path: str | Path = "data/processed/feature_config.yml", -) -> dict[str, Any]: - """Resolve feature sets from feature_config first, with legacy fallback.""" - cfg = load_feature_config(feature_config_path) - use_cfg = feature_source == "feature_config" or (feature_source == "auto" and bool(cfg)) - - if use_cfg and cfg: - catboost = [c for c in cfg.get("CATBOOST_FEATURES", []) if c in df.columns] - categorical = [c for c in cfg.get("CATEGORICAL_FEATURES", []) if c in catboost] - logreg = [c for c in cfg.get("LOGREG_FEATURES", []) if c in df.columns] - if not logreg: - logreg = [c for c in catboost if c not in categorical] - return { - "catboost_features": catboost, - "logreg_features": logreg, - "categorical_features": categorical, - "feature_source": "feature_config", - } - - fallback = get_available_features(df) - categorical = [c for c in CATEGORICAL_FEATURES if c in fallback] - return { - "catboost_features": fallback, - "logreg_features": [c for c in fallback if c not in categorical], - "categorical_features": categorical, - "feature_source": "legacy_defaults", - } - - -def temporal_train_val_split( - train_df: pd.DataFrame, - val_fraction: float = 0.15, - date_col: str = "issue_d", -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Split train into fit/validation keeping temporal order (tail as validation).""" - if len(train_df) < 10: - n_val = max(1, int(round(len(train_df) * val_fraction))) - return train_df.iloc[:-n_val].copy(), train_df.iloc[-n_val:].copy() - - val_fraction = float(np.clip(val_fraction, 0.05, 0.5)) - if date_col in train_df.columns: - ordered = train_df.sort_values(date_col).reset_index(drop=True) - else: - ordered = train_df.reset_index(drop=True) - n_val = max(1, int(round(len(ordered) * val_fraction))) - fit = ordered.iloc[:-n_val].copy() - val = ordered.iloc[-n_val:].copy() - logger.info(f"Temporal split ({date_col}): fit={len(fit):,}, val={len(val):,}") - return fit, val - - -def train_baseline( - X_train: pd.DataFrame, - y_train: pd.Series, - X_test: pd.DataFrame, - y_test: pd.Series, - sample_weight: np.ndarray | None = None, - **kwargs: Any, -) -> tuple[LogisticRegression, dict[str, Any]]: - """Train logistic regression baseline.""" - model = LogisticRegression( - C=1.0, - max_iter=1000, - solver="lbfgs", - class_weight="balanced", - **kwargs, - ) - model.fit(X_train, y_train, sample_weight=sample_weight) - y_prob = model.predict_proba(X_test)[:, 1] - auc = roc_auc_score(y_test, y_prob) - metrics: dict[str, Any] = {"auc_roc": float(auc), "model_type": "logistic_regression"} - logger.info(f"Baseline LR — AUC: {auc:.4f}") - return model, metrics - - -def _catboost_base_params(params: dict[str, Any] | None = None) -> dict[str, Any]: - base = { - "iterations": 1000, - "loss_function": "Logloss", - "learning_rate": 0.05, - "depth": 6, - "l2_leaf_reg": 3, - "auto_class_weights": "Balanced", - "eval_metric": "AUC", - "random_seed": 42, - "allow_writing_files": False, - "verbose": 100, - "early_stopping_rounds": 50, - } - if params: - base.update(params) - return base - - -def resolve_monotonic_constraints( - feature_names: list[str], - constraints_config: dict[str, int] | None = None, - config_path: str = "configs/pd_model.yaml", -) -> str | None: - """Build CatBoost monotonic_constraints string from config. - - Reads the constraint map from YAML challenger_pipeline.monotonic_constraints - and maps it to feature order. Returns a comma-separated string like - "0,1,-1,0,..." for CatBoost's monotone_constraints parameter. - - Args: - feature_names: Ordered list of features used by the model. - constraints_config: Direct constraint dict {feature: direction}. - If None, reads from YAML config. - config_path: Path to pd_model.yaml. - - Returns: - Constraint string for CatBoost, or None if no constraints configured. - """ - if constraints_config is None: - try: - from pathlib import Path as _P - - import yaml - - cfg_path = _P(config_path) - if cfg_path.exists(): - with open(cfg_path) as f: - cfg = yaml.safe_load(f) or {} - constraints_config = ( - cfg.get("challenger_pipeline", {}).get("monotonic_constraints") or {} - ) - else: - return None - except Exception: - return None - - if not constraints_config: - return None - - vector = [constraints_config.get(feat, 0) for feat in feature_names] - if all(v == 0 for v in vector): - return None - - n_constrained = sum(1 for v in vector if v != 0) - logger.info(f"Monotonic constraints: {n_constrained}/{len(feature_names)} features constrained") - return ",".join(str(v) for v in vector) - - -def train_catboost_default( - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - X_test: pd.DataFrame | None = None, - y_test: pd.Series | None = None, - cat_features: list[str] | None = None, - params: dict[str, Any] | None = None, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, -) -> tuple[CatBoostClassifier, dict[str, Any]]: - """Train default CatBoost model using temporal validation set.""" - if cat_features is None: - cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] - model = CatBoostClassifier(**_catboost_base_params(params)) - model.fit( - Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight), - eval_set=Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight), - use_best_model=True, - ) - y_val_prob = model.predict_proba(X_val)[:, 1] - val_auc = roc_auc_score(y_val, y_val_prob) - metrics: dict[str, Any] = { - "validation_auc": float(val_auc), - "best_iteration": int(model.get_best_iteration()), - "model_type": "catboost_default", - } - if X_test is not None and y_test is not None: - y_test_prob = model.predict_proba(X_test)[:, 1] - metrics["auc_roc"] = float(roc_auc_score(y_test, y_test_prob)) - logger.info( - f"CatBoost default — val_AUC: {val_auc:.4f}, best_iter: {model.get_best_iteration()}" - ) - return model, metrics - - -def train_catboost( - X_train: pd.DataFrame, - y_train: pd.Series, - X_test: pd.DataFrame, - y_test: pd.Series, - cat_features: list[str] | None = None, - params: dict[str, Any] | None = None, -) -> tuple[CatBoostClassifier, dict[str, Any]]: - """Backward-compatible CatBoost train helper (uses same set for val/test).""" - model, metrics = train_catboost_default( - X_train, - y_train, - X_test, - y_test, - X_test=X_test, - y_test=y_test, - cat_features=cat_features, - params=params, - ) - return model, { - "auc_roc": float(metrics.get("auc_roc", metrics["validation_auc"])), - "best_iteration": int(metrics["best_iteration"]), - "model_type": "catboost", - } - - -def tune_catboost_optuna( - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - n_trials: int = 50, - cat_features: list[str] | None = None, -) -> dict[str, Any]: - """Backward-compatible wrapper returning only best params.""" - from src.models.optuna_tuning import train_catboost_tuned_optuna - - _, metrics = train_catboost_tuned_optuna( - X_train, - y_train, - X_val, - y_val, - cat_features=cat_features, - n_trials=n_trials, - ) - best_params = metrics.get("best_params", {}) - logger.info(f"Best params (wrapper): {best_params}") - return dict(best_params) if isinstance(best_params, dict) else {} - - -def __getattr__(name: str) -> Any: - """Lazy re-export for train_catboost_tuned_optuna (avoids circular import).""" - if name == "train_catboost_tuned_optuna": - from src.models.optuna_tuning import train_catboost_tuned_optuna - - return train_catboost_tuned_optuna - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/models/pd_pipeline.py b/src/models/pd_pipeline.py deleted file mode 100644 index 4394252..0000000 --- a/src/models/pd_pipeline.py +++ /dev/null @@ -1,204 +0,0 @@ -"""sklearn Pipeline wrapper for the PD modeling workflow. - -Provides a reproducible fit/predict interface that chains: - feature resolution → CatBoost classifier → probability calibrator - -Compatible with sklearn cross-validation, MAPIE conformal wrappers, -and model serialization via joblib/pickle. -""" - -from __future__ import annotations - -from typing import Any, cast - -import numpy as np -import pandas as pd -from loguru import logger -from sklearn.base import BaseEstimator, ClassifierMixin -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import FunctionTransformer - - -def _make_feature_selector( - feature_names: list[str], - fill_value: float = 0.0, -) -> FunctionTransformer: - """Create a transformer that selects and orders features. - - Ensures the input DataFrame always has the expected columns in the - correct order, filling missing ones with ``fill_value``. - """ - - def _select(X: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame(index=X.index) - for col in feature_names: - out[col] = X[col] if col in X.columns else fill_value - return out - - return FunctionTransformer(_select, validate=False) - - -class CatBoostSklearnAdapter(BaseEstimator, ClassifierMixin): - """Thin adapter making CatBoostClassifier fully sklearn-compatible. - - Handles categorical feature declaration, sample weights via ``fit``, - and exposes ``predict_proba`` / ``predict`` as expected by sklearn - Pipeline and MAPIE wrappers. - """ - - def __init__( - self, - catboost_params: dict[str, Any] | None = None, - cat_features: list[str] | None = None, - ): - self.catboost_params = catboost_params or {} - self.cat_features = cat_features or [] - - def fit( - self, - X: pd.DataFrame, - y: np.ndarray | pd.Series, - sample_weight: np.ndarray | None = None, - eval_set: tuple | None = None, - ) -> CatBoostSklearnAdapter: - from catboost import CatBoostClassifier, Pool - - self.model_ = CatBoostClassifier(**self.catboost_params) - cat_feats = [c for c in self.cat_features if c in X.columns] - train_pool = Pool(X, y, cat_features=cat_feats, weight=sample_weight) - - fit_kwargs: dict[str, Any] = {"use_best_model": True} - if eval_set is not None: - from catboost import Pool as _P - - X_val, y_val = eval_set - fit_kwargs["eval_set"] = _P(X_val, y_val, cat_features=cat_feats) - - self.model_.fit(train_pool, **fit_kwargs) - self.classes_ = np.array([0, 1]) - self.is_fitted_ = True - return self - - def predict_proba(self, X: pd.DataFrame) -> np.ndarray: - return cast(np.ndarray, self.model_.predict_proba(X)) - - def predict(self, X: pd.DataFrame) -> np.ndarray: - return cast(np.ndarray, self.model_.predict(X)) - - -class CalibratedPDPipeline(BaseEstimator, ClassifierMixin): - """End-to-end PD pipeline: feature selection → CatBoost → calibrator. - - Exposes ``predict_pd()`` for calibrated probabilities and - ``predict_proba()`` for sklearn compatibility. - - Args: - feature_names: Ordered list of model features. - cat_features: Categorical feature names (subset of feature_names). - catboost_params: CatBoost hyperparameters. - calibrator: Fitted probability calibrator (Platt/Isotonic/VennAbers/Beta). - """ - - def __init__( - self, - feature_names: list[str], - cat_features: list[str] | None = None, - catboost_params: dict[str, Any] | None = None, - calibrator: Any | None = None, - ): - self.feature_names = feature_names - self.cat_features = cat_features or [] - self.catboost_params = catboost_params or {} - self.calibrator = calibrator - - self._pipeline = Pipeline( - [ - ("feature_selector", _make_feature_selector(feature_names)), - ("classifier", CatBoostSklearnAdapter(catboost_params, self.cat_features)), - ] - ) - - def fit( - self, - X: pd.DataFrame, - y: np.ndarray | pd.Series, - **fit_params: Any, - ) -> CalibratedPDPipeline: - """Fit the full pipeline (feature selection + CatBoost).""" - self._pipeline.fit(X, y, **{f"classifier__{k}": v for k, v in fit_params.items()}) - self.classes_ = np.array([0, 1]) - self.is_fitted_ = True - logger.info( - f"CalibratedPDPipeline fitted: {len(self.feature_names)} features, " - f"calibrator={'yes' if self.calibrator else 'no'}" - ) - return self - - def predict_proba(self, X: pd.DataFrame) -> np.ndarray: - """Raw predict_proba (uncalibrated).""" - return cast(np.ndarray, self._pipeline.predict_proba(X)) - - def predict_pd(self, X: pd.DataFrame) -> np.ndarray: - """Return calibrated PD estimates.""" - raw = self._pipeline.predict_proba(X)[:, 1] - if self.calibrator is not None: - from src.models.conformal import apply_probability_calibrator - - return apply_probability_calibrator(self.calibrator, raw) - return cast(np.ndarray, raw) - - def predict(self, X: pd.DataFrame) -> np.ndarray: - """Binary prediction at default threshold.""" - return cast(np.ndarray, self._pipeline.predict(X)) - - @classmethod - def from_artifacts( - cls, - model_path: str = "models/pd_canonical.cbm", - calibrator_path: str = "models/pd_canonical_calibrator.pkl", - contract_path: str = "models/pd_model_contract.json", - ) -> CalibratedPDPipeline: - """Load a pre-trained pipeline from canonical artifacts. - - This is the recommended way to instantiate for inference. - """ - import json - import pickle - from pathlib import Path - - from catboost import CatBoostClassifier - - # Load contract - with open(contract_path) as f: - contract = json.load(f) - feature_names = contract["feature_names"] - cat_features = contract.get("categorical_features", []) - - # Load model - model = CatBoostClassifier() - model.load_model(model_path) - - # Load calibrator - cal_path = Path(calibrator_path) - calibrator = None - if cal_path.exists(): - with open(cal_path, "rb") as f: - calibrator = pickle.load(f) - - # Build pipeline with pre-fitted model - instance = cls( - feature_names=feature_names, - cat_features=cat_features, - calibrator=calibrator, - ) - adapter = instance._pipeline.named_steps["classifier"] - adapter.model_ = model - adapter.classes_ = np.array([0, 1]) - adapter.is_fitted_ = True - instance.is_fitted_ = True - - logger.info( - f"Loaded CalibratedPDPipeline from artifacts: " - f"{len(feature_names)} features, calibrator={'yes' if calibrator else 'no'}" - ) - return instance diff --git a/src/models/venn_abers.py b/src/models/venn_abers.py deleted file mode 100644 index 137f06a..0000000 --- a/src/models/venn_abers.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Stable Venn-Abers calibration helpers for canonical PD artifacts. - -## Audit note (C15, 2026-05-10) - -The original review wondered whether this custom wrapper duplicated logic -that the upstream ``venn-abers`` (PyPI) package already covers. After -inspection the answer is **no** — :class:`VennAbersScoreCalibrator` is a thin -~50 LOC interface adapter, not a re-implementation of the Venn-Abers -calibration algorithm. The actual math lives in ``venn_abers.VennAbers``, -which we delegate to inside ``fit`` and ``_predict_bounds``. - -What this wrapper adds on top of the upstream class: - -1. Accepts a **1-D vector of raw probabilities** ``y_prob_raw`` and - internally expands it to the ``[p0, p1]`` binary matrix the upstream - ``VennAbers.fit`` and ``.predict_proba`` expect. CRPTO calibrates from - CatBoost ``predict_proba(...)[:, 1]`` slices, which are 1-D by design. -2. Returns a **point estimate** (mean of the bounds) via ``predict`` while - preserving access to the raw bounds via ``_predict_bounds``. Other - calibrators in the project (Platt sigmoid, isotonic) expose the same - ``predict``/``predict_proba`` surface, so keeping that contract keeps - ``models/pd_canonical_calibrator.pkl`` interchangeable with the rest of - the pipeline. -3. Clips bounds into ``[0, 1]`` defensively (upstream occasionally emits - ``1.0 + epsilon`` from floating point). -4. Sorts ``p_bounds`` so callers cannot rely on ``[lo, hi]`` ordering when - the upstream returns ``[hi, lo]`` for very small calibration sets. - -Decision: **keep the wrapper.** Removing it would force every call site -(``pd_pipeline.py``, calibration scripts, MRM card generation) to repeat the -1-D → 2-D expansion and the contract adaptation. The wrapper is also part -of the frozen champion's calibrator pickle, so replacing it would invalidate -``models/pd_canonical_calibrator.pkl``. - -Future work, if and when the calibrator is re-trained: - -- Consider folding this class into ``src/models/calibration.py`` alongside - the Platt/Isotonic wrappers so all calibrators live in one module. -- Add a ``predict_intervals`` method that returns ``(low, high)`` directly - for downstream conformal pipelines (today they go through the private - ``_predict_bounds``). -""" - -from __future__ import annotations - -from typing import Any, cast - -import numpy as np - - -class VennAbersScoreCalibrator: - """Score-based Venn-Abers calibrator over 1D raw probabilities. - - Thin adapter around ``venn_abers.VennAbers``. See module docstring for - the audit notes and rationale for keeping the wrapper. - """ - - def __init__(self) -> None: - self._wrapped: Any = None - self._is_fitted = False - - @staticmethod - def _as_binary_proba(y_prob_raw: np.ndarray) -> np.ndarray: - p1 = np.clip(np.asarray(y_prob_raw, dtype=float).reshape(-1), 0.0, 1.0) - p0 = 1.0 - p1 - return np.column_stack([p0, p1]) - - def fit(self, y_prob_raw: np.ndarray, y_true: np.ndarray) -> VennAbersScoreCalibrator: - from venn_abers import VennAbers - - X = self._as_binary_proba(y_prob_raw) - y = np.asarray(y_true, dtype=int) - wrapped = VennAbers() - wrapped.fit(X, y) - self._wrapped = wrapped - self._is_fitted = True - return self - - def _predict_bounds(self, y_prob_raw: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - if not self._is_fitted or self._wrapped is None: - raise RuntimeError("VennAbersScoreCalibrator is not fitted.") - X = self._as_binary_proba(y_prob_raw) - _, p_bounds = self._wrapped.predict_proba(X) - p0 = np.clip(np.asarray(p_bounds[:, 0], dtype=float), 0.0, 1.0) - p1 = np.clip(np.asarray(p_bounds[:, 1], dtype=float), 0.0, 1.0) - low = np.minimum(p0, p1) - high = np.maximum(p0, p1) - return low, high - - def predict(self, y_prob_raw: np.ndarray) -> np.ndarray: - low, high = self._predict_bounds(y_prob_raw) - return cast(np.ndarray, np.clip((low + high) / 2.0, 0.0, 1.0)) - - def predict_proba(self, y_prob_raw: np.ndarray) -> np.ndarray: - p1 = self.predict(y_prob_raw) - p0 = 1.0 - p1 - return np.column_stack([p0, p1]) - - def predict_intervals(self, y_prob_raw: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Public accessor for the ``(low, high)`` Venn-Abers bounds. - - Added to remove the need for downstream conformal code to dip into - the private ``_predict_bounds``. - """ - return self._predict_bounds(y_prob_raw) diff --git a/src/optimization/__init__.py b/src/optimization/__init__.py index 2d76a26..c3dd662 100644 --- a/src/optimization/__init__.py +++ b/src/optimization/__init__.py @@ -1,31 +1,9 @@ -"""Operations Research: portfolio optimization, robust optimization, SPO+.""" +"""Portfolio policy and linear-allocation primitives used by the IJDS audit.""" from src.optimization.policy import PolicyMode, all_policy_modes, resolve_policy_mode -from src.optimization.tail_satisficing_objective import ( - SatisficingMargin, - SatisficingThreshold, - TailSatisficingObjectiveResult, - entropic_oce, - evaluate_satisficing_margins, - funded_loss_rate, - normalize_weights, - score_tail_satisficing_objective, - weighted_cvar, - weighted_mean, -) __all__ = [ "PolicyMode", - "SatisficingMargin", - "SatisficingThreshold", - "TailSatisficingObjectiveResult", "all_policy_modes", - "entropic_oce", - "evaluate_satisficing_margins", - "funded_loss_rate", - "normalize_weights", "resolve_policy_mode", - "score_tail_satisficing_objective", - "weighted_cvar", - "weighted_mean", ] diff --git a/src/optimization/certificate_semantics.py b/src/optimization/certificate_semantics.py deleted file mode 100644 index 345d9df..0000000 --- a/src/optimization/certificate_semantics.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Shared semantics for the paper-facing funded-set decision certificate.""" - -from __future__ import annotations - -from dataclasses import dataclass - -import numpy as np -import pandas as pd - -IJDS_DECLARED_ALPHA_GRID: tuple[float, ...] = ( - 0.01, - 0.03, - 0.05, - 0.07, - 0.10, - 0.12, - 0.15, - 0.20, -) -IJDS_DECLARED_ALPHA_GRID_CSV = ",".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) - - -@dataclass(frozen=True) -class FundedCertificateMetrics: - """Exact funded-set metrics and their policy-aware upper bounds. - - ``endpoint_budget`` and ``markov_loss_threshold`` are exact for the supplied - funded weights. ``endpoint_budget_upper`` and ``markov_loss_cap`` additionally - use the declared effective-PD constraint and its optional solver slack. - """ - - alpha: float - risk_tolerance: float - n_funded: int - weighted_outcome: float - weighted_miscoverage: float - weighted_coverage: float - empirical_coverage_funded: float - weighted_pd_point: float - weighted_pd_effective: float - endpoint_budget: float - gamma_cp: float - gamma_internalized: float - gamma_residual: float - effective_constraint_slack: float - effective_constraint_excess: float - realized_risk_tolerance_excess: float - sqrt_alpha: float - endpoint_budget_upper: float - markov_loss_threshold: float - markov_loss_cap: float - - -def _one_dimensional_float_array(values: np.ndarray, *, name: str) -> np.ndarray: - array = np.asarray(values, dtype=float) - if array.ndim != 1: - raise ValueError(f"{name} must be one-dimensional, got shape={array.shape}") - if not np.all(np.isfinite(array)): - raise ValueError(f"{name} contains non-finite values.") - return array - - -def _validate_probability_array(values: np.ndarray, *, name: str) -> None: - tolerance = 1e-10 - if np.any(values < -tolerance) or np.any(values > 1.0 + tolerance): - raise ValueError(f"{name} must stay on the [0, 1] probability scale.") - - -def compute_funded_certificate_metrics( - weights: np.ndarray, - outcomes: np.ndarray, - pd_point: np.ndarray, - pd_high: np.ndarray, - pd_effective: np.ndarray, - *, - alpha: float, - risk_tolerance: float, - pd_cap_slack: float = 0.0, - funded_tolerance: float = 1e-8, -) -> FundedCertificateMetrics: - """Compute the exact and policy-aware funded-set certificate. - - The effective PD vector may come from any policy family. This is important - for capped and tail-focused policies, where the linear-blend shortcut - ``(1 - gamma) * Gamma_CP`` is not generally the residual endpoint premium. - """ - arrays = { - "weights": _one_dimensional_float_array(weights, name="weights"), - "outcomes": _one_dimensional_float_array(outcomes, name="outcomes"), - "pd_point": _one_dimensional_float_array(pd_point, name="pd_point"), - "pd_high": _one_dimensional_float_array(pd_high, name="pd_high"), - "pd_effective": _one_dimensional_float_array(pd_effective, name="pd_effective"), - } - lengths = {len(values) for values in arrays.values()} - if len(lengths) != 1: - shapes = {name: values.shape for name, values in arrays.items()} - raise ValueError(f"Certificate arrays must have the same length: {shapes}") - - weights_arr = arrays["weights"] - outcomes_arr = arrays["outcomes"] - point_arr = arrays["pd_point"] - high_arr = arrays["pd_high"] - effective_arr = arrays["pd_effective"] - if np.any(weights_arr < 0.0): - raise ValueError("weights must be nonnegative.") - weight_sum = float(weights_arr.sum()) - if not np.isclose(weight_sum, 1.0, rtol=0.0, atol=1e-8): - raise ValueError(f"weights must sum to one, got {weight_sum:.12g}") - - for name, values in ( - ("outcomes", outcomes_arr), - ("pd_point", point_arr), - ("pd_high", high_arr), - ("pd_effective", effective_arr), - ): - _validate_probability_array(values, name=name) - order_tolerance = 1e-10 - if np.any(high_arr + order_tolerance < point_arr): - raise ValueError("pd_high must be at least pd_point for every row.") - if np.any(effective_arr + order_tolerance < point_arr) or np.any( - effective_arr > high_arr + order_tolerance - ): - raise ValueError("pd_effective must lie between pd_point and pd_high.") - - alpha_value = float(alpha) - if not 0.0 < alpha_value < 1.0: - raise ValueError(f"alpha must lie in (0, 1), got {alpha_value}") - tolerance_value = float(risk_tolerance) - if not 0.0 <= tolerance_value <= 1.0: - raise ValueError(f"risk_tolerance must lie in [0, 1], got {tolerance_value}") - cap_slack = float(pd_cap_slack) - if not np.isfinite(cap_slack) or cap_slack < 0.0: - raise ValueError(f"pd_cap_slack must be finite and nonnegative, got {cap_slack}") - - funded = weights_arr > float(funded_tolerance) - miscoverage = outcomes_arr > high_arr - weighted_outcome = float(weights_arr @ outcomes_arr) - weighted_miscoverage = float(weights_arr @ miscoverage.astype(float)) - weighted_point = float(weights_arr @ point_arr) - weighted_effective = float(weights_arr @ effective_arr) - endpoint_budget = float(weights_arr @ high_arr) - gamma_cp = float(weights_arr @ np.clip(high_arr - point_arr, 0.0, 1.0)) - gamma_internalized = float(weights_arr @ np.clip(effective_arr - point_arr, 0.0, 1.0)) - gamma_residual = float(weights_arr @ np.clip(high_arr - effective_arr, 0.0, 1.0)) - if not np.isclose( - gamma_cp, - gamma_internalized + gamma_residual, - rtol=0.0, - atol=1e-9, - ): - raise ValueError("Conformal-premium decomposition is not internally consistent.") - - effective_cap = tolerance_value + cap_slack - effective_constraint_slack = max(0.0, effective_cap - weighted_effective) - effective_constraint_excess = max(0.0, weighted_effective - effective_cap) - endpoint_budget_upper = effective_cap + gamma_residual - sqrt_alpha = float(np.sqrt(alpha_value)) - empirical_coverage = float(1.0 - miscoverage[funded].mean()) if funded.any() else float("nan") - - return FundedCertificateMetrics( - alpha=alpha_value, - risk_tolerance=tolerance_value, - n_funded=int(funded.sum()), - weighted_outcome=weighted_outcome, - weighted_miscoverage=weighted_miscoverage, - weighted_coverage=1.0 - weighted_miscoverage, - empirical_coverage_funded=empirical_coverage, - weighted_pd_point=weighted_point, - weighted_pd_effective=weighted_effective, - endpoint_budget=endpoint_budget, - gamma_cp=gamma_cp, - gamma_internalized=gamma_internalized, - gamma_residual=gamma_residual, - effective_constraint_slack=effective_constraint_slack, - effective_constraint_excess=effective_constraint_excess, - realized_risk_tolerance_excess=max(0.0, weighted_outcome - tolerance_value), - sqrt_alpha=sqrt_alpha, - endpoint_budget_upper=endpoint_budget_upper, - markov_loss_threshold=endpoint_budget + sqrt_alpha, - markov_loss_cap=endpoint_budget_upper + sqrt_alpha, - ) - - -def add_policy_aware_bound_columns(frame: pd.DataFrame) -> pd.DataFrame: - """Add exact and policy-aware bound columns to alpha-grid evaluation rows. - - Historical pool93 evaluations already store the sufficient statistics for - this decomposition. Rehydrating the bounds from those columns corrects - capped/tail policy semantics without re-solving any portfolio. - """ - required = { - "alpha", - "weighted_pd_high", - "weighted_pd_constraint_used", - } - missing = sorted(required.difference(frame.columns)) - if missing: - raise ValueError(f"Bound-evaluation frame is missing required columns: {missing}") - tolerance_column = "risk_tolerance" if "risk_tolerance" in frame.columns else "tau" - if tolerance_column not in frame.columns: - raise ValueError("Bound-evaluation frame requires risk_tolerance or tau.") - - work = frame.copy() - if "realized_risk_tolerance_excess" not in work.columns and "violation" in work.columns: - work["realized_risk_tolerance_excess"] = pd.to_numeric( - work["violation"], errors="raise" - ).astype(float) - if ( - "empirical_risk_excess_leq_alpha" not in work.columns - and "bound_a_expected_violation_leq_alpha" in work.columns - ): - work["empirical_risk_excess_leq_alpha"] = work[ - "bound_a_expected_violation_leq_alpha" - ].astype(bool) - alpha = pd.to_numeric(work["alpha"], errors="raise").astype(float) - endpoint = pd.to_numeric(work["weighted_pd_high"], errors="raise").astype(float) - effective = pd.to_numeric(work["weighted_pd_constraint_used"], errors="raise").astype(float) - tolerance = pd.to_numeric(work[tolerance_column], errors="raise").astype(float) - slack = ( - pd.to_numeric(work["pd_cap_slack"], errors="raise").astype(float) - if "pd_cap_slack" in work.columns - else pd.Series(0.0, index=work.index, dtype=float) - ) - numeric = pd.concat( - { - "alpha": alpha, - "endpoint": endpoint, - "effective": effective, - "tolerance": tolerance, - "slack": slack, - }, - axis=1, - ) - if not np.isfinite(numeric.to_numpy(dtype=float)).all(): - raise ValueError("Bound-evaluation frame contains non-finite certificate inputs.") - if (alpha <= 0.0).any() or (alpha >= 1.0).any(): - raise ValueError("Bound-evaluation alpha values must lie in (0, 1).") - if (slack < 0.0).any(): - raise ValueError("Bound-evaluation pd_cap_slack must be nonnegative.") - - residual = endpoint - effective - if (residual < -1e-9).any(): - raise ValueError("weighted_pd_high must be at least weighted_pd_constraint_used.") - residual = residual.clip(lower=0.0) - effective_cap = tolerance + slack - sqrt_alpha = np.sqrt(alpha) - work["gamma_residual"] = residual - if "weighted_pd_point" in work.columns: - point = pd.to_numeric(work["weighted_pd_point"], errors="raise").astype(float) - work["gamma_internalized"] = (effective - point).clip(lower=0.0) - elif "gamma_cp" in work.columns: - gamma_cp = pd.to_numeric(work["gamma_cp"], errors="raise").astype(float) - work["gamma_internalized"] = (gamma_cp - residual).clip(lower=0.0) - work["effective_constraint_slack"] = (effective_cap - effective).clip(lower=0.0) - work["effective_constraint_excess"] = (effective - effective_cap).clip(lower=0.0) - work["endpoint_budget"] = endpoint - work["endpoint_budget_upper"] = effective_cap + residual - work["markov_loss_threshold"] = endpoint + sqrt_alpha - work["markov_loss_cap"] = work["endpoint_budget_upper"] + sqrt_alpha - return work diff --git a/src/optimization/cuopt_adapter.py b/src/optimization/cuopt_adapter.py index f5e9e57..67593d9 100644 --- a/src/optimization/cuopt_adapter.py +++ b/src/optimization/cuopt_adapter.py @@ -257,6 +257,7 @@ def solve_portfolio_cuopt_native( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=None, ) dm = _cuopt_data_model(lp_api, components) settings, applied_parameters, rejected_parameters = _cuopt_solver_settings( diff --git a/src/optimization/input_alignment.py b/src/optimization/input_alignment.py deleted file mode 100644 index 2091af6..0000000 --- a/src/optimization/input_alignment.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Deterministic alignment of candidate loans and conformal interval rows.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -import numpy as np -import pandas as pd - -AlignmentMode = Literal["id", "row_number", "position"] - - -@dataclass(frozen=True) -class AlignedCandidateIntervals: - """Candidate and interval frames that share the same sampled row order.""" - - candidates: pd.DataFrame - intervals: pd.DataFrame - mode: AlignmentMode - available_rows: int - - @property - def selected_rows(self) -> int: - return len(self.candidates) - - -def _normalized_limit(max_candidates: int | None) -> int | None: - if max_candidates is None or int(max_candidates) <= 0: - return None - return int(max_candidates) - - -def _sample_positions( - row_count: int, - *, - max_candidates: int | None, - random_state: int, -) -> np.ndarray: - if row_count <= 0: - raise ValueError("Candidate/interval alignment produced zero rows.") - limit = _normalized_limit(max_candidates) - sample_size = row_count if limit is None else min(row_count, limit) - if sample_size == row_count: - return np.arange(row_count, dtype=int) - positions = np.random.default_rng(random_state).choice( - row_count, - size=sample_size, - replace=False, - ) - return np.sort(positions.astype(int, copy=False)) - - -def _require_unique_columns(frame: pd.DataFrame, *, source: str) -> None: - duplicated = frame.columns[frame.columns.duplicated()].tolist() - if duplicated: - raise ValueError(f"{source} contains duplicate column names: {duplicated}") - - -def _validated_string_key(series: pd.Series, *, source: str) -> pd.Series: - if series.isna().any(): - raise ValueError(f"{source} alignment key contains missing values.") - keys = series.astype(str) - if keys.str.strip().eq("").any(): - raise ValueError(f"{source} alignment key contains blank values.") - duplicated = keys[keys.duplicated(keep=False)] - if not duplicated.empty: - examples = duplicated.drop_duplicates().head(3).tolist() - raise ValueError(f"{source} alignment key is not unique; examples={examples}") - return keys - - -def _validated_row_number_key(series: pd.Series) -> pd.Series: - if series.isna().any(): - raise ValueError("interval _row_number contains missing values.") - numeric = pd.to_numeric(series, errors="raise") - values = numeric.to_numpy(dtype=float) - if not np.isfinite(values).all() or not np.equal(values, np.floor(values)).all(): - raise ValueError("interval _row_number must contain finite integers.") - keys = pd.Series(values.astype(np.int64), index=series.index) - duplicated = keys[keys.duplicated(keep=False)] - if not duplicated.empty: - examples = duplicated.drop_duplicates().head(3).tolist() - raise ValueError(f"interval _row_number is not unique; examples={examples}") - return keys - - -def _unused_column_name(base: str, used: set[str]) -> str: - name = base - suffix = 1 - while name in used: - name = f"{base}_{suffix}" - suffix += 1 - used.add(name) - return name - - -def _align_by_key( - candidates: pd.DataFrame, - intervals: pd.DataFrame, - *, - candidate_keys: pd.Series, - interval_keys: pd.Series, - mode: AlignmentMode, - max_candidates: int | None, - random_state: int, -) -> AlignedCandidateIntervals: - used_columns = set(candidates.columns) | set(intervals.columns) - join_column = _unused_column_name("__crpto_alignment_key", used_columns) - interval_aliases = { - column: _unused_column_name(f"__crpto_interval_{index}", used_columns) - for index, column in enumerate(intervals.columns) - } - - candidate_work = candidates.copy() - candidate_work[join_column] = candidate_keys.to_numpy() - interval_work = intervals.rename(columns=interval_aliases).copy() - interval_work[join_column] = interval_keys.to_numpy() - merged = candidate_work.merge( - interval_work, - on=join_column, - how="inner", - sort=False, - validate="one_to_one", - ) - positions = _sample_positions( - len(merged), - max_candidates=max_candidates, - random_state=random_state, - ) - sampled = merged.iloc[positions].reset_index(drop=True) - aligned_candidates = sampled.loc[:, list(candidates.columns)].copy() - aligned_intervals = sampled.loc[:, list(interval_aliases.values())].rename( - columns={alias: source for source, alias in interval_aliases.items()} - ) - return AlignedCandidateIntervals( - candidates=aligned_candidates, - intervals=aligned_intervals, - mode=mode, - available_rows=len(merged), - ) - - -def align_candidate_intervals( - candidates: pd.DataFrame, - intervals: pd.DataFrame, - *, - max_candidates: int | None, - random_state: int, - allow_row_number: bool = True, -) -> AlignedCandidateIntervals: - """Align and sample candidate/interval rows under a strict one-to-one contract. - - Stable IDs take precedence. Legacy interval artifacts may instead provide a - zero-based ``_row_number``. A positional fallback is retained for older - artifacts, but samples from the full alignable universe rather than taking - a deterministic prefix. - """ - _require_unique_columns(candidates, source="candidates") - _require_unique_columns(intervals, source="intervals") - - if "id" in candidates.columns and "id" in intervals.columns: - return _align_by_key( - candidates, - intervals, - candidate_keys=_validated_string_key(candidates["id"], source="candidate id"), - interval_keys=_validated_string_key(intervals["id"], source="interval id"), - mode="id", - max_candidates=max_candidates, - random_state=random_state, - ) - - if allow_row_number and "_row_number" in intervals.columns: - return _align_by_key( - candidates, - intervals, - candidate_keys=pd.Series(np.arange(len(candidates), dtype=np.int64)), - interval_keys=_validated_row_number_key(intervals["_row_number"]), - mode="row_number", - max_candidates=max_candidates, - random_state=random_state, - ) - - available_rows = min(len(candidates), len(intervals)) - positions = _sample_positions( - available_rows, - max_candidates=max_candidates, - random_state=random_state, - ) - return AlignedCandidateIntervals( - candidates=candidates.iloc[positions].reset_index(drop=True).copy(), - intervals=intervals.iloc[positions].reset_index(drop=True).copy(), - mode="position", - available_rows=available_rows, - ) diff --git a/src/optimization/policy_evaluation.py b/src/optimization/policy_evaluation.py index 43e69e2..0cf11e9 100644 --- a/src/optimization/policy_evaluation.py +++ b/src/optimization/policy_evaluation.py @@ -42,6 +42,8 @@ def solve_policy_allocation( max_concentration: float = 0.25, risk_tolerance: float = 0.10, robust: bool = True, + pd_constraint_override: np.ndarray | None = None, + objective_rate_override: np.ndarray | None = None, uncertainty_aversion: float = 0.0, min_budget_utilization: float = 0.0, pd_cap_slack_penalty: float = 0.0, @@ -66,15 +68,22 @@ def solve_policy_allocation( effective_delta_cap = float(delta_cap_quantile) if robust else 1.0 effective_tail_focus = float(tail_focus_quantile) if robust else 1.0 effective_aversion = float(uncertainty_aversion) if robust else 0.0 - effective_pd = compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=effective_mode, - gamma=effective_gamma, - delta_cap_quantile=effective_delta_cap, - tail_focus_quantile=effective_tail_focus, - segment_labels=policy_segment_labels(loans, effective_mode), - ) + if pd_constraint_override is None: + effective_pd = compute_effective_pd( + pd_point=pd_point, + pd_high=pd_high, + policy_mode=effective_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, + segment_labels=policy_segment_labels(loans, effective_mode), + ) + else: + effective_pd = np.asarray(pd_constraint_override, dtype=float) + if effective_pd.shape != np.asarray(pd_point).shape: + raise ValueError("pd_constraint_override must align with pd_point.") + if not bool(np.isfinite(effective_pd).all()): + raise ValueError("pd_constraint_override must contain finite values.") solution = optimize_portfolio_allocation( loans=loans, pd_point=pd_point, @@ -90,6 +99,7 @@ def solve_policy_allocation( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=effective_pd, + objective_rate_override=objective_rate_override, time_limit=time_limit, threads=threads, solver_backend=solver_backend, diff --git a/src/optimization/portfolio_model.py b/src/optimization/portfolio_model.py index ebc9138..7c0024a 100644 --- a/src/optimization/portfolio_model.py +++ b/src/optimization/portfolio_model.py @@ -441,6 +441,7 @@ def solve_portfolio_highs_sparse( min_budget_utilization: float = 0.0, pd_cap_slack_penalty: float = 0.0, pd_constraint_override: np.ndarray | None = None, + objective_rate_override: np.ndarray | None = None, time_limit: int = 300, threads: int = 4, ) -> dict[str, Any]: @@ -464,6 +465,7 @@ def solve_portfolio_highs_sparse( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=objective_rate_override, ) options: dict[str, Any] = { @@ -526,6 +528,7 @@ def solve_portfolio_highspy_native( min_budget_utilization: float = 0.0, pd_cap_slack_penalty: float = 0.0, pd_constraint_override: np.ndarray | None = None, + objective_rate_override: np.ndarray | None = None, time_limit: int = 300, threads: int = 4, ) -> dict[str, Any]: @@ -557,6 +560,7 @@ def solve_portfolio_highspy_native( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=objective_rate_override, ) lp = _build_highspy_lp(highspy, components) solver = highspy.Highs() @@ -702,6 +706,7 @@ def _portfolio_lp_components( min_budget_utilization: float, pd_cap_slack_penalty: float, pd_constraint_override: np.ndarray | None, + objective_rate_override: np.ndarray | None, ) -> _PortfolioLpComponents: n = len(loans) if n == 0: @@ -718,8 +723,16 @@ def _portfolio_lp_components( else (high if robust else point) ) pd_uncertainty = np.clip(high - point, 0.0, 1.0) + if objective_rate_override is None: + objective_rate = rates - point * lgd_arr + else: + objective_rate = np.asarray(objective_rate_override, dtype=float) + if objective_rate.shape != point.shape: + raise ValueError("objective_rate_override must align with pd_point.") + if not bool(np.isfinite(objective_rate).all()): + raise ValueError("objective_rate_override must contain finite values.") objective = loan_amounts * ( - rates - point * lgd_arr - float(uncertainty_aversion) * pd_uncertainty * lgd_arr + objective_rate - float(uncertainty_aversion) * pd_uncertainty * lgd_arr ) rows, rhs, pd_cap_row_idx = _portfolio_constraint_rows( @@ -829,6 +842,7 @@ def optimize_portfolio_allocation( min_budget_utilization: float = 0.0, pd_cap_slack_penalty: float = 0.0, pd_constraint_override: np.ndarray | None = None, + objective_rate_override: np.ndarray | None = None, time_limit: int = 300, threads: int = 4, solver_backend: str = "highs", @@ -854,6 +868,7 @@ def optimize_portfolio_allocation( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=objective_rate_override, time_limit=time_limit, threads=threads, ) @@ -874,6 +889,7 @@ def optimize_portfolio_allocation( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=objective_rate_override, time_limit=time_limit, threads=threads, ) @@ -903,6 +919,7 @@ def optimize_portfolio_allocation( min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, pd_constraint_override=pd_constraint_override, + objective_rate_override=objective_rate_override, time_limit=time_limit, threads=threads, ) @@ -910,6 +927,8 @@ def optimize_portfolio_allocation( result["native_solver_error"] = str(exc) return result if backend == "cuopt": + if objective_rate_override is not None: + raise ValueError("objective_rate_override is not supported by the cuOpt backend.") from src.optimization.cuopt_adapter import solve_portfolio_cuopt_native return solve_portfolio_cuopt_native( @@ -938,6 +957,9 @@ def optimize_portfolio_allocation( "Use 'highs', 'highs_sparse', 'highspy', 'highs_pyomo', or 'cuopt'." ) + if objective_rate_override is not None: + raise ValueError("objective_rate_override is not supported by the Pyomo backend.") + model = build_portfolio_model( loans=loans, pd_point=pd_point, diff --git a/src/optimization/robust_opt.py b/src/optimization/robust_opt.py deleted file mode 100644 index 75a77bf..0000000 --- a/src/optimization/robust_opt.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Robust optimization using conformal prediction uncertainty sets. - -Converts CP intervals into uncertainty sets for robust portfolio optimization. -Supports box uncertainty (intervals) and ellipsoidal uncertainty. -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -from loguru import logger - - -def build_box_uncertainty_set( - pd_low: np.ndarray, - pd_high: np.ndarray, - lgd_low: np.ndarray | None = None, - lgd_high: np.ndarray | None = None, -) -> dict[str, np.ndarray]: - """Build box uncertainty set from conformal prediction intervals. - - The box set is: {PD : PD_low <= PD <= PD_high} for each loan. - This is directly from split conformal prediction with coverage guarantee. - """ - uncertainty_set = { - "pd_low": pd_low, - "pd_high": pd_high, - "pd_center": (pd_low + pd_high) / 2, - "pd_radius": (pd_high - pd_low) / 2, - } - - if lgd_low is not None and lgd_high is not None: - uncertainty_set.update( - { - "lgd_low": lgd_low, - "lgd_high": lgd_high, - "lgd_center": (lgd_low + lgd_high) / 2, - "lgd_radius": (lgd_high - lgd_low) / 2, - } - ) - - logger.info( - f"Box uncertainty set: {len(pd_low)} loans, " - f"avg PD width={uncertainty_set['pd_radius'].mean() * 2:.4f}" - ) - return uncertainty_set - - -def worst_case_expected_loss( - allocation: np.ndarray, - loan_amounts: np.ndarray, - pd_high: np.ndarray, - lgd_high: np.ndarray | None = None, - lgd_point: np.ndarray | None = None, -) -> float: - """Compute worst-case expected loss under box uncertainty. - - Uses upper bounds from conformal prediction for conservative estimate. - """ - lgd = ( - lgd_high - if lgd_high is not None - else (lgd_point if lgd_point is not None else np.ones_like(pd_high) * 0.45) - ) - return float(np.sum(allocation * loan_amounts * pd_high * lgd)) - - -def scenario_analysis( - allocation: np.ndarray, - loan_amounts: np.ndarray, - pd_low: np.ndarray, - pd_point: np.ndarray, - pd_high: np.ndarray, - lgd: np.ndarray, -) -> pd.DataFrame: - """Run scenario analysis: best-case, expected, worst-case.""" - scenarios = { - "best_case": np.sum(allocation * loan_amounts * pd_low * lgd), - "expected": np.sum(allocation * loan_amounts * pd_point * lgd), - "worst_case": np.sum(allocation * loan_amounts * pd_high * lgd), - } - result = pd.DataFrame([scenarios]) - result["range"] = result["worst_case"] - result["best_case"] - logger.info(f"Scenario analysis: {scenarios}") - return result diff --git a/src/utils/__init__.py b/src/utils/__init__.py index 4cb717b..21a845c 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -1 +1 @@ -"""Utility functions: MLflow tracking, visualization.""" +"""Artifact, isolated-run, and atomic-write utilities.""" diff --git a/src/utils/artifact_descriptor.py b/src/utils/artifact_descriptor.py new file mode 100644 index 0000000..2da2feb --- /dev/null +++ b/src/utils/artifact_descriptor.py @@ -0,0 +1,49 @@ +"""Lightweight repository artifact hashing without scientific dependencies.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from pathlib import Path +from typing import Any + + +def sha256_file(path: Path, *, block_size: int = 8 * 1024 * 1024) -> str: + """Compute a streaming SHA-256 digest.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(block_size): + digest.update(block) + return digest.hexdigest() + + +def relative_artifact_descriptor(path: Path, *, repo_root: Path) -> dict[str, Any]: + """Describe one repository-contained artifact by path, size, and hash.""" + resolved = path.resolve() + relative = resolved.relative_to(repo_root.resolve()).as_posix() + return { + "path": relative, + "bytes": int(resolved.stat().st_size), + "sha256": sha256_file(resolved), + } + + +def verified_artifact_path( + descriptor: Mapping[str, Any], + *, + repo_root: Path, + label: str, +) -> Path: + """Resolve one repository artifact only after path, size, and hash agree.""" + raw_path = descriptor.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise TypeError(f"{label} descriptor omits a path.") + path = (repo_root / raw_path).resolve() + path.relative_to(repo_root.resolve()) + if not path.is_file(): + raise FileNotFoundError(f"{label} artifact is missing: {path}") + actual = relative_artifact_descriptor(path, repo_root=repo_root) + for field in ("path", "bytes", "sha256"): + if actual[field] != descriptor.get(field): + raise RuntimeError(f"{label} artifact mismatched on {field}.") + return path diff --git a/src/utils/artifact_metadata.py b/src/utils/artifact_metadata.py deleted file mode 100644 index 803936d..0000000 --- a/src/utils/artifact_metadata.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Shared helpers for run-tag resolution and artifact metadata contracts.""" - -from __future__ import annotations - -import os -from collections.abc import Iterable -from datetime import UTC, datetime -from typing import Any - - -class MissingRunTagError(ValueError): - """Raised when an official artifact writer lacks a resolvable run tag.""" - - -def utc_now_iso() -> str: - return datetime.now(UTC).isoformat() - - -def _candidate_strings(values: Iterable[object | None]) -> list[str]: - out: list[str] = [] - for value in values: - text = str(value or "").strip() - if text: - out.append(text) - return out - - -def resolve_run_tag( - explicit: object | None = None, - *, - env_var: str = "PIPELINE_RUN_TAG", - fallback_candidates: Iterable[object | None] | None = None, - require_explicit: bool = False, - allow_untracked: bool = False, -) -> str: - candidates = _candidate_strings( - [explicit, os.environ.get(env_var, ""), *(fallback_candidates or ())] - ) - for candidate in candidates: - if candidate.lower() not in {"untracked", "unknown", "none"}: - return candidate - if allow_untracked: - return "untracked" - if require_explicit: - raise MissingRunTagError(f"Missing official run_tag. Provide --run-tag or set {env_var}.") - return "untracked" - - -def build_artifact_metadata( - *, - schema_version: str, - run_tag: object | None = None, - fallback_candidates: Iterable[object | None] | None = None, - require_explicit: bool = False, - allow_untracked: bool = False, - extra: dict[str, Any] | None = None, -) -> dict[str, Any]: - payload = { - "schema_version": schema_version, - "generated_at_utc": utc_now_iso(), - "run_tag": resolve_run_tag( - run_tag, - fallback_candidates=fallback_candidates, - require_explicit=require_explicit, - allow_untracked=allow_untracked, - ), - } - if extra: - payload.update(extra) - return payload diff --git a/src/utils/baseline_registry.py b/src/utils/baseline_registry.py deleted file mode 100644 index 9f672f4..0000000 --- a/src/utils/baseline_registry.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Helpers for resolving the current official baseline registry.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[2] -PRIMARY_BASELINE = ROOT / "configs" / "baselines" / "canonical_operational_baseline.json" -LEGACY_BASELINE = ROOT / "configs" / "baselines" / "core_official_baseline.json" - - -def _load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def load_official_baseline_registry() -> dict[str, Any]: - for path in (PRIMARY_BASELINE, LEGACY_BASELINE): - payload = _load_json(path) - if payload: - payload = dict(payload) - payload["_path"] = str(path) - return payload - return {} - - -def resolve_official_baseline_run_tag(default: str | None = None) -> str | None: - payload = load_official_baseline_registry() - value = str(payload.get("official_run_tag", "")).strip() - if value: - return value - return default diff --git a/src/utils/io_utils.py b/src/utils/io_utils.py deleted file mode 100644 index 048f067..0000000 --- a/src/utils/io_utils.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Shared I/O utilities for data loading with fallback and pickle compatibility.""" - -from __future__ import annotations - -import pickle -from pathlib import Path -from typing import Any - -import pandas as pd -from loguru import logger - - -def read_with_fallback(primary_path: str | Path, fallback_path: str | Path) -> pd.DataFrame: - """Read parquet from primary path; fall back to alternate if missing. - - Args: - primary_path: Preferred file path. - fallback_path: Alternate file path if primary is unavailable. - - Returns: - DataFrame loaded from whichever path exists. - - Raises: - FileNotFoundError: If neither path exists. - """ - primary = Path(primary_path) - fallback = Path(fallback_path) - if primary.exists(): - return pd.read_parquet(primary) - if fallback.exists(): - logger.warning(f"{primary} not found. Falling back to {fallback}") - return pd.read_parquet(fallback) - raise FileNotFoundError(f"Neither {primary} nor {fallback} exists") - - -def read_split_with_fe_fallback(path: str | Path) -> pd.DataFrame: - """Read a data split, trying *_fe variant and base variant as fallback. - - Handles the common pattern where feature-engineered splits (*_fe.parquet) - may or may not exist alongside base splits (*.parquet). - - Args: - path: Configured path (either *_fe.parquet or *.parquet). - - Returns: - DataFrame from whichever variant exists. - - Raises: - FileNotFoundError: If neither variant exists. - """ - p = Path(path) - if p.exists(): - return pd.read_parquet(p) - - alt = None - if p.name.endswith("_fe.parquet"): - alt = p.with_name(p.name.replace("_fe.parquet", ".parquet")) - elif p.name.endswith(".parquet"): - alt = p.with_name(p.name.replace(".parquet", "_fe.parquet")) - - if alt is not None and alt.exists(): - logger.warning(f"Configured path not found: {p}. Falling back to {alt}") - return pd.read_parquet(alt) - - raise FileNotFoundError(f"Neither configured path nor fallback exists: {p}") - - -class _CompatUnpickler(pickle.Unpickler): - """Restore legacy objects that were pickled from script-local ``__main__`` classes.""" - - def find_class(self, module: str, name: str) -> Any: - if module == "__main__" and name == "VennAbersScoreCalibrator": - from src.models.venn_abers import VennAbersScoreCalibrator - - return VennAbersScoreCalibrator - return super().find_class(module, name) - - -def load_pickle_compat(path: str | Path) -> Any: - target = Path(path) - with open(target, "rb") as f: - return _CompatUnpickler(f).load() diff --git a/src/utils/isolated_experiment.py b/src/utils/isolated_experiment.py new file mode 100644 index 0000000..23cda00 --- /dev/null +++ b/src/utils/isolated_experiment.py @@ -0,0 +1,271 @@ +"""Containment, provenance, and atomic I/O for isolated experiments.""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import os +import platform +import re +import subprocess +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pandas as pd +from catboost import CatBoostClassifier + +from src.utils.pipeline_runtime import atomic_write_text + +RUN_TAG_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$") + + +@dataclass(frozen=True) +class OutputPaths: + """Validated versioned output directories for one experiment run.""" + + data_dir: Path + model_dir: Path + + +def validate_run_tag(run_tag: str) -> str: + """Reject separator-bearing or traversal-capable experiment tags.""" + value = str(run_tag).strip() + if not RUN_TAG_PATTERN.fullmatch(value) or value in {".", ".."}: + raise ValueError(f"Unsafe run_tag: {run_tag!r}") + return value + + +def resolve_isolated_run_dir( + *, + repo_root: Path, + configured_root: str | Path, + allowed_relative_root: Path, + run_tag: str, +) -> Path: + """Resolve one direct child under an exact allowlisted experiment root.""" + safe_tag = validate_run_tag(run_tag) + root = repo_root.resolve() + configured = Path(configured_root) + resolved_base = ( + (root / configured).resolve() if not configured.is_absolute() else configured.resolve() + ) + allowed_base = (root / allowed_relative_root).resolve() + if resolved_base != allowed_base: + raise ValueError( + f"Output root {resolved_base} is not the allowlisted experiment root {allowed_base}." + ) + candidate = (resolved_base / safe_tag).resolve() + relative = candidate.relative_to(resolved_base) + if len(relative.parts) != 1 or relative.name != safe_tag: + raise ValueError("Experiment output must be a direct run-tag child.") + return candidate + + +def prepare_output_paths( + config: dict[str, Any], + *, + repo_root: Path, + allowed_data_root: Path, + allowed_model_root: Path, +) -> OutputPaths: + """Validate containment and enforce immutable no-overwrite outputs.""" + output = config["output"] + run_tag = str(config["run_tag"]) + data_dir = resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["data_root"]), + allowed_relative_root=allowed_data_root, + run_tag=run_tag, + ) + model_dir = resolve_isolated_run_dir( + repo_root=repo_root, + configured_root=str(output["model_root"]), + allowed_relative_root=allowed_model_root, + run_tag=run_tag, + ) + existing = [path for path in (data_dir, model_dir) if path.exists()] + if existing: + rendered = ", ".join(str(path) for path in existing) + raise FileExistsError( + f"Experiment output already exists ({rendered}); choose a fresh run tag." + ) + data_dir.mkdir(parents=True, exist_ok=True) + model_dir.mkdir(parents=True, exist_ok=True) + return OutputPaths(data_dir=data_dir, model_dir=model_dir) + + +def resolve_repo_input(path_like: str | Path, *, repo_root: Path) -> Path: + """Resolve a required input and reject paths outside the repository.""" + root = repo_root.resolve() + raw = Path(path_like) + path = (root / raw).resolve() if not raw.is_absolute() else raw.resolve() + try: + path.relative_to(root) + except ValueError as exc: + raise ValueError(f"Input path must remain inside the repository: {path}") from exc + if not path.is_file(): + raise FileNotFoundError(path) + return path + + +def sha256_file(path: Path, *, block_size: int = 8 * 1024 * 1024) -> str: + """Compute a streaming SHA-256 digest.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + while block := handle.read(block_size): + digest.update(block) + return digest.hexdigest() + + +def relative_artifact_descriptor(path: Path, *, repo_root: Path) -> dict[str, Any]: + """Describe an artifact with a repository-relative path and hash.""" + resolved = path.resolve() + relative = resolved.relative_to(repo_root.resolve()).as_posix() + return { + "path": relative, + "bytes": int(resolved.stat().st_size), + "sha256": sha256_file(resolved), + } + + +def implementation_provenance( + *, + config_path: Path, + relative_paths: Sequence[Path], + repo_root: Path, +) -> dict[str, Any]: + """Bind deterministic results to every scientific implementation file.""" + files = [config_path.resolve(), *((repo_root / path).resolve() for path in relative_paths)] + missing = [str(path) for path in files if not path.is_file()] + if missing: + raise FileNotFoundError(f"Implementation provenance files are missing: {missing}") + descriptors = [relative_artifact_descriptor(path, repo_root=repo_root) for path in files] + return { + "source_files": {str(item["path"]): item for item in descriptors}, + "hash_algorithm": "sha256", + } + + +def package_version(distribution: str) -> str | None: + """Return an installed distribution version when available.""" + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return None + + +def git_provenance(repo_root: Path) -> dict[str, Any]: + """Capture commit and dirty state without mutating Git.""" + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + status = subprocess.run( + ["git", "status", "--short"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + return { + "commit": commit, + "dirty": bool(status), + "dirty_entries": len(status), + "dirty_paths": status, + } + except (OSError, subprocess.CalledProcessError) as exc: + return {"commit": None, "dirty": None, "error": str(exc)} + + +def require_clean_tagged_head(repo_root: Path, tag: str) -> str: + """Require the protocol tag to resolve exactly to a clean current HEAD.""" + state = git_provenance(repo_root) + commit = state.get("commit") + if not isinstance(commit, str) or not commit: + raise RuntimeError("A readable Git HEAD is required before experiment execution.") + if state.get("dirty") is not False: + raise RuntimeError("Experiment execution requires a clean predeclared worktree.") + tagged_commit = resolve_git_tag(repo_root, tag) + if tagged_commit != commit: + raise RuntimeError( + f"Protocol tag {tag!r} points to {tagged_commit!r}, not current HEAD {commit!r}." + ) + return commit + + +def resolve_git_tag(repo_root: Path, tag: str) -> str: + """Resolve one required annotated or lightweight tag to its commit.""" + try: + commit = subprocess.run( + ["git", "rev-list", "-n", "1", str(tag)], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise RuntimeError(f"Required protocol tag is unavailable: {tag}") from exc + if not commit: + raise RuntimeError(f"Required protocol tag has no commit: {tag}") + return commit + + +def environment_provenance(repo_root: Path) -> dict[str, Any]: + """Capture scientific runtime and solver-relevant metadata.""" + distributions = [ + "catboost", + "highspy", + "numpy", + "optbinning", + "pandas", + "pyarrow", + "scikit-learn", + "scipy", + ] + highs_env_names = [ + "HIGHS_NATIVE_FALLBACK_SCIPY", + "HIGHS_PARALLEL", + "HIGHS_PRESOLVE", + "HIGHS_RESET_GLOBAL_SCHEDULER", + "HIGHS_SIMPLEX_STRATEGY", + "HIGHS_SOLVER", + ] + lock_path = repo_root / "uv.lock" + return { + "python": sys.version, + "executable": sys.executable, + "platform": platform.platform(), + "packages": {name: package_version(name) for name in distributions}, + "solver_environment": {name: os.environ.get(name) for name in highs_env_names}, + "uv_lock_sha256": sha256_file(lock_path) if lock_path.exists() else None, + } + + +def dataframe_schema(frame: pd.DataFrame) -> dict[str, Any]: + """Return a compact row, column, and dtype descriptor.""" + return { + "rows": int(len(frame)), + "columns": int(len(frame.columns)), + "dtypes": {str(column): str(dtype) for column, dtype in frame.dtypes.items()}, + } + + +def write_csv_atomic(frame: pd.DataFrame, path: Path) -> Path: + """Write stable LF-delimited CSV through the atomic writer.""" + return atomic_write_text(path, frame.to_csv(index=False, lineterminator="\n")) + + +def save_catboost_model_atomic(model: CatBoostClassifier, path: Path) -> Path: + """Save a CatBoost model without exposing a partial target.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") + model.save_model(str(temporary), format="cbm") + temporary.replace(path) + return path diff --git a/src/utils/mlflow_tracing.py b/src/utils/mlflow_tracing.py deleted file mode 100644 index 18db545..0000000 --- a/src/utils/mlflow_tracing.py +++ /dev/null @@ -1,195 +0,0 @@ -"""MLflow 3 helpers — Datasets API + Tracing + structured tags. - -This module is purely **additive**: existing scripts that call ``mlflow.log_*`` -directly keep working. The helpers below give new code a thin, opinionated -wrapper that: - -* registers an input parquet as a hashed :class:`mlflow.data.Dataset` so runs - remain traceable to bytes-exact files (``digest`` is a SHA256 of the file - contents); -* exposes ``@trace`` as a decorator that no-ops when MLflow tracing is - unavailable (e.g. running offline on a developer machine); -* applies the same ``paper.*`` tag schema CRPTO uses for the frozen champion - (``run_tag``, ``section``, ``policy``) so dashboards stay coherent across - experiments. - -Nothing here touches the champion artefacts: every function logs to whatever -MLflow tracking URI is configured (DagsHub by default — see ``.env.example``). -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable, Iterator, Mapping -from contextlib import contextmanager -from pathlib import Path -from typing import Any, TypeVar, cast - -T = TypeVar("T", bound=Callable[..., Any]) - -# We deliberately type ``mlflow`` as ``Any`` so the runtime fallback when the -# package is missing does not confuse type-checkers. The presence flag below is -# the single source of truth for code paths. -mlflow: Any -PandasDataset: Any -try: # pragma: no cover — exercised only when mlflow is installed. - import mlflow as _mlflow - from mlflow.data.pandas_dataset import PandasDataset as _PandasDataset - - mlflow = _mlflow - PandasDataset = _PandasDataset - _HAS_MLFLOW = True -except ImportError: # pragma: no cover - mlflow = None - PandasDataset = None - _HAS_MLFLOW = False - - -# --------------------------------------------------------------------------- -# Hashing & dataset registration -# --------------------------------------------------------------------------- - - -def _sha256_of_file(path: Path, chunk_size: int = 1 << 16) -> str: - h = hashlib.sha256() - with path.open("rb") as fh: - for chunk in iter(lambda: fh.read(chunk_size), b""): - h.update(chunk) - return h.hexdigest() - - -def register_parquet_dataset( - parquet_path: str | Path, - *, - name: str | None = None, - targets: str | None = None, - context: str = "training", -) -> Any: - """Log a parquet input to the active MLflow run with a SHA256 ``digest``. - - Returns the dataset object (or ``None`` if MLflow is not installed) so - callers can attach it to ``mlflow.evaluate``. - """ - if not _HAS_MLFLOW: - return None - import pandas as pd # local import keeps base deps light. - - path = Path(parquet_path) - if not path.is_file(): - raise FileNotFoundError(path) - digest = _sha256_of_file(path)[:16] - df = pd.read_parquet(path) - dataset = mlflow.data.from_pandas( - df, - source=str(path), - name=name or path.stem, - digest=digest, - targets=targets, - ) - mlflow.log_input(dataset, context=context) - return dataset - - -# --------------------------------------------------------------------------- -# Tracing (MLflow 3) — no-op fallback when not available -# --------------------------------------------------------------------------- - - -def trace(name: str | None = None) -> Callable[[T], T]: - """Decorator that wraps a function in an ``mlflow.trace`` span when possible.""" - - def decorator(fn: T) -> T: - if not _HAS_MLFLOW or not hasattr(mlflow, "trace"): - return fn - try: # pragma: no cover — depends on installed MLflow version. - span_name = name or getattr(fn, "__name__", "trace") - wrapped = mlflow.trace(name=span_name)(fn) - return cast(T, wrapped) - except Exception: - return fn - - return decorator - - -# --------------------------------------------------------------------------- -# Tag conventions -# --------------------------------------------------------------------------- - -PAPER_RUN_TAG = "ijds-rebaseline-2026-06-07" - -# Paper run tags are immutable once published. Allow only the canonical tag -# by default; callers performing an authorised revalidation must opt in. -_PROTECTED_PAPER_RUN_TAGS = frozenset({PAPER_RUN_TAG, "paper-thesis-final-economic-2026-04-06"}) - - -def set_paper_tags( - *, - run_tag: str = PAPER_RUN_TAG, - section: str | None = None, - policy: str | None = None, - extra: Mapping[str, Any] | None = None, - allow_new_run_tag: bool = False, -) -> None: - """Apply the CRPTO ``paper.*`` tag schema to the currently active run. - - Falls back to ``mlflow.log_param`` when ``set_tag`` is not available - (older client versions) and silently no-ops when MLflow is missing. - - Args: - allow_new_run_tag: Must be True when ``run_tag`` differs from the - canonical :data:`PAPER_RUN_TAG`. Prevents accidental mislabelling - of revalidation runs as the frozen paper run. - """ - if not _HAS_MLFLOW: - return - if run_tag not in _PROTECTED_PAPER_RUN_TAGS and not allow_new_run_tag: - raise ValueError( - f"run_tag={run_tag!r} is not the canonical paper tag " - f"({PAPER_RUN_TAG!r}). Pass allow_new_run_tag=True only when " - f"intentionally starting a revalidation cohort under a new tag." - ) - tags: dict[str, Any] = {"paper.run_tag": run_tag} - if section is not None: - tags["paper.section"] = section - if policy is not None: - tags["paper.policy"] = policy - if extra: - for k, v in extra.items(): - tags[f"paper.{k}"] = v - try: - mlflow.set_tags(tags) - except Exception: # pragma: no cover - for k, v in tags.items(): - mlflow.log_param(k, v) - - -# --------------------------------------------------------------------------- -# Context manager for a tagged run -# --------------------------------------------------------------------------- - - -@contextmanager -def paper_run( - run_name: str, - *, - section: str | None = None, - policy: str | None = None, - run_tag: str = PAPER_RUN_TAG, - nested: bool = False, - allow_new_run_tag: bool = False, -) -> Iterator[Any]: - """Start an MLflow run with the CRPTO paper tag schema applied. - - Yields the active run (or ``None`` when MLflow is unavailable). - """ - if not _HAS_MLFLOW: - yield None - return - with mlflow.start_run(run_name=run_name, nested=nested) as run: - set_paper_tags( - run_tag=run_tag, - section=section, - policy=policy, - allow_new_run_tag=allow_new_run_tag, - ) - yield run diff --git a/src/utils/optuna_storage.py b/src/utils/optuna_storage.py deleted file mode 100644 index 0a3b6c4..0000000 --- a/src/utils/optuna_storage.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Optuna 4 storage helpers — JournalStorage with a SQLite fallback. - -CRPTO sweeps (portfolio bound-aware search with 276k trials, conformal alpha -sweeps, CatBoost HPO) outgrew the default SQLite storage because writes become -the bottleneck. Optuna 4 ships a journal-based storage that is much faster for -write-heavy workloads. - -Usage:: - - from src.utils.optuna_storage import make_study - - study = make_study( - name="pd_catboost_hpo", - direction="maximize", - sampler="tpe", - ) - -The helper resolves the storage path from the ``OPTUNA_STORAGE`` env variable -when set, otherwise falls back to a journal file under -``data/processed/optuna/.log``. SQLite remains available via the -``backend="sqlite"`` keyword. -""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Literal - -import optuna -from loguru import logger - -DEFAULT_JOURNAL_DIR = Path("data/processed/optuna") - - -def _resolve_journal_path(study_name: str, directory: Path | None = None) -> Path: - base = directory if directory is not None else DEFAULT_JOURNAL_DIR - base.mkdir(parents=True, exist_ok=True) - return base / f"{study_name}.log" - - -_JOURNAL_URL_PREFIXES = ("journal+file:", "journalfile:", "journal:") - - -def _is_journal_url(url: str) -> bool: - return url.lower().startswith(_JOURNAL_URL_PREFIXES) - - -def _journal_path_from_url(url: str) -> str: - """Strip the ``journal:`` prefix from a URL, returning the bare filesystem path.""" - for prefix in _JOURNAL_URL_PREFIXES: - if url.lower().startswith(prefix): - value = url[len(prefix) :] - if value.startswith("///"): - return "/" + value[3:] - return value - return url - - -def make_storage( - study_name: str, - *, - backend: Literal["journal", "sqlite", "auto"] = "auto", - directory: Path | None = None, -) -> optuna.storages.BaseStorage: - """Return an Optuna storage backend appropriate for the current environment. - - Resolution order when ``backend="auto"``: - - 1. If ``OPTUNA_STORAGE`` is set and starts with ``journal:`` (or - ``journal+file:``, ``journalfile:``), use :class:`JournalFileStorage`. - 2. Else if ``OPTUNA_STORAGE`` is set with any RDB URL, hand it to - :class:`optuna.storages.RDBStorage` directly so callers can configure - SQLite/PostgreSQL/MySQL via env var without code changes. - 3. Otherwise, fall back to JournalStorage with a per-study log under - ``data/processed/optuna/.log``. - """ - env_url = os.environ.get("OPTUNA_STORAGE", "").strip() - if backend == "auto" and env_url: - if _is_journal_url(env_url): - journal_path = Path(_journal_path_from_url(env_url)) - journal_path.parent.mkdir(parents=True, exist_ok=True) - logger.info("Using OPTUNA_STORAGE journal file: {}", journal_path) - return _make_journal_storage(journal_path, study_name=study_name) - logger.info("Using OPTUNA_STORAGE RDB URL: {}", env_url) - return optuna.storages.RDBStorage(env_url) - - if backend == "sqlite": - url = env_url or f"sqlite:///data/processed/optuna/{study_name}.db" - Path(url.replace("sqlite:///", "")).parent.mkdir(parents=True, exist_ok=True) - return optuna.storages.RDBStorage(url) - - journal_path = _resolve_journal_path(study_name, directory=directory) - return _make_journal_storage(journal_path, study_name=study_name) - - -def _make_journal_storage(journal_path: Path, *, study_name: str) -> optuna.storages.BaseStorage: - # Optuna 4 deprecated ``JournalFileStorage`` in favour of - # ``optuna.storages.journal.JournalFileBackend``. On Windows, the default - # symlink-based lock fails without admin privileges; use - # ``JournalFileOpenLock`` (open-with-O_EXCL) when available. - try: - from optuna.storages import JournalStorage - from optuna.storages.journal import JournalFileBackend - except ImportError: - try: - # Optuna <4 fallback (legacy import path) - from optuna.storages import ( - JournalFileStorage as JournalFileBackend, - JournalStorage, - ) - except ImportError: # pragma: no cover — only fires on very old Optuna. - logger.warning("Optuna JournalStorage unavailable; falling back to SQLite.") - return make_storage(study_name, backend="sqlite") - - lock_obj: object | None = None - try: - from optuna.storages.journal import JournalFileOpenLock - - lock_obj = JournalFileOpenLock(str(journal_path)) - except ImportError: - lock_obj = None - - if lock_obj is not None: - backend_file = JournalFileBackend(str(journal_path), lock_obj=lock_obj) - else: - backend_file = JournalFileBackend(str(journal_path)) - return JournalStorage(backend_file) - - -def make_study( - *, - name: str, - direction: Literal["maximize", "minimize"] = "maximize", - sampler: Literal["tpe", "nsgaii", "random"] = "tpe", - pruner: Literal["median", "wilcoxon", "none"] = "median", - seed: int = 42, - backend: Literal["journal", "sqlite", "auto"] = "auto", - load_if_exists: bool = True, -) -> optuna.Study: - """Create or load an Optuna study with sensible CRPTO defaults.""" - if sampler == "tpe": - sampler_obj: optuna.samplers.BaseSampler = optuna.samplers.TPESampler(seed=seed) - elif sampler == "nsgaii": - sampler_obj = optuna.samplers.NSGAIISampler(seed=seed) - else: - sampler_obj = optuna.samplers.RandomSampler(seed=seed) - - if pruner == "median": - pruner_obj: optuna.pruners.BasePruner = optuna.pruners.MedianPruner() - elif pruner == "wilcoxon": - wilcoxon_cls = getattr(optuna.pruners, "WilcoxonPruner", None) - pruner_obj = wilcoxon_cls() if wilcoxon_cls else optuna.pruners.MedianPruner() - else: - pruner_obj = optuna.pruners.NopPruner() - - storage = make_storage(name, backend=backend) - return optuna.create_study( - study_name=name, - direction=direction, - sampler=sampler_obj, - pruner=pruner_obj, - storage=storage, - load_if_exists=load_if_exists, - ) diff --git a/src/utils/pipeline_runtime.py b/src/utils/pipeline_runtime.py index 0560bd1..c55d1e3 100644 --- a/src/utils/pipeline_runtime.py +++ b/src/utils/pipeline_runtime.py @@ -30,6 +30,14 @@ def atomic_write_json(path: str | Path, payload: dict[str, Any]) -> Path: ) +def atomic_write_strict_json(path: str | Path, payload: dict[str, Any]) -> Path: + """Atomically write portable JSON without coercing unsupported values.""" + return atomic_write_text( + path, + json.dumps(payload, indent=2, ensure_ascii=False, allow_nan=False) + "\n", + ) + + def atomic_write_pickle(path: str | Path, obj: Any) -> Path: target = Path(path) target.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/utils/pipeline_state.py b/src/utils/pipeline_state.py deleted file mode 100644 index 57039b8..0000000 --- a/src/utils/pipeline_state.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Aggregator over the many ``models/*_status.json`` files. - -The CRPTO pipeline writes one JSON per stage (conformal policy, fairness audit, -SPO comparison, alpha sweep, paper figures, ...). Twenty-plus files end up in -``models/`` with different schemas and overlapping concerns. This module exposes -a single namespaced view that helps both Quarto chapters and Claude Code skills -locate the right slice of state without hard-coding paths in every chunk. - -The default read path is safe against a frozen champion build. The writer is -additive for new stage statuses and refuses protected champion status files -unless the caller opts in explicitly. - -Example: - >>> from src.utils.pipeline_state import load_pipeline_state - >>> state = load_pipeline_state() - >>> state["conformal"]["policy"]["coverage"]["target"] - 0.9 - >>> state["paper"]["evidence"]["status"] - 'pass' - -Reads ``EXTRACTION_MANIFEST.json`` plus every ``models/*_status.json`` that -exists. Missing files are reported under ``state["_missing"]`` rather than -raising — chapters often run in partial environments and should degrade -gracefully. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -DEFAULT_REPO_ROOT = Path(__file__).resolve().parents[2] - - -# Namespaced map: target_namespace -> filename(s). ``None`` means optional. -# The first match wins; later entries are tried only if the earlier ones are absent. -_STATE_LAYOUT: dict[tuple[str, ...], list[str]] = { - ("models", "pd"): ["pd_training_status.json"], - ("conformal", "policy"): ["conformal_policy_status.json"], - ("conformal", "variant"): ["conformal_variant_selection_status.json"], - ("conformal", "sensitivity"): ["conformal_policy_sensitivity_status.json"], - ("conformal", "width_attribution"): ["pd_conformal_width_attribution_status.json"], - ("conformal", "cqr"): ["cqr_comparison_status.json"], - ("conformal", "cqr_mondrian"): ["cqr_mondrian_status.json"], - ("conformal", "uncertainty_baselines"): ["uncertainty_baselines_status.json"], - ("conformal", "alpha_sweep"): ["alpha_sweep_status.json"], - ("portfolio", "optimization"): ["portfolio_optimization_runtime_status.json"], - ("portfolio", "tradeoff"): ["portfolio_tradeoff_runtime_status.json"], - ("portfolio", "ab_attribution"): ["ab_attribution_status.json"], - ("portfolio", "spo_real"): ["spo_real_training_status.json"], - ("portfolio", "spo_comparison"): ["spo_comparison_status.json"], - ("fairness", "audit"): ["fairness_audit_status.json"], - ("fairness", "fairlearn"): ["fairlearn_fairness_status.json"], - ("governance", "general"): ["governance_status.json"], - ("governance", "mrm"): ["mrm_report_status.json"], - ("paper", "promotion"): ["final_project_promotion.json"], - ("paper", "evidence"): ["crpto_evidence_status.json"], - ("paper", "journal_package"): ["crpto_journal_package_status.json"], - ("paper", "figures"): ["paper_figures_status.json"], -} - - -@dataclass -class PipelineState: - """Container with the merged state plus diagnostics on what was missing.""" - - state: dict[str, Any] = field(default_factory=dict) - missing: list[str] = field(default_factory=list) - manifest: dict[str, Any] = field(default_factory=dict) - repo_root: Path = field(default_factory=lambda: DEFAULT_REPO_ROOT) - - def get(self, *keys: str, default: Any = None) -> Any: - """Tunnel into the namespaced state, returning ``default`` for any missing key.""" - node: Any = self.state - for k in keys: - if isinstance(node, Mapping) and k in node: - node = node[k] - else: - return default - return node - - def to_dict(self) -> dict[str, Any]: - """Plain-dict snapshot useful for ``mlflow.log_dict`` or Quarto chunks.""" - return { - "state": self.state, - "missing": list(self.missing), - "manifest": self.manifest, - "repo_root": str(self.repo_root), - } - - -def _load_json(path: Path) -> Any: - try: - return json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as err: - return {"_parse_error": str(err), "_path": str(path)} - - -# Reverse map for the writer: namespace -> canonical filename. The first filename -# of each ``_STATE_LAYOUT`` entry is the writer target so the loader can read it -# back round-trip. -_WRITE_TARGETS: dict[tuple[str, ...], str] = {ns: files[0] for ns, files in _STATE_LAYOUT.items()} - -_PROTECTED_WRITE_TARGETS = { - "final_project_promotion.json", - "pd_training_status.json", - "conformal_policy_status.json", -} - - -def _deep_merge(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]: - merged = dict(left) - for key, value in right.items(): - previous = merged.get(key) - if isinstance(previous, dict) and isinstance(value, dict): - merged[key] = _deep_merge(previous, value) - else: - merged[key] = value - return merged - - -def write_pipeline_state( - namespace: str | tuple[str, ...], - payload: dict[str, Any], - *, - repo_root: Path | str | None = None, - models_dir: str = "models", - merge: bool = False, - allow_protected: bool = False, -) -> Path: - """Write a status JSON for ``namespace`` and return the path it landed on. - - Use either a tuple (``("conformal", "policy")``) or the slash form - (``"conformal/policy"``). Unknown namespaces fall back to - ``models/_status.json`` so callers don't have to declare ahead. - - Args: - namespace: Identifier as it would appear in :func:`load_pipeline_state`. - payload: JSON-serialisable dict to persist. - repo_root: Defaults to the repository root inferred from this file. - models_dir: Sub-directory holding status JSONs. - merge: When True and the target file exists, recursively merges - ``payload`` into the previous content. When False, overwrites. - allow_protected: Must be True to write status files tied to the frozen - champion/promotion contract. - - Returns: - The :class:`pathlib.Path` the JSON was written to. - """ - if isinstance(namespace, str): - parts = tuple(p for p in namespace.split("/") if p) - else: - parts = tuple(namespace) - if not parts: - raise ValueError("namespace must be a non-empty tuple or 'a/b' string.") - - target_name = _WRITE_TARGETS.get(parts) - if target_name is None: - target_name = "_".join(parts) + "_status.json" - - root = Path(repo_root) if repo_root else DEFAULT_REPO_ROOT - target_path = root / models_dir / target_name - if target_name in _PROTECTED_WRITE_TARGETS and not allow_protected: - raise PermissionError( - f"{target_name} is a protected CRPTO champion status. " - "Pass allow_protected=True only in an explicit revalidation workflow." - ) - target_path.parent.mkdir(parents=True, exist_ok=True) - - if merge and target_path.is_file(): - prev = _load_json(target_path) - if isinstance(prev, dict): - payload = _deep_merge(prev, payload) - - target_path.write_text( - json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n", - encoding="utf-8", - ) - return target_path - - -def load_pipeline_state( - *, - repo_root: Path | str | None = None, - models_dir: str = "models", -) -> PipelineState: - """Merge every known ``models/*_status.json`` into a namespaced dictionary. - - Args: - repo_root: Defaults to the repository root inferred from this file's path. - models_dir: Sub-directory holding the JSONs. ``"models"`` by default. - """ - root = Path(repo_root) if repo_root else DEFAULT_REPO_ROOT - models = root / models_dir - - state: dict[str, Any] = {} - missing: list[str] = [] - - for ns, files in _STATE_LAYOUT.items(): - loaded: Any = None - for fname in files: - candidate = models / fname - if candidate.is_file(): - loaded = _load_json(candidate) - break - if loaded is None: - missing.append("/".join(ns)) - continue - _assign(state, ns, loaded) - - manifest_path = root / "EXTRACTION_MANIFEST.json" - manifest: dict[str, Any] = {} - if manifest_path.is_file(): - parsed = _load_json(manifest_path) - if isinstance(parsed, dict): - manifest = parsed - - return PipelineState(state=state, missing=missing, manifest=manifest, repo_root=root) - - -def _assign(target: dict[str, Any], path: tuple[str, ...], value: Any) -> None: - node = target - for k in path[:-1]: - node = node.setdefault(k, {}) - node[path[-1]] = value diff --git a/src/utils/pipeline_topology.py b/src/utils/pipeline_topology.py deleted file mode 100644 index 59f8809..0000000 --- a/src/utils/pipeline_topology.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Declarative pipeline-family and profile registry helpers. - -This module centralizes the semantic topology used by the orchestration -wrappers. The goal is to keep the new pipeline-first contract in tracked YAML -files instead of hardcoding family semantics inside the compatibility launcher. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] -PIPELINE_CONFIG_DIR = REPO_ROOT / "configs" / "pipelines" -PROFILE_CONFIG_DIR = REPO_ROOT / "configs" / "profiles" -LEGACY_PIPELINE_ALIASES = { - "canonical_rebuild": "core_canonical", - "champion_search": "search_pd", - "challenger_promotion": "search_pd", -} - -LEGACY_PROFILE_DEFAULTS = { - "canonical_rebuild": "canonical_operational", - "champion_search": "champion_search_max", - "challenger_promotion": "canonical_monotonic_promotion_full", -} - -FAMILY_DEFAULTS = { - "core_canonical": "core_canonical_cpu", - "search_pd": "search_pd_default", - "search_conformal": "search_conformal_default", - "search_portfolio": "search_portfolio_default", - "crpto_e2e": "crpto_e2e_default", - "diagnostics_governance": "diagnostics_governance_default", -} - - -def _load_yaml(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except Exception: - return {} - return dict(payload) if isinstance(payload, dict) else {} - - -def resolve_pipeline_family(name: str | None) -> str: - raw = str(name or "").strip().lower() - if not raw: - return "crpto_e2e" - return LEGACY_PIPELINE_ALIASES.get(raw, raw) - - -def legacy_family_alias(name: str | None) -> str | None: - raw = str(name or "").strip().lower() - if not raw: - return None - return raw if raw in LEGACY_PIPELINE_ALIASES else None - - -def load_pipeline_definition(name: str | None) -> dict[str, Any]: - family = resolve_pipeline_family(name) - payload = _load_yaml(PIPELINE_CONFIG_DIR / f"{family}.yaml") - if not payload: - return { - "schema_version": "2026-03-31.1", - "pipeline_family": family, - "description": "", - "artifact_scope": "research", - "promotion_state": "research_open", - "writes_canonical_artifacts": False, - "upstream_required": False, - "default_profile": FAMILY_DEFAULTS.get(family, "search_pd_default"), - "allowed_step_groups": ["preflight"], - "forbidden_step_groups": [], - "required_envs": [], - "produced_registries": [], - "papers_served": [], - } - payload["pipeline_family"] = family - if "default_profile" not in payload: - payload["default_profile"] = FAMILY_DEFAULTS.get(family, "search_pd_default") - return payload - - -def default_profile_for_family(name: str | None, sampling_profile: str | None = None) -> str: - raw = str(name or "").strip().lower() - if raw in LEGACY_PROFILE_DEFAULTS: - return LEGACY_PROFILE_DEFAULTS[raw] - family = resolve_pipeline_family(raw) - default = FAMILY_DEFAULTS.get(family) - if default: - return default - return str(sampling_profile or "").strip() or "search_pd_default" - - -def load_profile_config(profile_name: str | None) -> dict[str, Any]: - name = str(profile_name or "").strip() - if not name: - return {} - return _load_yaml(PROFILE_CONFIG_DIR / f"{name}.yaml") - - -def build_pipeline_contract( - *, - pipeline_family: str, - pipeline_profile_arg: str | None, - sampling_profile: str, - writes_canonical_artifacts_arg: bool | None, - upstream_canonical_run_tag: str | None, -) -> dict[str, Any]: - requested_family = str(pipeline_family).strip().lower() or "crpto_e2e" - family = resolve_pipeline_family(requested_family) - pipeline_def = load_pipeline_definition(family) - pipeline_profile = str(pipeline_profile_arg or "").strip() or default_profile_for_family( - requested_family, sampling_profile - ) - writes_canonical_artifacts = bool(pipeline_def.get("writes_canonical_artifacts", False)) - if writes_canonical_artifacts_arg is not None: - writes_canonical_artifacts = bool(writes_canonical_artifacts_arg) - return { - "requested_pipeline_family": requested_family, - "legacy_pipeline_alias": legacy_family_alias(requested_family), - "pipeline_family": family, - "pipeline_profile": pipeline_profile, - "description": str(pipeline_def.get("description", "") or ""), - "artifact_scope": str(pipeline_def.get("artifact_scope", "research") or "research"), - "promotion_state": str( - pipeline_def.get("promotion_state", "research_open") or "research_open" - ), - "writes_canonical_artifacts": writes_canonical_artifacts, - "upstream_required": bool(pipeline_def.get("upstream_required", False)), - "allowed_step_groups": list(pipeline_def.get("allowed_step_groups", []) or []), - "forbidden_step_groups": list(pipeline_def.get("forbidden_step_groups", []) or []), - "required_envs": list(pipeline_def.get("required_envs", []) or []), - "produced_registries": list(pipeline_def.get("produced_registries", []) or []), - "papers_served": list(pipeline_def.get("papers_served", []) or []), - "upstream_canonical_run_tag": upstream_canonical_run_tag, - } diff --git a/src/utils/replay_manifest.py b/src/utils/replay_manifest.py deleted file mode 100644 index 22182b1..0000000 --- a/src/utils/replay_manifest.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Helpers for frozen baseline replay manifests.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[2] -DEFAULT_MANIFEST_PATH = ROOT / "configs" / "baselines" / "clean_baseline_manifest.json" - - -def resolve_manifest_path(path_like: str | Path | None = None) -> Path: - if path_like is None: - return DEFAULT_MANIFEST_PATH - path = Path(path_like) - if not path.is_absolute(): - path = ROOT / path - return path.resolve() - - -def load_replay_manifest(path_like: str | Path | None = None) -> dict[str, Any]: - path = resolve_manifest_path(path_like) - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def save_replay_manifest(payload: dict[str, Any], path_like: str | Path | None = None) -> Path: - path = resolve_manifest_path(path_like) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - return path - - -def manifest_section( - manifest: dict[str, Any] | None, - section: str, -) -> dict[str, Any]: - if not isinstance(manifest, dict): - return {} - payload = manifest.get(section, {}) - return dict(payload) if isinstance(payload, dict) else {} - - -def sha256_for_path(path_like: str | Path) -> str | None: - path = Path(path_like) - if not path.is_absolute(): - path = ROOT / path - if not path.exists() or not path.is_file(): - return None - digest = hashlib.sha256() - with open(path, "rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def artifact_descriptor(path_like: str | Path) -> dict[str, Any]: - path = Path(path_like) - if not path.is_absolute(): - path = ROOT / path - rel = ( - path.relative_to(ROOT).as_posix() - if path.is_absolute() and ROOT in path.parents - else Path(path_like).as_posix() - ) - return { - "path": rel, - "exists": bool(path.exists()), - "sha256": sha256_for_path(path), - } diff --git a/src/utils/script_helpers.py b/src/utils/script_helpers.py deleted file mode 100644 index 72656e9..0000000 --- a/src/utils/script_helpers.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Shared helpers for the publication/orchestration scripts under ``scripts/``. - -These consolidate the small ``_load_json`` / ``_write_json`` / ``_write_table`` -style functions that were previously copy-pasted across the paper-facing -scripts. Two properties are contractual: - -1. **Byte-stable output on every OS.** JSON and table writers always emit - LF line endings (``newline=""``) so regenerated artifacts stay bit-exact - with the hashes recorded in ``EXTRACTION_MANIFEST.json`` regardless of - platform. Writing with platform-default newlines on Windows produces CRLF - files that break ``just validate-champion``. -2. **Idempotent writes.** ``write_table`` compares against the existing file - and skips the write when the bytes are unchanged, preserving mtimes and - keeping DVC/freeze caches quiet. - -Scripts that are dependencies of protected DVC stages (``train_pd_model.py``, -``optimize_portfolio.py``, ``generate_conformal_intervals.py``, -``validate_conformal_policy.py``, ``run_portfolio_bound_exact_eval.py``) keep -their local copies until the champion-touching refactor lane migrates them -behind a drift gate. -""" - -from __future__ import annotations - -import json -import math -import os -from pathlib import Path -from typing import Any, cast - -import numpy as np -import pandas as pd -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def load_json(path: Path) -> dict[str, Any]: - """Load a UTF-8 JSON file into a dict.""" - return cast(dict[str, Any], json.loads(path.read_text(encoding="utf-8"))) - - -def try_load_json(path: Path) -> dict[str, Any]: - """Load a UTF-8 JSON file, returning ``{}`` when the file does not exist.""" - if not path.exists(): - return {} - return load_json(path) - - -def load_yaml(path: Path) -> dict[str, Any]: - """Load a UTF-8 YAML file into a dict.""" - return cast(dict[str, Any], yaml.safe_load(path.read_text(encoding="utf-8"))) - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - """Write ``payload`` as indented, key-sorted JSON with LF line endings.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - newline="", - ) - - -def write_table( - name: str, - frame: pd.DataFrame, - *, - table_dir: Path, - root: Path = REPO_ROOT, - float_precision: int = 6, -) -> list[Path]: - """Write a publication table as ``.csv`` and ``.tex``. - - Output is LF-only and the write is skipped when the on-disk bytes already - match, so frozen tables under ``reports/crpto/tables`` keep their manifest - hashes when regenerated from unchanged inputs. - """ - table_dir = table_dir.resolve() - root = root.resolve() - table_dir.mkdir(parents=True, exist_ok=True) - csv_path = table_dir / f"{name}.csv" - tex_path = table_dir / f"{name}.tex" - csv_text = frame.to_csv(index=False, lineterminator="\n") - tex_text = frame.to_latex( - index=False, - escape=True, - float_format=lambda value: f"{value:.{float_precision}f}", - ) - for path, text in [(csv_path, csv_text), (tex_path, tex_text)]: - if path.exists() and path.read_bytes().decode("utf-8") == text: - continue - path.write_text(text, encoding="utf-8", newline="") - print(f"Wrote {csv_path.relative_to(root).as_posix()}") # noqa: T201 - print(f"Wrote {tex_path.relative_to(root).as_posix()}") # noqa: T201 - return [csv_path, tex_path] - - -def artifact_path(path_like: str | Path) -> Path: - """Resolve a relative artifact path under ``GPU_REPLAY_ARTIFACT_ROOT`` if set.""" - path = Path(path_like) - root = str(os.environ.get("GPU_REPLAY_ARTIFACT_ROOT", "")).strip() - return (Path(root) / path) if root else path - - -def resolve_repo_artifact_path( - path_like: str | Path, - *, - root: Path = REPO_ROOT, -) -> Path: - """Resolve relative or foreign-OS paths that point inside this repository. - - Experiment manifests may have been written from WSL and therefore contain - ``/mnt/c/...//...`` paths. The artifact identity is repository-relative, - so a native Windows replay should resolve the suffix under its current root. - Absolute paths outside this repository are left unchanged. - """ - path = Path(path_like) - if path.exists(): - return path.resolve() - - normalized_parts = [part for part in str(path_like).replace("\\", "/").split("/") if part] - root_name = root.name.casefold() - matching_indices = [ - index for index, part in enumerate(normalized_parts) if part.casefold() == root_name - ] - if matching_indices: - suffix = normalized_parts[matching_indices[-1] + 1 :] - return root.joinpath(*suffix) - if not path.is_absolute(): - return root / path - return path - - -def first_existing(*paths: Path) -> Path: - """Return the first existing path, falling back to the last candidate.""" - if not paths: - raise ValueError("first_existing requires at least one candidate path") - for path in paths: - if path.exists(): - return path - return paths[-1] - - -def parse_percent_series(series: pd.Series, *, nan_percent: float = 12.0) -> np.ndarray: - """Convert a percent-scale column (numeric or ``'12.5%'`` strings) to decimals. - - Missing values are imputed with ``nan_percent`` (in percent units) before - dividing by 100. - """ - if pd.api.types.is_numeric_dtype(series): - values = pd.to_numeric(series, errors="coerce").to_numpy(dtype=float) - else: - values = ( - series.astype(str) - .str.strip() - .str.rstrip("%") - .pipe(pd.to_numeric, errors="coerce") - .to_numpy(dtype=float) - ) - values = np.nan_to_num(values, nan=nan_percent) - return cast(np.ndarray, values / 100.0) - - -def resolve_interval_columns(intervals: pd.DataFrame) -> tuple[str, str, str]: - """Resolve the point/low/high column names of a conformal-interval frame.""" - col_point = "y_pred" if "y_pred" in intervals.columns else "pd_point" - col_low = "pd_low_90" if "pd_low_90" in intervals.columns else "pd_low" - col_high = "pd_high_90" if "pd_high_90" in intervals.columns else "pd_high" - return col_point, col_low, col_high - - -# Full policy identity used by the promotion artifacts; scripts that compare -# against partial metric dicts pass a narrower field tuple instead. -POLICY_MATCH_FIELDS = ( - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - "min_budget_utilization", - "pd_cap_slack_penalty", -) - - -def policy_matches( - row: Any, - policy: dict[str, Any], - fields: tuple[str, ...] = POLICY_MATCH_FIELDS, - *, - atol: float = 1e-9, -) -> bool: - """Return True when ``row`` and ``policy`` agree on every policy field. - - ``row`` may be a pandas Series or a plain mapping. String fields compare - by string equality; numeric fields by absolute tolerance. A field missing - on either side, or a non-coercible numeric value, is a mismatch. - """ - for field in fields: - if field not in row or field not in policy: - return False - left = row[field] - right = policy[field] - if isinstance(right, str): - if str(left) != right: - return False - continue - try: - left_f = float(left) - right_f = float(right) - except (TypeError, ValueError): - return False - if math.isnan(left_f) or math.isnan(right_f) or abs(left_f - right_f) > atol: - return False - return True diff --git a/src/utils/threshold_semantics.py b/src/utils/threshold_semantics.py deleted file mode 100644 index 06a54e4..0000000 --- a/src/utils/threshold_semantics.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Shared helpers for canonical threshold semantics across artifacts and UI.""" - -from __future__ import annotations - -import json -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -DEFAULT_PATH = Path("models/threshold_semantics.json") -SCHEMA_VERSION = "2026-03-13.1" - -DEFAULT_BUSINESS_MEANING = { - "pd_internal_selected_threshold": ( - "Threshold interno de screening/seleccion PD usado en busqueda y analisis tecnico." - ), - "pd_internal_fallback_threshold": ( - "Fallback interno tomado de la policy de fairness durante la busqueda de threshold PD." - ), - "fairness_primary_threshold": ( - "Threshold operativo principal para auditoria de fairness y narrativa de aprobacion." - ), - "decision_policy_global_threshold": ( - "Threshold operativo global aplicado por la policy de decision/aprobacion." - ), -} - - -def _safe_float(value: Any) -> float | None: - try: - out = float(value) - except (TypeError, ValueError): - return None - return out - - -def load_threshold_semantics(path: str | Path = DEFAULT_PATH) -> dict[str, Any]: - target = Path(path) - if not target.exists(): - return {} - try: - payload = json.loads(target.read_text(encoding="utf-8")) - except Exception: - return {} - return payload if isinstance(payload, dict) else {} - - -def write_threshold_semantics( - *, - pd_internal_selected_threshold: float | None = None, - pd_internal_fallback_threshold: float | None = None, - fairness_primary_threshold: float | None = None, - decision_policy_global_threshold: float | None = None, - source_artifacts: dict[str, str] | None = None, - business_meaning: dict[str, str] | None = None, - run_tag: str | None = None, - path: str | Path = DEFAULT_PATH, - extra: dict[str, Any] | None = None, -) -> dict[str, Any]: - target = Path(path) - payload = load_threshold_semantics(target) - resolved_run_tag = str(run_tag or payload.get("run_tag") or "untracked").strip() or "untracked" - current_sources = payload.get("source_artifacts", {}) if isinstance(payload, dict) else {} - merged_sources = { - **(current_sources if isinstance(current_sources, dict) else {}), - **(source_artifacts or {}), - } - meanings = { - **DEFAULT_BUSINESS_MEANING, - **( - payload.get("business_meaning", {}) - if isinstance(payload.get("business_meaning", {}), dict) - else {} - ), - **(business_meaning or {}), - } - - resolved = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_tag": resolved_run_tag, - "pd_internal_selected_threshold": _safe_float( - pd_internal_selected_threshold - if pd_internal_selected_threshold is not None - else payload.get("pd_internal_selected_threshold") - ), - "pd_internal_fallback_threshold": _safe_float( - pd_internal_fallback_threshold - if pd_internal_fallback_threshold is not None - else payload.get("pd_internal_fallback_threshold") - ), - "fairness_primary_threshold": _safe_float( - fairness_primary_threshold - if fairness_primary_threshold is not None - else payload.get("fairness_primary_threshold") - ), - "decision_policy_global_threshold": _safe_float( - decision_policy_global_threshold - if decision_policy_global_threshold is not None - else payload.get("decision_policy_global_threshold") - ), - "source_artifacts": merged_sources, - "business_meaning": meanings, - } - if extra: - resolved.update(extra) - - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(resolved, indent=2, ensure_ascii=False), encoding="utf-8") - return resolved - - -def resolve_operational_threshold( - semantics: dict[str, Any] | None = None, - default: float = 0.5, -) -> float: - payload = semantics or {} - for key in ("decision_policy_global_threshold", "fairness_primary_threshold"): - value = _safe_float(payload.get(key)) - if value is not None: - return value - return float(default) - - -def resolve_pd_internal_threshold( - semantics: dict[str, Any] | None = None, - default: float = 0.5, -) -> float: - payload = semantics or {} - for key in ("pd_internal_selected_threshold", "pd_internal_fallback_threshold"): - value = _safe_float(payload.get(key)) - if value is not None: - return value - return float(default) - - -def optimal_threshold_cost_matrix( - y_true: Any, - y_prob: Any, - fn_cost: float = 1.0, - fp_cost: float = 0.12, - n_steps: int = 99, -) -> dict[str, Any]: - """Find classification threshold that minimises expected misclassification cost. - - In credit risk, missing a default (FN) costs LGD × EAD; rejecting a good - borrower (FP) costs foregone interest. This function sweeps thresholds and - returns the one with minimum total expected cost. - - Args: - y_true: Binary ground truth (1 = default). - y_prob: Predicted default probability. - fn_cost: Cost weight for false negatives. Typically normalised LGD × EAD. - fp_cost: Cost weight for false positives. Typically approx. interest rate. - n_steps: Number of candidate thresholds evaluated between 0.01 and 0.99. - - Returns: - Dict with optimal_threshold, min_cost, fn_cost, fp_cost, cost_at_035, - and fn_fp_ratio. - """ - import numpy as np - - y_true_arr = np.asarray(y_true, dtype=float) - y_prob_arr = np.asarray(y_prob, dtype=float) - thresholds = np.linspace(0.01, 0.99, n_steps) - costs = [] - for t in thresholds: - y_pred = (y_prob_arr >= t).astype(int) - fp = int(((y_pred == 1) & (y_true_arr == 0)).sum()) - fn = int(((y_pred == 0) & (y_true_arr == 1)).sum()) - costs.append(fn * fn_cost + fp * fp_cost) - - best_idx = int(np.argmin(costs)) - idx_035 = int(np.argmin(np.abs(thresholds - 0.35))) - return { - "optimal_threshold": float(thresholds[best_idx]), - "min_cost": float(costs[best_idx]), - "fn_cost": float(fn_cost), - "fp_cost": float(fp_cost), - "fn_fp_ratio": float(fn_cost / fp_cost) if fp_cost > 0 else None, - "cost_at_035": float(costs[idx_035]), - "threshold_at_035": 0.35, - "n_thresholds_evaluated": n_steps, - } diff --git a/src/utils/visualization.py b/src/utils/visualization.py deleted file mode 100644 index 84be781..0000000 --- a/src/utils/visualization.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Plotting utilities for credit risk analysis.""" - -from __future__ import annotations - -import matplotlib.pyplot as plt -import numpy as np - - -def plot_calibration_curve( - y_true: np.ndarray, - y_prob: np.ndarray, - n_bins: int = 10, - title: str = "Calibration Curve", - ax: plt.Axes | None = None, -) -> plt.Axes: - """Plot reliability diagram (calibration curve).""" - from sklearn.calibration import calibration_curve - - if ax is None: - _, ax = plt.subplots(1, 1, figsize=(8, 6)) - - fraction_pos, mean_pred = calibration_curve(y_true, y_prob, n_bins=n_bins) - ax.plot(mean_pred, fraction_pos, "s-", label="Model") - ax.plot([0, 1], [0, 1], "k--", label="Perfect calibration") - ax.set_xlabel("Mean predicted probability") - ax.set_ylabel("Fraction of positives") - ax.set_title(title) - ax.legend() - return ax - - -def plot_murphy_diagram( - y_true: np.ndarray, - forecasts: dict[str, np.ndarray], - n_thresholds: int = 200, - title: str = "Murphy Diagram", - ax: plt.Axes | None = None, -) -> plt.Axes: - """Plot Murphy diagram: bin-free calibration diagnostic. - - For each threshold theta in [0, 1], plots the elementary score - S_theta(p, y) = (1{y <= theta} - 1{p <= theta})^2 averaged over - all observations. A perfectly calibrated model produces a flat curve. - Regions where one model's curve lies below another indicate superiority - at that threshold. - - Args: - y_true: Binary labels (0/1). - forecasts: Dict mapping model name to predicted probabilities. - n_thresholds: Number of threshold points to evaluate. - title: Plot title. - ax: Optional axes to draw on. - - References: - Ehm et al. (2016), "Of Quantiles and Expectiles". - """ - y = np.asarray(y_true, dtype=float) - thetas = np.linspace(0.01, 0.99, n_thresholds) - - if ax is None: - _, ax = plt.subplots(1, 1, figsize=(10, 5)) - - for name, phat in forecasts.items(): - phat = np.asarray(phat, dtype=float) - scores = np.empty(len(thetas)) - for i, theta in enumerate(thetas): - indicator_y = (y <= theta).astype(float) - indicator_p = (phat <= theta).astype(float) - scores[i] = np.mean((indicator_y - indicator_p) ** 2) - ax.plot(thetas, scores, label=name, linewidth=1.5) - - ax.set_xlabel("Threshold $\\theta$") - ax.set_ylabel("Mean elementary score $S_\\theta$") - ax.set_title(title) - ax.legend() - ax.grid(True, alpha=0.3) - return ax - - -def plot_conformal_intervals( - y_true: np.ndarray, - y_pred: np.ndarray, - y_intervals: np.ndarray, - n_samples: int = 100, - title: str = "Conformal Prediction Intervals", -) -> plt.Figure: - """Plot prediction intervals with true values.""" - fig, ax = plt.subplots(figsize=(14, 6)) - idx = np.argsort(y_pred)[:n_samples] - - # Accept both MAPIE raw shape (n, 2, 1) and squeezed shape (n, 2). - if y_intervals.ndim == 3: - low = y_intervals[idx, 0, 0] - high = y_intervals[idx, 1, 0] - elif y_intervals.ndim == 2: - low = y_intervals[idx, 0] - high = y_intervals[idx, 1] - else: - raise ValueError(f"Unexpected y_intervals shape: {y_intervals.shape}") - - ax.fill_between(range(len(idx)), low, high, alpha=0.3, label="90% CI") - ax.scatter(range(len(idx)), y_true[idx], s=10, c="red", label="True", zorder=5) - ax.plot(range(len(idx)), y_pred[idx], "b-", linewidth=0.8, label="Predicted") - ax.set_xlabel("Sample (sorted by prediction)") - ax.set_ylabel("Probability of Default") - ax.set_title(title) - ax.legend() - return fig - - -def plot_portfolio_allocation( - allocation: dict[int, float], - loan_amounts: np.ndarray, - title: str = "Portfolio Allocation", -) -> plt.Figure: - """Plot portfolio allocation distribution.""" - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - - funded = [i for i, v in allocation.items() if v > 0.01] - amounts = [allocation[i] * loan_amounts[i] for i in funded] - - axes[0].hist(amounts, bins=30, edgecolor="black") - axes[0].set_title("Distribution of Funded Amounts") - axes[0].set_xlabel("Funded Amount ($)") - - axes[1].pie( - [len(funded), len(allocation) - len(funded)], - labels=[f"Funded ({len(funded)})", f"Rejected ({len(allocation) - len(funded)})"], - autopct="%1.1f%%", - ) - axes[1].set_title("Approval Rate") - - fig.suptitle(title) - fig.tight_layout() - return fig diff --git a/tests/test_configs/test_lineage_consistency.py b/tests/test_configs/test_lineage_consistency.py deleted file mode 100644 index 9700c12..0000000 --- a/tests/test_configs/test_lineage_consistency.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Anti-regression guard for the april-lineage unification (2026-06-10). - -The pre-unification PD metrics (AUC 0.7127 / Brier 0.1546 / ECE 0.0062, and -their short-form variants in figures) came from canonical retrains that never -fed the certificate. After the unification every ACTIVE narrative surface -must cite the certificate lineage (AUC 0.7139 / Brier 0.1544 / ECE 0.0070). - -This test sweeps the active paper, book, figure-source and figure-script -surfaces for the retired values so they cannot silently reappear (e.g. via a -merge of an older branch). Historical research notes under ``docs/research`` -and frozen one-shot tables are intentionally NOT scanned: they are dated -records of what was true at the time. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[2] - -# Active narrative surfaces: anything a reviewer or jury reads as current. -ACTIVE_GLOBS = ( - "paper/CRPTO_ijds.qmd", - "paper/supplement_ijds.qmd", - "paper/submission/CRPTO_ijds_submission.tex", - "book/chapters/*.qmd", - "reports/crpto/figures/*.svg", - "scripts/generate_crpto_figures.py", -) - -# Retired PD-metric strings. Tokens are specific enough to avoid collisions -# with unrelated quantities (checked at introduction time): bare "0.006"-style -# short forms are only banned in their labelled figure/text contexts. -RETIRED_TOKENS = ( - "0.7127", - "AUC 0.712,", - "AUC `0.7127`", - "ECE 0.0062", - "ECE = 0.0062", - "ECE 0.006<", # svg text node short form - "Brier `0.1546`", - "Brier score `0.1546`", -) - - -def _active_files() -> list[Path]: - files: list[Path] = [] - for pattern in ACTIVE_GLOBS: - files.extend(ROOT.glob(pattern)) - return [f for f in files if f.is_file()] - - -def test_active_surfaces_do_not_cite_retired_pd_metrics() -> None: - files = _active_files() - assert files, "Active-surface glob list resolved to nothing — fix the test." - offenders: list[str] = [] - for path in files: - text = path.read_text(encoding="utf-8", errors="ignore") - for token in RETIRED_TOKENS: - if token in text: - line_no = text[: text.index(token)].count("\n") + 1 - offenders.append(f"{path.relative_to(ROOT).as_posix()}:{line_no} -> {token!r}") - assert not offenders, ( - "Retired pre-unification PD metrics found on active surfaces " - "(certificate lineage is AUC 0.7139 / Brier 0.1544 / ECE 0.0070):\n " - + "\n ".join(offenders) - ) - - -def test_canonical_sources_carry_certificate_lineage() -> None: - """pipeline_summary and metrics_summary must agree on the unified metrics.""" - import json - - summary_path = ROOT / "data" / "processed" / "pipeline_summary.json" - metrics_path = ROOT / "reports" / "dvc" / "metrics_summary.json" - if not (summary_path.is_file() and metrics_path.is_file()): - pytest.skip("Canonical metric artifacts not available locally.") - summary = json.loads(summary_path.read_text(encoding="utf-8")) - metrics = json.loads(metrics_path.read_text(encoding="utf-8"))["metrics"] - assert summary["pd_auc"] == pytest.approx(metrics["pd.auc"], rel=1e-12) - assert summary["pd_auc"] == pytest.approx(0.713852, abs=5e-6) - assert summary["pd_brier"] == pytest.approx(0.154393, abs=5e-6) - assert summary["pd_ece"] == pytest.approx(0.006998, abs=5e-6) diff --git a/tests/test_configs/test_params_config_sync.py b/tests/test_configs/test_params_config_sync.py deleted file mode 100644 index 842cfc1..0000000 --- a/tests/test_configs/test_params_config_sync.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Guard against silent drift between ``params.yaml`` and its source configs. - -``params.yaml`` is a read-only documentary view: scripts read -``configs/crpto_*.yaml`` and ``models/final_project_promotion.json``, while -DVC uses ``params.yaml`` entries as cache keys for the protected stages. -Because the two surfaces are maintained by hand, a value edited on one side -can silently diverge on the other (this happened with -``pd.catboost.learning_rate``). These tests pin the keys that are true -mirrors. Keys that are documentary-only (``portfolio.policy_mode``, -``conformal.mondrian.partition_by``) have no canonical config counterpart -and are intentionally not asserted. - -Unifying params.yaml with the configs remains a deferred refactor (see the -header comment in params.yaml); until then this test is the sync contract. -""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest -import yaml - -ROOT = Path(__file__).resolve().parents[2] - - -def _load_yaml(path: Path) -> dict[str, Any]: - return yaml.safe_load(path.read_text(encoding="utf-8")) - - -@pytest.fixture(scope="module") -def params() -> dict[str, Any]: - return _load_yaml(ROOT / "params.yaml") - - -@pytest.fixture(scope="module") -def pd_config() -> dict[str, Any]: - return _load_yaml(ROOT / "configs" / "crpto_pd_model.yaml") - - -@pytest.fixture(scope="module") -def conformal_config() -> dict[str, Any]: - return _load_yaml(ROOT / "configs" / "crpto_conformal_policy.yaml") - - -@pytest.fixture(scope="module") -def optimization_config() -> dict[str, Any]: - return _load_yaml(ROOT / "configs" / "crpto_optimization.yaml") - - -@pytest.fixture(scope="module") -def promotion() -> dict[str, Any]: - path = ROOT / "models" / "final_project_promotion.json" - if not path.is_file(): - pytest.skip("final_project_promotion.json not available locally.") - return json.loads(path.read_text(encoding="utf-8")) - - -def test_pd_params_mirror_pd_config(params: dict[str, Any], pd_config: dict[str, Any]) -> None: - model_params = pd_config["model"]["params"] - mirrored = params["pd"]["catboost"] - assert params["pd"]["model"] == pd_config["model"]["type"] - assert mirrored["depth"] == model_params["depth"] - assert mirrored["iterations"] == model_params["iterations"] - assert mirrored["learning_rate"] == pytest.approx(model_params["learning_rate"], rel=1e-12) - assert params["pd"]["calibration"] == pd_config["calibration"]["method"] - - -def test_conformal_coverage_targets_mirror_policy_config( - params: dict[str, Any], conformal_config: dict[str, Any] -) -> None: - targets = params["conformal"]["coverage_targets"] - policy = conformal_config["policy"] - assert targets == [ - pytest.approx(policy["target_coverage_90_min"]), - pytest.approx(policy["target_coverage_95_min"]), - ] - - -def test_portfolio_params_mirror_optimization_config( - params: dict[str, Any], optimization_config: dict[str, Any] -) -> None: - portfolio = optimization_config["portfolio"] - assert params["portfolio"]["max_concentration"] == pytest.approx(portfolio["max_concentration"]) - assert params["portfolio"]["max_portfolio_pd"] == pytest.approx(portfolio["max_portfolio_pd"]) - - -def test_champion_params_mirror_promotion_artifact( - params: dict[str, Any], promotion: dict[str, Any] -) -> None: - champion = promotion["final_champion"] - assert params["paper"]["run_tag"] == promotion["run_tag"] - assert params["champion"]["v_alpha_001"] == pytest.approx( - champion["alpha01_weighted_miscoverage_V"], abs=1e-12 - ) - assert params["champion"]["gamma_cp_alpha_001"] == pytest.approx( - champion["alpha01_gamma_cp"], abs=1e-12 - ) - assert params["champion"]["alpha_exact_pass"] == bool(champion["alpha01_exact_pass"]) - # return_robust is a display mirror rounded to cents. - assert params["champion"]["return_robust"] == pytest.approx( - round(float(champion["realized_total_return"]), 2) - ) - - -def test_robust_region_mirror_matches_promotion( - params: dict[str, Any], promotion: dict[str, Any] -) -> None: - region = promotion["robust_region_summary"] - expected = f"{region['n_alpha01_passers']}/{region['n_unique_policies']}" - assert params["paper"]["region_robust"] == expected - - -def test_variant_selection_rank1_path_exists(params: dict[str, Any]) -> None: - rank1 = params["conformal"]["variant_selection"]["rank1_path"] - run_dir = ROOT / "data" / "processed" / "portfolio_bound_aware" / rank1 - if not run_dir.parent.exists(): - pytest.skip("portfolio_bound_aware artifacts not available locally — run `dvc pull`.") - assert run_dir.is_dir(), f"rank1_path does not resolve to a run directory: {rank1}" diff --git a/tests/test_configs/test_params_view_generator.py b/tests/test_configs/test_params_view_generator.py deleted file mode 100644 index 2b5dc26..0000000 --- a/tests/test_configs/test_params_view_generator.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Tests for the generated ``params.yaml`` DVC view.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import yaml - -from scripts.build_params_view import build_params_view - -ROOT = Path(__file__).resolve().parents[2] - - -def _load_yaml(path: Path) -> dict[str, Any]: - payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - assert isinstance(payload, dict) - return payload - - -def test_generated_params_view_matches_tracked_params() -> None: - assert build_params_view(ROOT) == _load_yaml(ROOT / "params.yaml") diff --git a/tests/test_crpto_final_sync.py b/tests/test_crpto_final_sync.py deleted file mode 100644 index 254194c..0000000 --- a/tests/test_crpto_final_sync.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import csv -import json -from pathlib import Path - -import pytest - -EXPECTED_RUN_TAG = "ijds-rebaseline-2026-06-07" -EXPECTED_LABEL = "bound_aware_276k_economic_champion" -EXPECTED_RETURN = 170464.5429284627 -EXPECTED_V = 0.028875 -EXPECTED_GAMMA_CP = 0.187987 -LEGACY_TOKENS = [ - "paper_" + "estrella", - "paper-" + "estrella", - "reports/paper_" + "material", - "paper1" + "_table", - "estrella" + "_fig", -] - - -def load_json(path: str) -> dict: - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def load_key_metrics() -> dict[str, str]: - with Path("reports/crpto/tables/crpto_table0_key_metrics.csv").open(encoding="utf-8") as handle: - return {row["metric"]: row["value"] for row in csv.DictReader(handle)} - - -def test_crpto_champion_artifacts_agree() -> None: - """Validate the historical frozen rebaseline chain (two-tag scheme). - - This run tag remains the frozen upstream provenance chain. The active - manuscript policy is guarded by tests/test_ijds_active_claim_sync.py. - """ - assert Path("data/processed/final_project_summary.parquet.dvc").exists() - promotion = load_json("models/final_project_promotion.json") - policy = load_json("models/champion_portfolio_policy.json") - registry = load_json("models/champion_registry.json") - metrics = load_json("reports/dvc/metrics_summary.json") - table0 = load_key_metrics() - - champion = promotion["final_champion"] - selected_policy = policy["selected_policy"] - registry_portfolio = registry["portfolio"] - - assert promotion["run_tag"] == EXPECTED_RUN_TAG - assert champion["label"] == EXPECTED_LABEL - assert policy["run_tag"] == EXPECTED_RUN_TAG - assert registry_portfolio["run_tag"] == EXPECTED_RUN_TAG - assert registry_portfolio["selection_stage"] == "ijds_rebaseline_economic_v1" - - for field in ("risk_tolerance", "gamma", "uncertainty_aversion"): - assert selected_policy[field] == pytest.approx(champion[field]) - assert registry_portfolio["selected_policy"][field] == pytest.approx(champion[field]) - assert selected_policy["policy_mode"] == champion["policy_mode"] - assert registry_portfolio["selected_policy"]["policy_mode"] == champion["policy_mode"] - - assert champion["realized_total_return"] == pytest.approx(EXPECTED_RETURN) - assert champion["alpha01_exact_pass"] is True - assert champion["alpha01_weighted_miscoverage_V"] == pytest.approx(EXPECTED_V) - assert champion["alpha01_gamma_cp"] == pytest.approx(EXPECTED_GAMMA_CP) - - assert metrics["crpto.final.robust_return"] == pytest.approx(EXPECTED_RETURN) - assert metrics["crpto.final.alpha01_exact_pass"] == 1.0 - assert metrics["crpto.final.alpha01_weighted_miscoverage_V"] == pytest.approx(EXPECTED_V) - assert metrics["crpto.final.alpha01_gamma_cp"] == pytest.approx(EXPECTED_GAMMA_CP) - assert metrics["crpto.final.robust_region_n_policies"] == 45.0 - assert metrics["crpto.final.robust_region_alpha01_pass_rate"] == 1.0 - - assert table0["run_tag"] == EXPECTED_RUN_TAG - assert table0["champion_label"] == EXPECTED_LABEL - assert float(table0["robust_return"]) == pytest.approx(EXPECTED_RETURN) - assert table0["alpha01_exact_pass"] == "True" - assert float(table0["alpha01_weighted_miscoverage_V"]) == pytest.approx(EXPECTED_V) - assert float(table0["alpha01_gamma_cp"]) == pytest.approx(EXPECTED_GAMMA_CP) - - -def test_crpto_status_paths_are_sanitized() -> None: - for path in [ - "models/crpto_evidence_status.json", - "models/crpto_journal_package_status.json", - "models/crpto_tail_constrained_reopt_status.json", - "models/crpto_distribution_robustness_status.json", - ]: - text = Path(path).read_text(encoding="utf-8").lower() - assert not any(token in text for token in LEGACY_TOKENS) - - -def test_crpto_tables_and_figures_exist() -> None: - required = [ - "reports/crpto/tables/crpto_table0_key_metrics.csv", - "reports/crpto/tables/crpto_tableA7_funded_set_loans.csv", - "reports/crpto/tables/crpto_tableA18_robust_region_policy_family.csv", - "reports/crpto/tables/crpto_tableA22_tail_constrained_reoptimization.csv", - "reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv", - "reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv", - "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", - "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", - "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", - "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", - "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", - "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", - "reports/crpto/figures/crpto_fig1_journal_pipeline.png", - "reports/crpto/figures/crpto_fig1_journal_pipeline.pdf", - "reports/crpto/figures/crpto_fig1_journal_pipeline.svg", - "book/assets/figures/publication/crpto_fig1_journal_pipeline.png", - "book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf", - "book/assets/figures/publication/crpto_fig1_journal_pipeline.svg", - "reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png", - "reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png", - "reports/crpto/figures/crpto_fig20_bound_claim_layers.png", - ] - for item in required: - assert Path(item).exists(), item diff --git a/tests/test_data/test_outcome_observability.py b/tests/test_data/test_outcome_observability.py new file mode 100644 index 0000000..baebd74 --- /dev/null +++ b/tests/test_data/test_outcome_observability.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.data.outcome_observability import ( + archive_status_default_from_status, + audit_outcome_label_availability, + build_outcome_label_availability, + parse_last_payment_dates, + terminal_outcome_from_status, + validate_minimum_label_retention, +) + + +def test_archive_status_and_reconstructed_terminal_endpoint_are_distinct() -> None: + statuses = pd.Series(["Fully Paid", "Charged Off", "Default", "Current"]) + + archive_outcome = archive_status_default_from_status(statuses) + terminal_outcome = terminal_outcome_from_status(statuses) + + assert archive_outcome.iloc[:3].tolist() == [0, 1, 1] + assert pd.isna(archive_outcome.iloc[3]) + assert terminal_outcome.iloc[:2].tolist() == [0, 1] + assert terminal_outcome.iloc[2:].isna().all() + + +def test_terminal_protocol_leaves_default_and_other_states_unresolved() -> None: + statuses = pd.Series( + [ + "Fully Paid", + "Does not meet the credit policy. Status:Fully Paid", + "Charged Off", + "Does not meet the credit policy. Status:Charged Off", + "Default", + "Current", + ] + ) + + outcome = terminal_outcome_from_status(statuses) + + assert outcome.iloc[:4].tolist() == [0, 0, 1, 1] + assert outcome.iloc[4:].isna().all() + + +def test_last_payment_months_parse_conservatively_to_month_end() -> None: + parsed = parse_last_payment_dates(pd.Series(["Jan-2017", "2017-02-15", pd.NA])) + + assert parsed.iloc[0] == pd.Timestamp("2017-01-31") + assert parsed.iloc[1] == pd.Timestamp("2017-02-15") + assert pd.isna(parsed.iloc[2]) + + +def test_label_availability_respects_missing_dates_cutoff_and_chargeoff_lag() -> None: + statuses = pd.Series(["Fully Paid", "Charged Off", "Charged Off", "Default"]) + last_payments = pd.Series(["Jan-2017", "Jan-2017", pd.NA, "Jan-2017"]) + + before_lag = build_outcome_label_availability( + statuses, + last_payments, + cutoff="2017-07-30", + charged_off_lag_months=6, + ) + at_lag = build_outcome_label_availability( + statuses, + last_payments, + cutoff="2017-07-31", + charged_off_lag_months=6, + ) + + assert before_lag["label_available"].tolist() == [True, False, False, False] + assert at_lag["label_available"].tolist() == [True, True, False, False] + assert at_lag.loc[1, "label_available_at"] == pd.Timestamp("2017-07-31") + + +def test_block_audit_counts_retention_and_strict_99_percent_gate() -> None: + passing = pd.DataFrame( + { + "design_split": ["conformal_fit"] * 101, + "loan_status": ["Fully Paid"] * 100 + ["Current"], + "last_pymnt_d": ["Jan-2012"] * 101, + } + ) + audit = audit_outcome_label_availability(passing, cutoff="2013-01-31") + + assert audit.loc[0, "total_rows"] == 101 + assert audit.loc[0, "label_available_rows"] == 100 + assert audit.loc[0, "unresolved_outcome_rows"] == 1 + assert audit.loc[0, "retention_rate"] == pytest.approx(100 / 101) + validate_minimum_label_retention(audit, minimum_retention=0.99) + + boundary = audit.copy() + boundary.loc[0, "retention_rate"] = 0.99 + with pytest.raises(RuntimeError, match=r"must exceed 99\.00%"): + validate_minimum_label_retention(boundary, minimum_retention=0.99) diff --git a/tests/test_evaluation/test_backtesting.py b/tests/test_evaluation/test_backtesting.py deleted file mode 100644 index fc55ff8..0000000 --- a/tests/test_evaluation/test_backtesting.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Unit tests for backtesting functions (Kupiec, Christoffersen).""" - -import numpy as np -import pytest - -from src.evaluation.backtesting import ( - christoffersen_test, - interval_violations, - kupiec_pof_test, - population_stability_index, - winkler_interval_score, -) - -# ── Kupiec POF Test ── - - -def test_kupiec_perfect_coverage(): - """Zero violations with nominal alpha → should not reject.""" - violations = np.zeros(1000) - result = kupiec_pof_test(violations, alpha=0.10) - assert result["n_violations"] == 0 - assert result["violation_rate"] == 0.0 - assert result["nominal_alpha"] == 0.10 - - -def test_kupiec_exact_nominal_rate(): - """Violation rate exactly matching alpha → should not reject.""" - rng = np.random.RandomState(42) - violations = (rng.random(1000) < 0.10).astype(float) - result = kupiec_pof_test(violations, alpha=0.10) - assert result["p_value"] > 0.01 # Should not reject at reasonable level - assert result["reject"] is False - - -def test_kupiec_excessive_violations(): - """30% violations vs 10% nominal → should reject.""" - rng = np.random.RandomState(42) - violations = (rng.random(500) < 0.30).astype(float) - result = kupiec_pof_test(violations, alpha=0.10) - assert result["reject"] is True - assert result["p_value"] < 0.05 - - -def test_kupiec_empty_array(): - """Empty violations array → graceful return.""" - result = kupiec_pof_test(np.array([]), alpha=0.10) - assert result["n_total"] == 0 - assert result["p_value"] == 1.0 - assert result["reject"] is False - - -def test_kupiec_all_violations(): - """All violations → should reject.""" - violations = np.ones(100) - result = kupiec_pof_test(violations, alpha=0.10) - assert result["violation_rate"] == 1.0 - assert result["reject"] is True - - -def test_kupiec_returns_all_keys(): - """Check output dict has expected keys.""" - violations = np.array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0]) - result = kupiec_pof_test(violations, alpha=0.10) - expected_keys = { - "lr_statistic", - "p_value", - "reject", - "n_violations", - "n_total", - "violation_rate", - "nominal_alpha", - } - assert expected_keys == set(result.keys()) - - -# ── Christoffersen Test ── - - -def test_christoffersen_independent_violations(): - """IID violations at nominal rate → should not reject independence.""" - rng = np.random.RandomState(123) - violations = (rng.random(2000) < 0.10).astype(float) - result = christoffersen_test(violations, alpha=0.10) - # Should not reject independence for truly IID sequence - assert result["p_ind"] > 0.01 - assert result["reject_ind"] is False - - -def test_christoffersen_clustered_violations(): - """Clustered violations → should reject independence.""" - # Create strongly clustered pattern: 50 violations in a row, then 450 no violations - violations = np.zeros(500) - violations[100:150] = 1.0 # 50 consecutive violations → strong clustering - result = christoffersen_test(violations, alpha=0.10) - assert result["reject_ind"] is True - - -def test_christoffersen_returns_all_keys(): - """Check output dict has expected keys.""" - violations = np.array([0, 0, 1, 0, 1, 0, 0, 0, 1, 0]) - result = christoffersen_test(violations, alpha=0.10) - expected_keys = { - "lr_uc", - "p_uc", - "reject_uc", - "lr_ind", - "p_ind", - "reject_ind", - "lr_cc", - "p_cc", - "reject_cc", - "transition_matrix", - } - assert expected_keys == set(result.keys()) - - -def test_christoffersen_transition_matrix_sums(): - """Transition counts should sum to n-1.""" - violations = np.array([0, 1, 1, 0, 0, 1, 0, 1, 0, 0]) - result = christoffersen_test(violations, alpha=0.10) - tm = result["transition_matrix"] - assert tm["n00"] + tm["n01"] + tm["n10"] + tm["n11"] == len(violations) - 1 - - -def test_christoffersen_single_element(): - """Single-element array → graceful return.""" - violations = np.array([0.0]) - result = christoffersen_test(violations, alpha=0.10) - assert result["lr_ind"] == 0.0 - assert result["p_ind"] == 1.0 - - -def test_christoffersen_joint_statistic(): - """Joint LR should be sum of UC and IND components.""" - violations = np.array([0, 1, 0, 0, 1, 0, 1, 1, 0, 0] * 20) - result = christoffersen_test(violations, alpha=0.10) - assert result["lr_cc"] == pytest.approx(result["lr_uc"] + result["lr_ind"], abs=1e-6) - - -# ── PSI (existing function) ── - - -def test_psi_identical_distributions(): - """Same distribution → PSI ≈ 0.""" - rng = np.random.RandomState(42) - data = rng.random(1000) - psi = population_stability_index(data, data) - assert psi < 0.01 - - -def test_psi_shifted_distribution(): - """Shifted distribution → PSI > 0.""" - rng = np.random.RandomState(42) - expected = rng.random(1000) - actual = rng.random(1000) + 0.5 - psi = population_stability_index(expected, actual) - assert psi > 0.1 - - -# ── Interval quality/statistical utilities ── - - -def test_interval_violations_inside_and_outside(): - y_true = np.array([0.2, 0.6, 0.9, 0.1]) - low = np.array([0.0, 0.5, 0.7, 0.2]) - high = np.array([0.3, 0.7, 0.8, 0.4]) - v = interval_violations(y_true, low, high) - assert v.tolist() == [0.0, 0.0, 1.0, 1.0] - - -def test_winkler_score_penalizes_misses_more_than_hits(): - y_true = np.array([0.5, 0.95]) - low = np.array([0.4, 0.4]) - high = np.array([0.6, 0.6]) - scores = winkler_interval_score(y_true, low, high, alpha=0.10) - assert scores.shape == (2,) - assert scores[1] > scores[0] diff --git a/tests/test_evaluation/test_fairness.py b/tests/test_evaluation/test_fairness.py deleted file mode 100644 index 74022cc..0000000 --- a/tests/test_evaluation/test_fairness.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Unit tests for fairness metrics.""" - -import numpy as np -import pytest - -from src.evaluation.fairness import ( - demographic_parity_difference, - disparate_impact_ratio, - equalized_odds_gap, - fairness_report, -) - -# ── Demographic Parity Difference ── - - -def test_dpd_perfect_parity(): - """Identical positive rates across groups → DPD = 0.""" - y_pred = np.array([1, 0, 1, 1, 0, 1]) # A: 2/3, B: 2/3 - groups = np.array(["A", "A", "A", "B", "B", "B"]) - result = demographic_parity_difference(y_pred, groups) - assert result["dpd"] == pytest.approx(0.0, abs=1e-9) - - -def test_dpd_worst_case(): - """One group all positive, other all negative → DPD = 1.0.""" - y_pred = np.array([1, 1, 1, 0, 0, 0]) - groups = np.array(["A", "A", "A", "B", "B", "B"]) - result = demographic_parity_difference(y_pred, groups) - assert result["dpd"] == pytest.approx(1.0) - assert result["max_rate_group"] == "A" - assert result["min_rate_group"] == "B" - - -def test_dpd_known_values(): - """Known rates: A=0.75, B=0.25 → DPD=0.50.""" - y_pred = np.array([1, 1, 1, 0, 1, 0, 0, 0]) - groups = np.array(["A", "A", "A", "A", "B", "B", "B", "B"]) - result = demographic_parity_difference(y_pred, groups) - assert result["dpd"] == pytest.approx(0.50) - - -# ── Equalized Odds Gap ── - - -def test_eo_gap_perfect_parity(): - """Same TPR and FPR across groups → EO gap = 0.""" - y_true = np.array([1, 0, 1, 0]) - y_pred = np.array([1, 0, 1, 0]) - groups = np.array(["A", "A", "B", "B"]) - result = equalized_odds_gap(y_true, y_pred, groups) - assert result["eo_gap"] == pytest.approx(0.0, abs=1e-9) - - -def test_eo_gap_worst_case(): - """Group A perfect, Group B inverted → large gap.""" - y_true = np.array([1, 0, 1, 0]) - y_pred = np.array([1, 0, 0, 1]) - groups = np.array(["A", "A", "B", "B"]) - result = equalized_odds_gap(y_true, y_pred, groups) - assert result["tpr_gap"] == pytest.approx(1.0) - assert result["fpr_gap"] == pytest.approx(1.0) - assert result["eo_gap"] == pytest.approx(1.0) - - -# ── Disparate Impact Ratio ── - - -def test_dir_above_threshold(): - """Equal rates → DIR = 1.0.""" - y_pred = np.array([1, 0, 1, 0]) - groups = np.array(["A", "A", "B", "B"]) - result = disparate_impact_ratio(y_pred, groups) - assert result["dir"] == pytest.approx(1.0, abs=0.01) - - -def test_dir_zero_rate_group(): - """Group with 0 positive predictions → DIR ≈ 0.""" - y_pred = np.array([1, 1, 0, 0]) - groups = np.array(["A", "A", "B", "B"]) - result = disparate_impact_ratio(y_pred, groups) - assert result["dir"] < 0.01 - - -def test_dir_single_group(): - """Single group → DIR = 1.0 (no comparison possible).""" - y_pred = np.array([1, 0, 1]) - groups = np.array(["A", "A", "A"]) - result = disparate_impact_ratio(y_pred, groups) - assert result["dir"] == pytest.approx(1.0) - - -# ── Fairness Report ── - - -def test_fairness_report_keys(): - """Report should have required columns.""" - y_true = np.array([1, 0, 1, 0, 1, 0]) - y_proba = np.array([0.9, 0.1, 0.8, 0.2, 0.7, 0.3]) - groups_dict = {"attr_a": np.array(["X", "X", "X", "Y", "Y", "Y"])} - result = fairness_report(y_true, y_proba, groups_dict) - assert "attribute" in result.columns - assert "dpd" in result.columns - assert "eo_gap" in result.columns - assert "dir" in result.columns - assert "passed_all" in result.columns - - -def test_fairness_report_multi_attribute(): - """Two attributes should produce two rows.""" - y_true = np.array([1, 0, 1, 0]) - y_proba = np.array([0.9, 0.1, 0.8, 0.2]) - groups_dict = { - "attr_a": np.array(["X", "X", "Y", "Y"]), - "attr_b": np.array(["P", "Q", "P", "Q"]), - } - result = fairness_report(y_true, y_proba, groups_dict) - assert len(result) == 2 - assert list(result["attribute"]) == ["attr_a", "attr_b"] - - -def test_fairness_report_passed_flags(): - """Perfect parity should pass all thresholds.""" - y_true = np.array([1, 0, 1, 0]) - y_proba = np.array([0.9, 0.1, 0.9, 0.1]) - groups_dict = {"perfect": np.array(["A", "A", "B", "B"])} - result = fairness_report(y_true, y_proba, groups_dict) - assert result["passed_all"].iloc[0] is True or result["passed_all"].iloc[0] == True # noqa: E712 - - -# ── Conformal Fairness Report ── - - -def test_conformal_fairness_equal_coverage(): - """Equal coverage across groups → should pass.""" - from src.evaluation.fairness import conformal_fairness_report - - y_true = np.array([0.3, 0.7, 0.4, 0.8]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - groups_dict = {"attr": np.array(["A", "A", "B", "B"])} - result = conformal_fairness_report(y_true, y_intervals, groups_dict) - assert len(result) == 1 - assert result["coverage_disparity"].iloc[0] == pytest.approx(0.0) - assert result["passed_coverage"].iloc[0] is True or result["passed_coverage"].iloc[0] == True # noqa: E712 - - -def test_conformal_fairness_disparate_coverage(): - """One group fully covered, other not → large disparity.""" - from src.evaluation.fairness import conformal_fairness_report - - y_true = np.array([0.5, 0.5, 0.5, 0.5]) - # Group A: covered (interval contains 0.5) - # Group B: not covered (interval does not contain 0.5) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.6, 0.9], [0.6, 0.9]]) - groups_dict = {"attr": np.array(["A", "A", "B", "B"])} - result = conformal_fairness_report(y_true, y_intervals, groups_dict) - assert result["coverage_disparity"].iloc[0] == pytest.approx(1.0) - assert result["passed_coverage"].iloc[0] is False or result["passed_coverage"].iloc[0] == False # noqa: E712 - - -def test_conformal_fairness_returns_all_keys(): - """Check output has required columns.""" - from src.evaluation.fairness import conformal_fairness_report - - y_true = np.array([0.5, 0.5, 0.5, 0.5]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - groups_dict = {"attr": np.array(["X", "X", "Y", "Y"])} - result = conformal_fairness_report(y_true, y_intervals, groups_dict) - expected_cols = { - "attribute", - "n_groups", - "min_coverage", - "max_coverage", - "coverage_disparity", - "min_avg_width", - "max_avg_width", - "width_ratio", - "passed_coverage", - "passed_width", - "passed_all", - "group_details", - } - assert expected_cols.issubset(set(result.columns)) - - -def test_conformal_fairness_multi_attribute(): - """Two attributes should produce two rows.""" - from src.evaluation.fairness import conformal_fairness_report - - y_true = np.array([0.3, 0.7, 0.4, 0.8]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - groups_dict = { - "attr_a": np.array(["A", "A", "B", "B"]), - "attr_b": np.array(["X", "Y", "X", "Y"]), - } - result = conformal_fairness_report(y_true, y_intervals, groups_dict) - assert len(result) == 2 diff --git a/tests/test_evaluation/test_maturity_safe_portfolio.py b/tests/test_evaluation/test_maturity_safe_portfolio.py new file mode 100644 index 0000000..5918c10 --- /dev/null +++ b/tests/test_evaluation/test_maturity_safe_portfolio.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from src.evaluation import maturity_safe_portfolio as portfolio +from src.optimization.policy import PolicyMode +from src.optimization.policy_selection import LinearPolicyCandidate + + +def _decision_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "id": ["a", "b"], + "issue_d": pd.to_datetime(["2016-04-01", "2016-04-01"]), + "loan_amnt": [100.0, 200.0], + "purpose": ["x", "y"], + "contractual_rate": [0.10, 0.20], + "pd_point": [0.10, 0.25], + "conformal_lower": [0.00, 0.10], + "conformal_upper": [0.20, 0.40], + "conformal_group": [0, 1], + } + ) + + +def _config(*, strict: bool) -> dict[str, object]: + return { + "payoff": {"lgd": 0.45}, + "policy": { + "budget": 200.0, + "max_concentration_by_purpose": 1.0, + "min_budget_utilization_solver": 1.0, + }, + "execution": { + "solver_time_limit_seconds": 10, + "threads": 1, + "solver_backend": "highspy", + "strict_solver_backend": strict, + "random_seed": 42, + }, + } + + +def _candidate() -> LinearPolicyCandidate: + return LinearPolicyCandidate( + candidate_id="linear-001", + risk_tolerance=0.17, + gamma=0.5, + uncertainty_aversion=0.0, + min_budget_utilization=1.0, + ) + + +def _fake_solver(actual_backend: str, captured: dict[str, np.ndarray]): + def solve(**kwargs: object) -> SimpleNamespace: + point = np.asarray(kwargs["pd_point"], dtype=float) + contractual_rates = np.asarray(kwargs["int_rates"], dtype=float) + objective_rates = np.asarray(kwargs["objective_rate_override"], dtype=float) + allocation = np.array([1.0, 0.5]) + exposure = allocation * _decision_frame()["loan_amnt"].to_numpy(dtype=float) + captured["int_rates"] = contractual_rates + captured["objective_rate_override"] = objective_rates + return SimpleNamespace( + solution={ + "solver_status": "Optimal", + "solver_backend": actual_backend, + "objective_value": float(exposure @ objective_rates), + }, + allocation=allocation, + effective_pd=point, + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + delta_cap_quantile=1.0, + tail_focus_quantile=1.0, + objective_risk_mode="legacy", + ) + + return solve + + +def test_coherent_objective_is_passed_explicitly_to_solver( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, np.ndarray] = {} + monkeypatch.setattr(portfolio, "solve_policy_allocation", _fake_solver("highspy", captured)) + + solved = portfolio.solve_coherent_policy( + _decision_frame(), + _candidate(), + config=_config(strict=True), + robust=True, + ) + + np.testing.assert_allclose(captured["int_rates"], np.array([0.10, 0.20])) + np.testing.assert_allclose( + captured["objective_rate_override"], + np.array([0.045, 0.0375]), + ) + np.testing.assert_allclose(solved.expected_payoff_rate, np.array([0.045, 0.0375])) + + +def test_strict_highspy_backend_rejects_silent_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + portfolio, + "solve_policy_allocation", + _fake_solver("highspy_fallback_highs_sparse", {}), + ) + + with pytest.raises(RuntimeError, match="Strict solver backend mismatch"): + portfolio.solve_coherent_policy( + _decision_frame(), + _candidate(), + config=_config(strict=True), + robust=True, + ) + + +def test_non_strict_backend_retains_fallback_compatibility( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + portfolio, + "solve_policy_allocation", + _fake_solver("highspy_fallback_highs_sparse", {}), + ) + + solved = portfolio.solve_coherent_policy( + _decision_frame(), + _candidate(), + config=_config(strict=False), + robust=True, + ) + + assert solved.result.solution["solver_backend"] == "highspy_fallback_highs_sparse" + + +def test_prejoined_evaluation_separates_menu_and_funded_censoring_counts() -> None: + joined = _decision_frame().assign( + allocation_fraction=[1.0, 0.5], + exposure=[100.0, 100.0], + weight=[0.5, 0.5], + pd_effective=[0.1, 0.25], + expected_payoff_rate=[0.045, 0.0375], + expected_payoff_contribution=[4.5, 3.75], + role="primary_oot", + period="2016-04", + policy_label="guardrail_linear-001", + candidate_id="linear-001", + snapshot_default=pd.Series([0, pd.NA], dtype="Int8"), + snapshot_resolution=pd.Series(["fully_paid", "right_censored"], dtype="string"), + ) + base_record = { + "role": "primary_oot", + "period": "2016-04", + "policy_label": "guardrail_linear-001", + "robust_guardrail": True, + "total_allocated": 200.0, + } + + record, evaluated = portfolio.evaluate_prejoined_frozen_allocation( + base_record, + joined, + config=_config(strict=True), + n_unresolved_candidates=7, + ) + + assert record["n_unresolved_candidates"] == 7 + assert record["n_unresolved_positive_exposure"] == 1 + assert record["unresolved_exposure_share"] == pytest.approx(0.5) + assert evaluated["realized_payoff_lower"].notna().all() diff --git a/tests/test_evaluation/test_metrics.py b/tests/test_evaluation/test_metrics.py deleted file mode 100644 index 47e98de..0000000 --- a/tests/test_evaluation/test_metrics.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Unit tests for evaluation metrics.""" - -import numpy as np - -from src.evaluation.metrics import ( - classification_metrics, - compute_all_metrics, - conformal_metrics, - ks_statistic, - regression_metrics, -) - -# ── classification_metrics ── - - -def test_classification_metrics_keys(): - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 100) - y_prob = rng.random(100) - result = classification_metrics(y_true, y_prob) - expected_keys = { - "auc_roc", - "gini", - "brier_score", - "ece", - "log_loss", - "ks_statistic", - "pr_auc", - "recall_at_0p35", - "f1_at_0p35", - } - if "d2_brier_score" in result: - expected_keys.add("d2_brier_score") - assert expected_keys == set(result.keys()) - - -def test_classification_metrics_perfect(): - y_true = np.array([0, 0, 0, 1, 1, 1]) - y_prob = np.array([0.0, 0.1, 0.2, 0.8, 0.9, 1.0]) - result = classification_metrics(y_true, y_prob) - assert result["auc_roc"] == 1.0 - assert result["gini"] == 1.0 - - -def test_classification_metrics_bounded(): - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 200) - y_prob = rng.random(200) - result = classification_metrics(y_true, y_prob) - assert 0 <= result["auc_roc"] <= 1 - assert -1 <= result["gini"] <= 1 - assert 0 <= result["brier_score"] <= 1 - assert 0 <= result["ece"] <= 1 - assert 0 <= result["ks_statistic"] <= 1 - if "d2_brier_score" in result: - assert np.isfinite(result["d2_brier_score"]) - - -# ── ks_statistic ── - - -def test_ks_statistic_perfect_separation(): - y_true = np.array([0, 0, 0, 1, 1, 1]) - y_prob = np.array([0.1, 0.2, 0.3, 0.7, 0.8, 0.9]) - ks = ks_statistic(y_true, y_prob) - assert ks == 1.0 - - -def test_ks_statistic_bounded(): - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 200) - y_prob = rng.random(200) - ks = ks_statistic(y_true, y_prob) - assert 0 <= ks <= 1 - - -# ── regression_metrics ── - - -def test_regression_metrics_perfect(): - y_true = np.array([1.0, 2.0, 3.0]) - y_pred = np.array([1.0, 2.0, 3.0]) - result = regression_metrics(y_true, y_pred) - assert result["mae"] == 0.0 - assert result["rmse"] == 0.0 - assert result["r2"] == 1.0 - - -def test_regression_metrics_keys(): - y_true = np.array([1.0, 2.0]) - y_pred = np.array([1.1, 2.2]) - result = regression_metrics(y_true, y_pred) - assert set(result.keys()) == {"mae", "rmse", "r2"} - - -# ── conformal_metrics ── - - -def test_conformal_metrics_full_coverage(): - y_true = np.array([0.5, 0.5, 0.5]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - result = conformal_metrics(y_true, y_intervals, alpha=0.1) - assert result["empirical_coverage"] == 1.0 - assert result["avg_width"] == 1.0 - - -def test_conformal_metrics_keys(): - y_true = np.array([0.5]) - y_intervals = np.array([[0.0, 1.0]]) - result = conformal_metrics(y_true, y_intervals, alpha=0.1) - expected = { - "empirical_coverage", - "target_coverage", - "coverage_gap", - "avg_width", - "median_width", - "width_std", - "width_90th_pct", - } - assert expected == set(result.keys()) - - -# ── compute_all_metrics ── - - -def test_compute_all_metrics_without_conformal(): - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 100) - y_prob = rng.random(100) - result = compute_all_metrics(y_true, y_prob) - assert "auc_roc" in result - assert "empirical_coverage" not in result - - -def test_compute_all_metrics_with_conformal(): - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 100).astype(float) - y_prob = rng.random(100) - y_intervals = np.column_stack([y_prob - 0.3, y_prob + 0.3]) - result = compute_all_metrics(y_true, y_prob, y_intervals=y_intervals, alpha=0.1) - assert "auc_roc" in result - assert "empirical_coverage" in result diff --git a/tests/test_evaluation/test_model_shift.py b/tests/test_evaluation/test_model_shift.py deleted file mode 100644 index ba18f8a..0000000 --- a/tests/test_evaluation/test_model_shift.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from src.evaluation.model_shift import interpret_model_shift - - -def test_interpret_model_shift_distinguishes_structural_shift_only() -> None: - out = interpret_model_shift( - c2st_auc=0.66, - c2st_materiality="high", - score_psi=0.18, - auc_delta=0.005, - brier_increase=0.001, - calibration_gap_delta=0.002, - distribution_warning_ratio=0.25, - score_psi_max=0.15, - auc_delta_max=0.05, - brier_increase_max=0.02, - calibration_gap_delta_max=0.02, - ) - - assert out["shift_type"] == "structural_shift_only" - assert out["governance_posture"] == "warning_only" - - -def test_interpret_model_shift_detects_mixed_shift() -> None: - out = interpret_model_shift( - c2st_auc=0.68, - c2st_materiality="high", - score_psi=0.20, - auc_delta=0.08, - brier_increase=0.04, - calibration_gap_delta=0.05, - distribution_warning_ratio=0.40, - score_psi_max=0.15, - auc_delta_max=0.05, - brier_increase_max=0.02, - calibration_gap_delta_max=0.02, - ) - - assert out["shift_type"] == "mixed_shift" - assert out["governance_posture"] == "candidate_gate" - - -def test_interpret_model_shift_detects_predictive_degradation_without_structural_shift() -> None: - out = interpret_model_shift( - c2st_auc=0.50, - c2st_materiality="none", - score_psi=0.02, - auc_delta=0.08, - brier_increase=0.001, - calibration_gap_delta=0.001, - distribution_warning_ratio=0.0, - score_psi_max=0.15, - auc_delta_max=0.05, - brier_increase_max=0.02, - calibration_gap_delta_max=0.02, - ) - - assert out["shift_type"] == "predictive_degradation" - assert out["governance_posture"] == "candidate_gate" - assert "Operational metrics dominate" in str(out["pvalue_interpretation"]) - - -def test_interpret_model_shift_keeps_stable_cases_on_monitor_posture() -> None: - out = interpret_model_shift( - c2st_auc=0.50, - c2st_materiality="none", - score_psi=0.01, - auc_delta=0.0, - brier_increase=0.0, - calibration_gap_delta=0.0, - distribution_warning_ratio=0.0, - score_psi_max=0.15, - auc_delta_max=0.05, - brier_increase_max=0.02, - calibration_gap_delta_max=0.02, - ) - - assert out["shift_type"] == "stable" - assert out["governance_posture"] == "monitor" diff --git a/tests/test_evaluation/test_pd_validation_interpretation.py b/tests/test_evaluation/test_pd_validation_interpretation.py deleted file mode 100644 index 1a8f874..0000000 --- a/tests/test_evaluation/test_pd_validation_interpretation.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import pandas as pd - -from src.evaluation.pd_validation_interpretation import ( - classify_overall_gap_materiality, - summarize_slice_materiality, - validation_interpretation_status, -) - - -def test_classify_overall_gap_materiality_assigns_low_band() -> None: - out = classify_overall_gap_materiality(0.2200, 0.2165) - assert out["materiality_band"] == "low" - assert abs(float(out["gap_bp"])) < 50.0 - - -def test_summarize_slice_materiality_counts_grade_and_band_breaches() -> None: - grade = pd.DataFrame( - { - "observed_default_rate": [0.10, 0.30], - "mean_predicted_pd": [0.07, 0.31], - } - ) - band = pd.DataFrame({"rate_gap": [0.002, 0.020, -0.018]}) - out = summarize_slice_materiality(grade, band) - assert out["grade_material_breaches"] == 1 - assert out["band_material_breaches"] == 2 - - -def test_validation_interpretation_escalates_to_warning_for_slice_breaches() -> None: - quarter = pd.DataFrame({"abs_gap_bp": [80.0, 20.0], "rate_gap": [0.008, 0.002]}) - out = validation_interpretation_status( - overall_backtesting={ - "observed_default_rate": 0.2198, - "mean_predicted_pd": 0.2161, - "predicted_pd_inside_jeffreys": False, - "exact_binomial_p_value": 1e-6, - "hl_p_value": 0.0, - }, - slice_materiality={ - "grade_material_breaches": 2, - "max_grade_gap_bp": 500.0, - "band_material_breaches": 1, - "max_band_gap_bp": 120.0, - }, - quarter_report=quarter, - rare_event={"max_decile_gap_bp": 150.0}, - ) - assert out["overall_pass"] is True - assert out["severity"] == "warning" diff --git a/tests/test_evaluation/test_policy_contrast_bounds.py b/tests/test_evaluation/test_policy_contrast_bounds.py new file mode 100644 index 0000000..aba0077 --- /dev/null +++ b/tests/test_evaluation/test_policy_contrast_bounds.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +from itertools import product + +import numpy as np +import pandas as pd +import pytest + +from src.evaluation.policy_contrast_bounds import ( + PolicyContrastIndex, + sharp_policy_contrast_bounds, +) + + +def _policy_pair() -> pd.DataFrame: + pair = pd.DataFrame( + { + "id": pd.Series(["shared", "a-only", "shared", "b-only"], dtype="string"), + "role": ["primary_oot"] * 4, + "policy_label": ["policy-a", "policy-a", "policy-b", "policy-b"], + "exposure": [60.0, 40.0, 40.0, 60.0], + "contractual_rate": [0.10, 0.08, 0.10, 0.12], + "conformal_lower": [0.0, 0.0, 0.0, 0.1], + "conformal_upper": [0.8, 0.9, 0.8, 1.0], + "expected_payoff_contribution": [5.0, 2.0, 3.0, 4.0], + } + ) + pair["snapshot_default"] = pd.Series([pd.NA, 0, pd.NA, 1], dtype="Int8") + return pair + + +def test_sharp_policy_contrast_preserves_matching_nullable_outcome_fact() -> None: + bounds = sharp_policy_contrast_bounds( + _policy_pair(), + policy_a="policy-a", + policy_b="policy-b", + role="primary_oot", + lgd=0.45, + ) + + assert bounds["funded_union_loans"] == 3 + assert bounds["unresolved_union_loans"] == 1 + assert np.isfinite(bounds["realized_payoff_difference_lower"]) + assert bounds["realized_payoff_difference_lower"] <= bounds["realized_payoff_difference_upper"] + assert bounds["realized_payoff_identification_width"] == pytest.approx( + bounds["realized_payoff_difference_upper"] - bounds["realized_payoff_difference_lower"] + ) + assert bounds["weighted_default_identification_width"] == pytest.approx( + bounds["weighted_default_difference_upper"] - bounds["weighted_default_difference_lower"] + ) + assert bounds["weighted_miscoverage_identification_width"] == pytest.approx( + bounds["weighted_miscoverage_difference_upper"] + - bounds["weighted_miscoverage_difference_lower"] + ) + + +def test_identification_width_uses_only_unresolved_exposure_difference() -> None: + bounds = sharp_policy_contrast_bounds( + _policy_pair(), + policy_a="policy-a", + policy_b="policy-b", + role="primary_oot", + lgd=0.45, + ) + + # Only the shared loan is unresolved; its exposure difference is USD 20. + assert bounds["realized_payoff_identification_width"] == pytest.approx(11.0) + assert bounds["weighted_default_identification_width"] == pytest.approx(0.2) + assert bounds["weighted_miscoverage_identification_width"] == pytest.approx(0.2) + + +def test_sharp_bounds_equal_exhaustive_binary_completions() -> None: + loan_ids = ["resolved", "missing-a", "missing-b"] + exposure_a = np.array([50.0, 30.0, 20.0]) + exposure_b = np.array([20.0, 50.0, 30.0]) + rates = np.array([0.08, 0.12, 0.16]) + conformal_lower = np.array([0.0, 0.2, 0.0]) + conformal_upper = np.array([0.8, 1.0, 0.7]) + outcomes = np.array([0.0, np.nan, np.nan]) + allocations = pd.concat( + [ + pd.DataFrame( + { + "id": loan_ids, + "role": "primary_oot", + "policy_label": policy, + "exposure": exposures, + "expected_payoff_contribution": exposures * 0.01, + "contractual_rate": rates, + "conformal_lower": conformal_lower, + "conformal_upper": conformal_upper, + "snapshot_default": pd.Series([0, pd.NA, pd.NA], dtype="Int8"), + } + ) + for policy, exposures in (("policy-a", exposure_a), ("policy-b", exposure_b)) + ], + ignore_index=True, + ) + bounds = sharp_policy_contrast_bounds( + allocations, + policy_a="policy-a", + policy_b="policy-b", + role="primary_oot", + lgd=0.45, + ) + + delta_exposure = exposure_a - exposure_b + delta_weight = exposure_a / exposure_a.sum() - exposure_b / exposure_b.sum() + completed_metrics: dict[str, list[float]] = { + "realized_payoff": [], + "realized_payoff_rate": [], + "weighted_default": [], + "weighted_miscoverage": [], + } + for completion in product((0.0, 1.0), repeat=2): + realized = outcomes.copy() + realized[~np.isfinite(realized)] = completion + payoff = (1.0 - realized) * rates - realized * 0.45 + completed_metrics["realized_payoff"].append(float((delta_exposure * payoff).sum())) + completed_metrics["realized_payoff_rate"].append(float((delta_weight * payoff).sum())) + completed_metrics["weighted_default"].append(float((delta_weight * realized).sum())) + miscovered = np.where(realized == 0.0, conformal_lower > 0.0, conformal_upper < 1.0) + completed_metrics["weighted_miscoverage"].append(float((delta_weight * miscovered).sum())) + + for metric, values in completed_metrics.items(): + assert bounds[f"{metric}_difference_lower"] == pytest.approx(min(values)) + assert bounds[f"{metric}_difference_upper"] == pytest.approx(max(values)) + + +@pytest.mark.parametrize( + ("column", "conflicting_value"), + [ + ("contractual_rate", 0.11), + ("conformal_lower", 0.1), + ("conformal_upper", 0.9), + ("snapshot_default", 0), + ], +) +def test_sharp_policy_contrast_rejects_conflicting_policy_facts( + column: str, + conflicting_value: float, +) -> None: + pair = _policy_pair() + pair.loc[ + pair["id"].eq("shared") & pair["policy_label"].eq("policy-b"), + column, + ] = conflicting_value + + with pytest.raises(ValueError, match="Conflicting policy facts"): + sharp_policy_contrast_bounds( + pair, + policy_a="policy-a", + policy_b="policy-b", + role="primary_oot", + lgd=0.45, + ) + + +@pytest.mark.parametrize("reverse_rows", [False, True]) +def test_indexed_bounds_equal_slow_oracle_on_adversarial_union(reverse_rows: bool) -> None: + allocations = _policy_pair() + extra = allocations.iloc[[0, 1]].copy() + extra["policy_label"] = "policy-c" + extra["exposure"] = [0.0, 100.0] + extra["expected_payoff_contribution"] = [-1.0, 9.0] + allocations = pd.concat([allocations, extra], ignore_index=True) + if reverse_rows: + allocations = allocations.iloc[::-1].reset_index(drop=True) + + oracle = sharp_policy_contrast_bounds( + allocations, + policy_a="policy-a", + policy_b="policy-b", + role="primary_oot", + lgd=0.45, + ) + optimized = PolicyContrastIndex(allocations, role="primary_oot").sharp_bounds( + policy_a="policy-a", + policy_b="policy-b", + lgd=0.45, + ) + + assert optimized.keys() == oracle.keys() + for key, expected in oracle.items(): + if isinstance(expected, float): + assert optimized[key] == pytest.approx(expected, rel=0.0, abs=1e-15) + else: + assert optimized[key] == expected + + +def test_indexed_bounds_reject_duplicate_allocation_and_external_fact_ids() -> None: + allocations = _policy_pair() + facts = allocations[ + [ + "id", + "contractual_rate", + "conformal_lower", + "conformal_upper", + "snapshot_default", + ] + ].drop_duplicates() + + duplicate_allocation = pd.concat([allocations, allocations.iloc[[0]]], ignore_index=True) + with pytest.raises(ValueError, match="appears more than once"): + PolicyContrastIndex(duplicate_allocation, role="primary_oot") + + duplicate_facts = pd.concat([facts, facts.iloc[[0]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicate IDs"): + PolicyContrastIndex( + allocations, + role="primary_oot", + loan_facts=duplicate_facts, + ) + + +def test_indexed_bounds_reject_conflicting_or_missing_external_facts() -> None: + allocations = _policy_pair() + facts = allocations[ + [ + "id", + "contractual_rate", + "conformal_lower", + "conformal_upper", + "snapshot_default", + ] + ].drop_duplicates() + conflict = facts.iloc[[0]].copy() + conflict["contractual_rate"] = 0.99 + with pytest.raises(ValueError, match="Conflicting policy facts"): + PolicyContrastIndex( + allocations, + role="primary_oot", + loan_facts=pd.concat([facts, conflict], ignore_index=True), + ) + + with pytest.raises(ValueError, match="ID mismatch: missing=1"): + PolicyContrastIndex( + allocations, + role="primary_oot", + loan_facts=facts.loc[~facts["id"].eq("b-only")], + ) + + +@pytest.mark.parametrize( + ("column", "value", "message"), + [ + ("exposure", -1.0, "negative exposure"), + ("expected_payoff_contribution", float("inf"), "non-finite expected"), + ("contractual_rate", float("inf"), "contractual rate values must be finite"), + ("conformal_lower", -0.1, "0 <= lower <= upper <= 1"), + ("conformal_upper", 1.1, "0 <= lower <= upper <= 1"), + ], +) +def test_indexed_bounds_reject_invalid_numeric_domains( + column: str, + value: float, + message: str, +) -> None: + allocations = _policy_pair() + allocations.loc[allocations["id"].eq("a-only"), column] = value + + if column in {"exposure", "expected_payoff_contribution"}: + with pytest.raises(ValueError, match=message): + PolicyContrastIndex(allocations, role="primary_oot") + return + + index = PolicyContrastIndex(allocations, role="primary_oot") + with pytest.raises(ValueError, match=message): + index.sharp_bounds(policy_a="policy-a", policy_b="policy-b", lgd=0.45) + + +def test_indexed_bounds_reject_invalid_lgd() -> None: + index = PolicyContrastIndex(_policy_pair(), role="primary_oot") + + with pytest.raises(ValueError, match="LGD"): + index.sharp_bounds(policy_a="policy-a", policy_b="policy-b", lgd=float("nan")) + + +def test_indexed_bounds_can_normalize_both_policies_to_committed_capital() -> None: + index = PolicyContrastIndex(_policy_pair(), role="primary_oot") + + bounds = index.sharp_bounds( + policy_a="policy-a", + policy_b="policy-b", + lgd=0.45, + normalization_capital_a=125.0, + normalization_capital_b=125.0, + ) + + assert bounds["policy_a_capital"] == 100.0 + assert bounds["policy_a_normalization_capital"] == 125.0 + assert bounds["policy_b_normalization_capital"] == 125.0 + assert bounds["weighted_default_identification_width"] == pytest.approx(0.16) + + +def test_indexed_bounds_reject_normalizer_below_invested_capital() -> None: + index = PolicyContrastIndex(_policy_pair(), role="primary_oot") + + with pytest.raises(ValueError, match="below invested"): + index.sharp_bounds( + policy_a="policy-a", + policy_b="policy-b", + lgd=0.45, + normalization_capital_a=99.0, + ) + + +def test_indexed_bounds_accepts_scale_small_solver_budget_residual() -> None: + pair = _policy_pair() + pair.loc[pair["policy_label"].eq("policy-a"), "exposure"] *= 150_000.00000001 + pair.loc[pair["policy_label"].eq("policy-b"), "exposure"] *= 150_000.0 + pair["expected_payoff_contribution"] = pair["exposure"] * 0.05 + index = PolicyContrastIndex(pair, role="primary_oot") + + bounds = index.sharp_bounds( + policy_a="policy-a", + policy_b="policy-b", + lgd=0.45, + normalization_capital_a=15_000_000.0, + normalization_capital_b=15_000_000.0, + ) + + assert bounds["policy_a_normalization_capital"] == 15_000_000.0 diff --git a/tests/test_evaluation/test_standardized_credit_payoff.py b/tests/test_evaluation/test_standardized_credit_payoff.py new file mode 100644 index 0000000..ca75e71 --- /dev/null +++ b/tests/test_evaluation/test_standardized_credit_payoff.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from src.evaluation.standardized_credit_payoff import ( + expected_objective_coefficients, + realized_standardized_payoff_bounds, +) + + +def test_expected_objective_coefficients_use_coherent_payoff() -> None: + coefficients = expected_objective_coefficients( + np.array([0.0, 1.0, 0.2]), + np.array([0.1, 0.1, 0.2]), + lgd=0.45, + ) + + np.testing.assert_allclose(coefficients, np.array([0.1, -0.45, 0.07])) + + +@pytest.mark.parametrize( + ("probability", "rate", "lgd"), + [ + (np.nan, 0.1, 0.45), + (1.01, 0.1, 0.45), + (0.1, np.inf, 0.45), + (0.1, -0.01, 0.45), + (0.1, 0.1, np.nan), + (0.1, 0.1, 1.01), + ], +) +def test_expected_objective_rejects_invalid_inputs( + probability: float, + rate: float, + lgd: float, +) -> None: + with pytest.raises(ValueError): + expected_objective_coefficients( + np.array([probability]), + np.array([rate]), + lgd=lgd, + ) + + +def test_realized_bounds_reject_infinite_outcomes_and_invalid_parameters() -> None: + with pytest.raises(ValueError, match="binary or NaN"): + realized_standardized_payoff_bounds( + np.array([np.inf]), + np.array([0.1]), + lgd=0.45, + ) + with pytest.raises(ValueError, match="Contractual rates"): + realized_standardized_payoff_bounds( + np.array([0.0]), + np.array([1.1]), + lgd=0.45, + ) + with pytest.raises(ValueError, match="LGD"): + realized_standardized_payoff_bounds( + np.array([0.0]), + np.array([0.1]), + lgd=-0.1, + ) diff --git a/tests/test_experiments/test_champion_reopen_orchestration.py b/tests/test_experiments/test_champion_reopen_orchestration.py deleted file mode 100644 index 5f7dca4..0000000 --- a/tests/test_experiments/test_champion_reopen_orchestration.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -from scripts.experiments.run_champion_claim_max_downstream import ( - _is_paper_facing_case, - _portfolio_command, - _select_downstream_candidates, -) -from scripts.experiments.run_champion_reopen import _build_commands - - -def _config() -> dict[str, object]: - return { - "run_tag": "unit", - "seeds": [42, 52], - "output": { - "report_dir": "reports/crpto/experiments/champion_reopen", - "model_dir": "models/experiments/champion_reopen", - }, - "champion_reopen": { - "selector_model": "models/experiments/tabprep/selector.cbm", - "ranking_method": "pvc", - "shap_rows": 123, - "smoke_sample_rows": 1000, - "smoke_cases": ["core42", "pooltop42_tab20"], - "feature_search_cases": ["pool93_top90"], - "seed_replay_cases": ["pool93_top90", "pool93_business80"], - }, - "calibration": {"candidates": ["platt", "temperature"]}, - } - - -def test_smoke_commands_are_sampled_and_checkpointed() -> None: - commands = _build_commands( - config=_config(), - stage="smoke", - run_tag="unit-run", - sample_rows_override=None, - ) - - assert len(commands) == 1 - command = commands[0] - assert "--sample-rows" in command.command - assert "1000" in command.command - assert command.expected_output.endswith("selected_feature_experiment_summary.json") - assert "reports/run_logs/champion_reopen" in command.log_path.replace("\\", "/") - - -def test_seed_replay_uses_fixed_tabprep_seed_for_catboost_seed_stability() -> None: - commands = _build_commands( - config=_config(), - stage="seed_replay", - run_tag="unit-run", - sample_rows_override=None, - ) - - assert len(commands) == 2 - for command in commands: - tabprep_idx = command.command.index("--tabprep-seed") - assert command.command[tabprep_idx + 1] == "42" - assert "--full-data" in command.command - - -def test_claim_max_downstream_keeps_mandatory_and_paper_facing_cases() -> None: - candidates = [ - {"case_name": "opaque_rank1", "paper_facing": False, "claim_pd_score": 9.0}, - {"case_name": "opaque_rank2", "paper_facing": False, "claim_pd_score": 8.0}, - {"case_name": "pool93_woe", "paper_facing": True, "claim_pd_score": 7.0}, - {"case_name": "pool93", "paper_facing": True, "claim_pd_score": 6.0}, - {"case_name": "pool93_business80", "paper_facing": True, "claim_pd_score": 5.0}, - ] - - selected, policy = _select_downstream_candidates( - candidates, - top_k=2, - mandatory_cases=["pool93"], - paper_facing_top_k=2, - ) - - assert [row["case_name"] for row in selected] == [ - "opaque_rank1", - "opaque_rank2", - "pool93_woe", - "pool93", - ] - assert policy["missing_mandatory_cases"] == [] - assert "mandatory_case" in selected[3]["selection_reasons"] - assert "paper_facing_top_k" in selected[2]["selection_reasons"] - - -def test_claim_max_downstream_can_skip_dedicated_pool93_lane() -> None: - candidates = [ - {"case_name": "pool93_woe", "paper_facing": True, "claim_pd_score": 7.0}, - {"case_name": "pool93", "paper_facing": True, "claim_pd_score": 6.0}, - {"case_name": "pool93_business80", "paper_facing": True, "claim_pd_score": 5.0}, - ] - - selected, policy = _select_downstream_candidates( - candidates, - top_k=2, - mandatory_cases=["pool93"], - paper_facing_top_k=3, - skip_cases=["pool93"], - ) - - assert [row["case_name"] for row in selected] == ["pool93_woe", "pool93_business80"] - assert policy["skip_cases"] == ["pool93"] - assert policy["missing_mandatory_cases"] == [] - - -def test_claim_max_paper_facing_case_detection_includes_pool93_family() -> None: - assert _is_paper_facing_case("pool93") - assert _is_paper_facing_case("pool93_woe") - assert _is_paper_facing_case("pool93_business80") - assert _is_paper_facing_case("pooltop72_business80") - assert not _is_paper_facing_case("pooltop72_tab60") - - -def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: - command = _portfolio_command( - portfolio_profile={ - "candidate_policy_families": ["blended_uncertainty"], - "execution": { - "solver_backend": "cuopt", - "exact_solver_backend": "highs", - "frontier_only": True, - "exact_python_executable": "python-exact", - }, - "frontier": { - "proxy_candidates_per_conformal_finalist": 100000, - "exact_max_candidates": 0, - "exact_random_states": "42,52,62", - "exact_checkpoint_every": 25, - "exact_threads": 8, - "exact_rerank_top_k": 60, - }, - "grids": { - "risk_grid": "0.16", - "gamma_grid": "0.45", - "aversion_grid": "0", - "delta_cap_grid": "1.0", - "tail_focus_grid": "1.0", - "alpha_grid": "0.01", - "random_states": "42", - "budget_profiles": "free", - }, - "incumbent_region": { - "risk_neighbors": "0.16", - "gamma_neighbors": "0.45", - "policy_modes": "blended_uncertainty", - }, - "cuopt": { - "method": "concurrent", - "num_gpus": 1, - "extra_parameters": {"tolerance": "tight"}, - }, - }, - conformal_intervals_path=tmp_path / "intervals.parquet", - run_label="unit", - output_dir=tmp_path / "out", - model_dir=tmp_path / "model", - ) - - assert command[command.index("--max-candidates") + 1] == "100000" - assert command[command.index("--exact-max-candidates") + 1] == "0" - assert command[command.index("--exact-random-states") + 1] == "42,52,62" - assert command[command.index("--exact-checkpoint-every") + 1] == "25" - assert command[command.index("--exact-threads") + 1] == "8" - assert command[command.index("--budget-profiles") + 1] == "free" - assert command[command.index("--exact-python-executable") + 1] == "python-exact" - assert command[command.index("--cuopt-method") + 1] == "concurrent" - assert command[command.index("--cuopt-num-gpus") + 1] == "1" - assert "tolerance=tight" in command - assert "--frontier-only" in command diff --git a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py deleted file mode 100644 index db4dd2d..0000000 --- a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from scripts.experiments.run_ijds_calibration_selected_policy_challenger import ( - FORBIDDEN_SELECTOR_COLUMNS, - _funded_allocation_frame, - _measure_ex_ante_solution, - _selector_input_frame, -) -from src.optimization.policy import PolicyMode -from src.optimization.policy_evaluation import PolicyAllocationResult -from src.optimization.policy_selection import LinearPolicyCandidate - - -def test_ex_ante_measurement_contains_no_outcome_fields() -> None: - frame = pd.DataFrame( - { - "_loan_amount": [100.0, 200.0], - "_pd_point": [0.10, 0.20], - "_pd_high": [0.30, 0.50], - } - ) - candidate = LinearPolicyCandidate( - candidate_id="linear-001", - risk_tolerance=0.20, - gamma=0.50, - uncertainty_aversion=0.0, - ) - result = PolicyAllocationResult( - solution={ - "solver_status": "Optimal", - "objective_value": 10.0, - }, - allocation=np.array([1.0, 0.5]), - effective_pd=np.array([0.20, 0.35]), - policy_mode=PolicyMode.BLENDED_UNCERTAINTY, - gamma=0.50, - delta_cap_quantile=1.0, - tail_focus_quantile=1.0, - objective_risk_mode="point_pd_plus_aversion", - ) - - record = _measure_ex_ante_solution(frame, candidate, result, alpha=0.10) - - assert not FORBIDDEN_SELECTOR_COLUMNS.intersection(record) - assert record["expected_objective"] == 10.0 - assert record["weighted_pd_effective"] == 0.275 - - -def test_selector_input_frame_drops_all_outcomes() -> None: - frame = pd.DataFrame( - { - "id": ["a"], - "loan_amnt": [100.0], - "purpose": ["credit_card"], - "issue_d": ["2017-11-01"], - "default_flag": [1], - "y_true": [1], - "loan_status": ["Charged Off"], - "_outcome": [1.0], - "_pd_point": [0.1], - "_pd_low": [0.0], - "_pd_high": [0.3], - "_loan_amount": [100.0], - "_int_rate": [0.2], - } - ) - - selector = _selector_input_frame(frame) - - assert not {"default_flag", "y_true", "loan_status", "_outcome"}.intersection(selector.columns) - - -def test_funded_allocation_frame_reconciles_exposure_and_return() -> None: - frame = pd.DataFrame( - { - "id": ["a", "b"], - "issue_d": ["2020-01-01", "2020-02-01"], - "grade": ["A", "B"], - "_loan_amount": [100.0, 200.0], - "_int_rate": [0.10, 0.20], - "_outcome": [0.0, 1.0], - "_pd_point": [0.10, 0.20], - "_pd_low": [0.0, 0.0], - "_pd_high": [0.30, 0.50], - } - ) - result = PolicyAllocationResult( - solution={"solver_status": "Optimal", "objective_value": 10.0}, - allocation=np.array([1.0, 0.5]), - effective_pd=np.array([0.20, 0.35]), - policy_mode=PolicyMode.BLENDED_UNCERTAINTY, - gamma=0.50, - delta_cap_quantile=1.0, - tail_focus_quantile=1.0, - objective_risk_mode="point_pd_plus_aversion", - ) - - funded = _funded_allocation_frame(frame, result, role="selected", lgd=0.45) - - assert funded["funded_exposure"].sum() == 200.0 - assert funded["funded_weight"].sum() == 1.0 - assert funded["realized_return_contribution"].sum() == -35.0 diff --git a/tests/test_experiments/test_ijds_endpoint_availability_sensitivity.py b/tests/test_experiments/test_ijds_endpoint_availability_sensitivity.py new file mode 100644 index 0000000..f1e10b6 --- /dev/null +++ b/tests/test_experiments/test_ijds_endpoint_availability_sensitivity.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from scripts.experiments.run_ijds_endpoint_availability_sensitivity import ( + _primary_loan_facts, + _window_loan_facts, +) +from src.evaluation.policy_contrast_bounds import sharp_policy_contrast_bounds +from src.ijds_audit.evaluation import indexed_portfolio_contrasts +from src.models.binary_conformal_guardrail import BinaryOutcomeConformalRecipe + + +def _adversarial_contrast_grid() -> tuple[pd.DataFrame, pd.DataFrame]: + facts = pd.DataFrame( + { + "id": ["a", "b", "c", "d", "e", "f"], + "contractual_rate": [0.05, 0.08, 0.11, 0.13, 0.17, 0.19], + "conformal_lower": [0.0, 0.1, 0.0, 0.2, 0.0, 1.0], + "conformal_upper": [0.8, 1.0, 0.9, 0.7, 1.0, 1.0], + "snapshot_default": pd.Series([0, pd.NA, 1, 0, pd.NA, 1], dtype="Int8"), + } + ) + fact_lookup = facts.set_index("id") + specifications = { + "guardrail_p1": ("guardrail", 0.12, {"a": 40.0, "b": 60.0, "f": 0.0}), + "c0_same_numeric_cap_p1": ( + "c0_same_numeric_cap", + 0.12, + {"a": 25.0, "c": 75.0}, + ), + "c1_development_mean_p1": ( + "c1_development_mean", + 0.14, + {"b": 100.0}, + ), + "c2_contemporaneous_p1": ( + "c2_contemporaneous", + 0.16, + {"d": 35.0, "e": 65.0}, + ), + "guardrail_p2": ("guardrail", 0.18, {"c": 30.0, "d": 70.0}), + "c0_same_numeric_cap_p2": ( + "c0_same_numeric_cap", + 0.18, + {"c": 100.0}, + ), + "c1_development_mean_p2": ( + "c1_development_mean", + 0.20, + {"a": 10.0, "d": 90.0}, + ), + "c2_contemporaneous_p2": ( + "c2_contemporaneous", + 0.22, + {"e": 45.0, "f": 55.0}, + ), + "point_cap_frontier_0.1": ( + "point_cap_frontier", + 0.10, + {"a": 20.0, "e": 80.0}, + ), + "point_cap_frontier_0.2": ( + "point_cap_frontier", + 0.20, + {"b": 50.0, "c": 50.0}, + ), + "point_cap_frontier_0.3": ( + "point_cap_frontier", + 0.30, + {"d": 100.0}, + ), + } + rows: list[dict[str, object]] = [] + for policy_label, (rule, cap, exposures) in specifications.items(): + for loan_id, exposure in exposures.items(): + loan = fact_lookup.loc[loan_id] + rows.append( + { + "id": loan_id, + "role": "primary_oot", + "window_id": "w-adversarial", + "policy_label": policy_label, + "exposure": exposure, + "expected_payoff_contribution": exposure + * (float(loan["contractual_rate"]) - 0.02), + "comparator_rule": rule, + "frontier_cap": cap, + "contractual_rate": loan["contractual_rate"], + "conformal_lower": loan["conformal_lower"], + "conformal_upper": loan["conformal_upper"], + "snapshot_default": loan["snapshot_default"], + } + ) + allocations = pd.DataFrame(rows).sample(frac=1.0, random_state=712).reset_index(drop=True) + return allocations, facts + + +def _slow_contrast_grid( + allocations: pd.DataFrame, + *, + policy_ids: tuple[str, ...], + lgd: float, +) -> pd.DataFrame: + labels = { + str(label): frame + for label, frame in allocations.groupby("policy_label", observed=True, sort=False) + } + frontier = ( + allocations.loc[ + allocations["comparator_rule"].eq("point_cap_frontier"), + ["policy_label", "frontier_cap"], + ] + .drop_duplicates() + .sort_values("frontier_cap") + ) + rows: list[dict[str, object]] = [] + for policy_id in policy_ids: + guardrail = f"guardrail_{policy_id}" + comparators = ( + ("c0_same_numeric_cap", f"c0_same_numeric_cap_{policy_id}"), + ("c1_development_mean", f"c1_development_mean_{policy_id}"), + ("c2_contemporaneous", f"c2_contemporaneous_{policy_id}"), + ) + for rule, comparator in comparators: + pair = pd.concat([labels[guardrail], labels[comparator]], ignore_index=True) + rows.append( + { + "window_id": "w-adversarial", + "paired_policy_id": policy_id, + "comparator_rule": rule, + "frontier_cap": float(labels[comparator]["frontier_cap"].iloc[0]), + **sharp_policy_contrast_bounds( + pair, + policy_a=guardrail, + policy_b=comparator, + role="primary_oot", + lgd=lgd, + ), + } + ) + for item in frontier.itertuples(index=False): + comparator = str(item.policy_label) + pair = pd.concat([labels[guardrail], labels[comparator]], ignore_index=True) + rows.append( + { + "window_id": "w-adversarial", + "paired_policy_id": policy_id, + "comparator_rule": "point_cap_frontier", + "frontier_cap": float(item.frontier_cap), + **sharp_policy_contrast_bounds( + pair, + policy_a=guardrail, + policy_b=comparator, + role="primary_oot", + lgd=lgd, + ), + } + ) + return pd.DataFrame(rows) + + +def test_indexed_grid_matches_slow_public_oracle() -> None: + allocations, facts = _adversarial_contrast_grid() + policy_ids = ("p1", "p2") + + oracle = _slow_contrast_grid( + allocations, + policy_ids=policy_ids, + lgd=0.45, + ) + optimized = indexed_portfolio_contrasts( + allocations, + loan_facts=facts, + window_id="w-adversarial", + policy_ids=policy_ids, + lgd=0.45, + ) + + pd.testing.assert_frame_equal( + optimized, + oracle, + check_exact=False, + rtol=0.0, + atol=1e-15, + ) + + +def test_indexed_grid_rejects_conflicting_named_comparator_caps() -> None: + allocations, facts = _adversarial_contrast_grid() + duplicate = ( + allocations.loc[allocations["policy_label"].eq("c0_same_numeric_cap_p1")].iloc[[0]].copy() + ) + duplicate["id"] = "f" + duplicate["frontier_cap"] = 0.99 + duplicate["exposure"] = 0.0 + duplicate["expected_payoff_contribution"] = 0.0 + allocations = pd.concat([allocations, duplicate], ignore_index=True) + + with pytest.raises(RuntimeError, match="conflicting cap values"): + indexed_portfolio_contrasts( + allocations, + loan_facts=facts, + window_id="w-adversarial", + policy_ids=("p1", "p2"), + lgd=0.45, + ) + + +def _endpoint_alignment_frames() -> tuple[pd.DataFrame, pd.DataFrame]: + allocations = pd.DataFrame( + { + "id": ["a", "a", "b"], + "role": ["primary_oot"] * 3, + "period": ["2016-01", "2016-01", "2016-02"], + "contractual_rate": [0.10, 0.10, 0.15], + } + ) + outcomes = pd.DataFrame( + { + "id": ["a", "b", "unused"], + "role": ["primary_oot", "primary_oot", "policy_development"], + "period": ["2016-01", "2016-02", "2012-01"], + "snapshot_default": pd.Series([0, pd.NA, 1], dtype="Int8"), + "snapshot_resolution": ["fully_paid", "right_censored", "charged_off"], + } + ) + return allocations, outcomes + + +def test_primary_fact_join_is_exact_and_preserves_nullable_outcomes() -> None: + allocations, outcomes = _endpoint_alignment_frames() + + facts = _primary_loan_facts(allocations, outcomes) + + assert facts["id"].tolist() == ["a", "b"] + assert facts["snapshot_default"].isna().tolist() == [False, True] + + +@pytest.mark.parametrize("mutation", ["duplicate", "missing", "role", "period"]) +def test_primary_fact_join_rejects_id_or_metadata_drift(mutation: str) -> None: + allocations, outcomes = _endpoint_alignment_frames() + if mutation == "duplicate": + outcomes = pd.concat([outcomes, outcomes.iloc[[0]]], ignore_index=True) + match = "one nonmissing row per ID" + elif mutation == "missing": + outcomes = outcomes.loc[~outcomes["id"].eq("b")] + match = "missing=1" + elif mutation == "role": + outcomes.loc[outcomes["id"].eq("a"), "role"] = "policy_development" + match = "role or period" + else: + outcomes.loc[outcomes["id"].eq("a"), "period"] = "2016-03" + match = "role or period" + + with pytest.raises(RuntimeError, match=match): + _primary_loan_facts(allocations, outcomes) + + +def test_window_facts_require_complete_unique_score_alignment() -> None: + allocations, outcomes = _endpoint_alignment_frames() + base = _primary_loan_facts(allocations, outcomes) + scores = pd.DataFrame( + { + "id": ["a", "b", "unused"], + "design_split": ["primary_oot", "primary_oot", "policy_development"], + "pd_catboost_platt": [0.1, 0.8, 0.5], + } + ) + recipe = BinaryOutcomeConformalRecipe( + alpha=0.1, + requested_groups=1, + bin_edges=(0.0, 1.0), + residual_quantiles=(0.2,), + group_counts=(10,), + finite_sample_ranks=(9,), + raw_finite_sample_ranks=(9,), + ) + + facts = _window_loan_facts(base, scores, recipe) + assert facts["conformal_lower"].tolist() == pytest.approx([0.0, 0.6]) + assert facts["conformal_upper"].tolist() == pytest.approx([0.3, 1.0]) + + with pytest.raises(RuntimeError, match="missing=1"): + _window_loan_facts(base, scores.loc[~scores["id"].eq("b")], recipe) + with pytest.raises(RuntimeError, match="one nonmissing row per ID"): + _window_loan_facts( + base, + pd.concat([scores, scores.iloc[[0]]], ignore_index=True), + recipe, + ) diff --git a/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py b/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py deleted file mode 100644 index 4ab34c0..0000000 --- a/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from scripts.experiments.run_ijds_exact_alpha_grid_challenger import ( - _base_grid_frame, - _replay_differences, -) -from src.models.conformal_alpha_grid import ExactAlphaIntervals - - -def _result() -> ExactAlphaIntervals: - return ExactAlphaIntervals( - target_alpha=0.10, - used_alpha=0.095, - point=np.array([0.2, 0.4]), - low=np.array([0.0, 0.1]), - high=np.array([0.5, 0.8]), - partition_labels=pd.Series(["a", "b"]), - partition_metadata={}, - diagnostics={}, - ) - - -def test_base_grid_keeps_only_traceability_columns() -> None: - source = pd.DataFrame( - { - "_row_number": [0], - "id": ["x"], - "y_true": [0.0], - "grade": ["A"], - "pd_high_90": [0.5], - } - ) - - assert _base_grid_frame(source).columns.tolist() == [ - "_row_number", - "id", - "y_true", - "grade", - ] - - -def test_reference_replay_reports_exact_match() -> None: - source = pd.DataFrame( - { - "y_pred": [0.2, 0.4], - "pd_low_90": [0.0, 0.1], - "pd_high_90": [0.5, 0.8], - } - ) - - assert _replay_differences(_result(), source) == { - "point_max_abs": 0.0, - "low_max_abs": 0.0, - "high_max_abs": 0.0, - } diff --git a/tests/test_experiments/test_ijds_portfolio_structure_sensitivity.py b/tests/test_experiments/test_ijds_portfolio_structure_sensitivity.py new file mode 100644 index 0000000..7021469 --- /dev/null +++ b/tests/test_experiments/test_ijds_portfolio_structure_sensitivity.py @@ -0,0 +1,71 @@ +"""Locked numerical contracts for the V5 recovery source and active V6.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from scripts.experiments.run_ijds_portfolio_structure_sensitivity import ( + STATUS_V6, + _frontier_config, + _load_config, +) +from src.ijds_challengers.config import load_frontier_config + +REPO = Path(__file__).resolve().parents[2] +CONFIG_DIR = REPO / "configs/experiments" +V6_CONFIG = CONFIG_DIR / "ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml" + + +@pytest.mark.parametrize( + "name", + [ + "ijds_portfolio_structure_sensitivity_2026-07-15_v5.yaml", + "ijds_portfolio_structure_sensitivity_2026-07-15_v6.yaml", + ], +) +def test_retained_structural_protocols_keep_the_locked_grid(name: str) -> None: + config = _load_config(CONFIG_DIR / name) + + assert config["structural_grid"]["scenarios"] == 36 + assert config["claim_boundary"]["outcome_based_scenario_selection"] is False + assert config["claim_boundary"]["policy_winner"] is False + + +def test_v6_recovers_only_complete_v5_shards_and_recomputes_one() -> None: + config = _load_config(V6_CONFIG) + recovery = config["execution"]["recovery"] + + assert config["protocol_status"] == STATUS_V6 + assert config["execution"]["freeze_workers"] == 1 + assert recovery["expected_recovered_scenarios"] == 35 + assert recovery["expected_missing_scenario_ids"] == ["b0500k_p020_l025"] + assert config["lineage"]["scientific_grid_changed"] is False + assert config["lineage"]["outcomes_inspected_for_amendment"] is False + + +def test_v6_overrides_only_the_diagnosed_order_tolerance() -> None: + structural = _load_config(V6_CONFIG) + parent = load_frontier_config(REPO / structural["parent"]["frontier_config"]) + + amended = _frontier_config(parent, structural) + + assert parent["solver"]["order_exposure_distance_tolerance"] == 1.0e-10 + assert amended["solver"]["order_exposure_distance_tolerance"] == 1.0e-8 + assert amended["solver"]["order_objective_tolerance_dollars"] == 1.0e-5 + assert amended["frontier"]["normalized_score"]["minimum_endpoint_retry_slacks"] == [ + 1.0e-12, + 1.0e-10, + ] + + +def test_v6_rejects_a_larger_unlocked_order_tolerance(tmp_path: Path) -> None: + payload = yaml.safe_load(V6_CONFIG.read_text(encoding="utf-8")) + payload["numerics"]["order_exposure_distance_tolerance"] = 1.0e-7 + changed = tmp_path / "changed.yaml" + changed.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + with pytest.raises(ValueError, match="must remain 1e-8"): + _load_config(changed) diff --git a/tests/test_experiments/test_tabprep_challenger_contract.py b/tests/test_experiments/test_tabprep_challenger_contract.py deleted file mode 100644 index 15244da..0000000 --- a/tests/test_experiments/test_tabprep_challenger_contract.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.experiments.run_tabprep_challenger import ( - _assert_isolated_output, - _model_params, -) - - -def test_tabprep_output_guard_allows_only_experiment_roots() -> None: - _assert_isolated_output(Path("models/experiments/tabprep/run/model.cbm")) - _assert_isolated_output(Path("models/experiments/champion_search/run/model.cbm")) - _assert_isolated_output(Path("data/processed/experiments/tabprep/run/predictions.parquet")) - _assert_isolated_output( - Path("data/processed/experiments/champion_search/run/predictions.parquet") - ) - _assert_isolated_output(Path("reports/crpto/experiments/tabprep/run/audit.json")) - _assert_isolated_output(Path("reports/crpto/experiments/champion_search/run/audit.json")) - - with pytest.raises(ValueError, match="protected CRPTO artifact"): - _assert_isolated_output(Path("models/pd_canonical.cbm")) - - with pytest.raises(ValueError, match="outputs must stay"): - _assert_isolated_output(Path("models/not_tabprep/model.cbm")) - - -def test_model_params_keep_monotone_constraints_only_for_existing_features() -> None: - config = { - "model": { - "params": { - "iterations": 10, - "monotone_constraints": "installment:1,annual_inc:-1,dti:1", - "feature_weights": {"installment": 1.5, "missing": 2.0}, - } - }, - "tabprep": {"monotonic_constraints": {"loan_to_income": 1}}, - } - - params = _model_params( - config, - model_features=["installment", "annual_inc", "tp_ar__ratio__a__b"], - seed=52, - ) - - assert params["random_seed"] == 52 - assert params["monotone_constraints"] == "(1,-1,0)" - assert params["feature_weights"] == [1.5, 1.0, 1.0] diff --git a/tests/test_experiments/test_tabprep_feature_selection_catboost.py b/tests/test_experiments/test_tabprep_feature_selection_catboost.py deleted file mode 100644 index 03b3105..0000000 --- a/tests/test_experiments/test_tabprep_feature_selection_catboost.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from scripts.experiments.run_tabprep_feature_selection_catboost import ( - _build_case_features, - _rank_pool_features, -) -from src.features.feature_engineering import TARGET - - -def _ranking_frame(n_rows: int = 80) -> pd.DataFrame: - dti = np.linspace(3.0, 35.0, n_rows) - fico = np.linspace(760.0, 620.0, n_rows) - grade = np.where(dti > 22.0, "C", "A") - target = ((dti > 20.0) | (grade == "C")).astype(int) - target[np.arange(n_rows) % 7 == 0] = 0 - return pd.DataFrame( - { - "dti": dti, - "fico_score": fico, - "grade": grade, - "noise": np.resize([0.1, 0.2, 0.3, 0.4], n_rows), - TARGET: target, - } - ) - - -def test_pooltop_cases_fail_when_pool_ranking_is_empty() -> None: - with pytest.raises(ValueError, match="requested top 42 pool features"): - _build_case_features( - cases=["pooltop42_tab20"], - core_features=["dti"], - catboost_features=["dti", "fico_score"], - pool_features=["dti", "fico_score"], - pool_ranking=[], - woe_features=[], - generated_ranking=[f"tp_feature_{idx}" for idx in range(20)], - business_ranking=[f"tp_feature_{idx}" for idx in range(20)], - ) - - -def test_pool_feature_ranking_falls_back_when_selector_has_no_pool_scores() -> None: - frame = _ranking_frame() - train_fit = frame.iloc[:50].copy() - train_val = frame.iloc[50:].copy() - - ranking, diagnostics = _rank_pool_features( - pool_features=["dti", "fico_score", "grade", "noise"], - selector_scores={}, - train_fit=train_fit, - train_val=train_val, - categorical_features=["grade"], - ) - - assert ranking - assert {"feature", "ranking_score", "ranking_source", "iv", "psi_fit_to_val"}.issubset( - diagnostics.columns - ) - assert diagnostics["ranking_source"].eq("fallback_univariate").all() - assert set(ranking[:3]).intersection({"dti", "fico_score", "grade"}) - - -def test_pool_feature_ranking_blends_selector_scores_when_present() -> None: - frame = _ranking_frame() - train_fit = frame.iloc[:50].copy() - train_val = frame.iloc[50:].copy() - - ranking, diagnostics = _rank_pool_features( - pool_features=["dti", "fico_score", "grade", "noise"], - selector_scores={"noise": 100.0}, - train_fit=train_fit, - train_val=train_val, - categorical_features=["grade"], - ) - - assert ranking[0] == "noise" - assert diagnostics.loc[diagnostics["feature"] == "noise", "ranking_source"].iloc[0] == ( - "selector_blend" - ) diff --git a/tests/test_features/test_feature_config_equivalence.py b/tests/test_features/test_feature_config_equivalence.py deleted file mode 100644 index 9652683..0000000 --- a/tests/test_features/test_feature_config_equivalence.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Equivalence tests for feature_config representations.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import pytest - -from src.features.feature_config_io import ( - DEFAULT_PARQUET_PATH, - DEFAULT_YAML_PATH, - load_feature_config, -) - -CORE_EQUIVALENCE_KEYS = ( - "CATBOOST_FEATURES", - "CATEGORICAL_FEATURES", - "LOGREG_FEATURES", - "NUMERIC_FEATURES", - "FLAG_FEATURES", - "WOE_FEATURES", - "INTERACTION_FEATURES", - "CHALLENGER_FEATURE_POOL_V2", - "CORE_FEATURE_SET_V2", - "MISSINGNESS_INDICATORS", - "iv_scores", - "schema_version", -) - -OPTIONAL_EQUIVALENCE_KEYS = ( - "binning", - "BINNING", - "monotone_constraints", - "MONOTONE_CONSTRAINTS", -) - - -def _skip_if_missing(path: Path) -> None: - if not path.is_file(): - pytest.skip(f"{path} not available locally.") - - -@pytest.mark.integration -def test_frozen_feature_config_yaml_parquet_equivalence() -> None: - """The champion Parquet table must preserve the YAML feature contract.""" - _skip_if_missing(DEFAULT_YAML_PATH) - _skip_if_missing(DEFAULT_PARQUET_PATH) - - yaml_cfg = load_feature_config(prefer="yaml") - parquet_cfg = load_feature_config(prefer="parquet") - - assert set(parquet_cfg) == set(yaml_cfg) - for key in CORE_EQUIVALENCE_KEYS: - assert parquet_cfg[key] == yaml_cfg[key] - - for key in OPTIONAL_EQUIVALENCE_KEYS: - if key in yaml_cfg or key in parquet_cfg: - assert parquet_cfg.get(key) == yaml_cfg.get(key) - - -def test_loader_preserves_explicit_pickle_escape_hatch(tmp_path: Path) -> None: - """Consumers can still force the legacy pickle when auditing format drift.""" - pkl = tmp_path / "feature_config.pkl" - yml = tmp_path / "feature_config.yml" - import joblib - import yaml - - joblib.dump({"CATBOOST_FEATURES": ["from_pickle"]}, pkl) - yml.write_text(yaml.safe_dump({"CATBOOST_FEATURES": ["from_yaml"]}), encoding="utf-8") - - yaml_cfg: dict[str, Any] = load_feature_config(pickle_path=pkl, yaml_path=yml) - pickle_cfg: dict[str, Any] = load_feature_config( - pickle_path=pkl, - yaml_path=yml, - prefer="pickle", - ) - - assert yaml_cfg["CATBOOST_FEATURES"] == ["from_yaml"] - assert pickle_cfg["CATBOOST_FEATURES"] == ["from_pickle"] diff --git a/tests/test_features/test_feature_config_io.py b/tests/test_features/test_feature_config_io.py deleted file mode 100644 index e6eeb10..0000000 --- a/tests/test_features/test_feature_config_io.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Round-trip tests for ``src.features.feature_config_io``.""" - -from __future__ import annotations - -from pathlib import Path - -import joblib -import pytest -import yaml - -from src.features.feature_config_io import ( - DEFAULT_PICKLE_PATH, - load_feature_config, - pickle_to_yaml, - save_feature_config, -) - -SAMPLE_CONFIG: dict = { - "NUMERIC_FEATURES": ["loan_amnt", "annual_inc", "dti"], - "CATEGORICAL_FEATURES": ["grade", "purpose"], - "WOE_FEATURES": ["grade_woe", "purpose_woe"], - "iv_scores": {"grade": 0.42, "purpose": 0.18}, - "schema_version": "v2", -} - - -def test_save_then_load_round_trip(tmp_path: Path) -> None: - save_feature_config(SAMPLE_CONFIG, repo_root=tmp_path) - loaded = load_feature_config(repo_root=tmp_path) - assert loaded == SAMPLE_CONFIG - - -def test_load_prefers_yaml_over_pickle(tmp_path: Path) -> None: - yaml_target = tmp_path / "data" / "processed" / "feature_config.yml" - pkl_target = tmp_path / "data" / "processed" / "feature_config.pkl" - yaml_target.parent.mkdir(parents=True, exist_ok=True) - yaml_target.write_text(yaml.safe_dump({"NUMERIC_FEATURES": ["from_yaml"]})) - joblib.dump({"NUMERIC_FEATURES": ["from_pickle"]}, pkl_target) - loaded = load_feature_config(repo_root=tmp_path) - assert loaded["NUMERIC_FEATURES"] == ["from_yaml"] - - -def test_load_auto_falls_back_to_pickle_when_yaml_absent(tmp_path: Path) -> None: - pkl_target = tmp_path / "data" / "processed" / "feature_config.pkl" - pkl_target.parent.mkdir(parents=True, exist_ok=True) - joblib.dump({"NUMERIC_FEATURES": ["from_pickle"]}, pkl_target) - loaded = load_feature_config(repo_root=tmp_path, prefer="auto") - assert loaded["NUMERIC_FEATURES"] == ["from_pickle"] - - -def test_load_prefer_yaml_does_not_fall_back_to_pickle(tmp_path: Path) -> None: - pkl_target = tmp_path / "data" / "processed" / "feature_config.pkl" - pkl_target.parent.mkdir(parents=True, exist_ok=True) - joblib.dump({"NUMERIC_FEATURES": ["from_pickle"]}, pkl_target) - with pytest.raises(FileNotFoundError): - load_feature_config(repo_root=tmp_path, prefer="yaml") - - -def test_load_prefer_yaml_raises_when_missing(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - load_feature_config(repo_root=tmp_path, prefer="yaml") - - -def test_load_prefer_pickle_raises_when_missing(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - load_feature_config(repo_root=tmp_path, prefer="pickle") - - -def test_pickle_to_yaml_round_trip(tmp_path: Path) -> None: - pkl = tmp_path / "src.pkl" - yml = tmp_path / "dst.yml" - joblib.dump(SAMPLE_CONFIG, pkl) - pickle_to_yaml(pickle_path=pkl, yaml_path=yml) - loaded = yaml.safe_load(yml.read_text(encoding="utf-8")) - assert loaded == SAMPLE_CONFIG - - -def test_save_with_parquet_round_trip(tmp_path: Path) -> None: - save_feature_config(SAMPLE_CONFIG, repo_root=tmp_path, also_parquet=True) - yml = tmp_path / "data" / "processed" / "feature_config.yml" - parquet = tmp_path / "data" / "processed" / "feature_config.parquet" - - assert yml.is_file() - assert parquet.is_file() - assert load_feature_config(repo_root=tmp_path, prefer="parquet") == SAMPLE_CONFIG - - -def test_save_also_pickle_writes_both(tmp_path: Path) -> None: - save_feature_config(SAMPLE_CONFIG, repo_root=tmp_path, also_pickle=True) - yml = tmp_path / "data" / "processed" / "feature_config.yml" - pkl = tmp_path / "data" / "processed" / "feature_config.pkl" - assert yml.is_file() - assert pkl.is_file() - assert joblib.load(pkl) == SAMPLE_CONFIG - - -def test_save_rejects_unknown_prefer_value(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="prefer must be"): - load_feature_config(repo_root=tmp_path, prefer="json") - - -# --------------------------------------------------------------------------- -# Live champion artefact (optional — skipped if not present locally) -# --------------------------------------------------------------------------- - - -@pytest.mark.integration -def test_champion_pickle_yaml_round_trip(tmp_path: Path) -> None: - """Legacy pickle audits can still round-trip to YAML when a pickle exists.""" - if not DEFAULT_PICKLE_PATH.is_file(): - pytest.skip(f"{DEFAULT_PICKLE_PATH} not available locally.") - original = joblib.load(DEFAULT_PICKLE_PATH) - yml = tmp_path / "feature_config.yml" - pickle_to_yaml(pickle_path=DEFAULT_PICKLE_PATH, yaml_path=yml) - round_tripped = yaml.safe_load(yml.read_text(encoding="utf-8")) - assert round_tripped == original diff --git a/tests/test_features/test_feature_engineering_contract.py b/tests/test_features/test_feature_engineering_contract.py index 0d96de2..1f395db 100644 --- a/tests/test_features/test_feature_engineering_contract.py +++ b/tests/test_features/test_feature_engineering_contract.py @@ -12,12 +12,29 @@ TARGET, build_feature_config, build_feature_manifest, + normalize_raw_columns, ) CONTRACT_PATH = Path("models/pd_model_contract.json") EXPECTED_CONTRACT_OMISSIONS = {"rev_utilization", "high_util_pct"} +def test_lending_club_credit_line_dates_use_the_declared_month_year_format() -> None: + normalized = normalize_raw_columns( + pd.DataFrame( + { + "issue_d": ["2016-04-01", "2016-05-01"], + "earliest_cr_line": ["Jan-1985", "Sep-2007"], + } + ) + ) + + assert normalized["earliest_cr_line"].tolist() == [ + pd.Timestamp("1985-01-01"), + pd.Timestamp("2007-09-01"), + ] + + def _load_contract() -> dict[str, object]: if not CONTRACT_PATH.is_file(): pytest.skip(f"{CONTRACT_PATH} is not available in this checkout.") @@ -26,14 +43,20 @@ def _load_contract() -> dict[str, object]: def test_frozen_pd_contract_matches_current_catboost_feature_order() -> None: contract = _load_contract() - feature_names = list(contract["feature_names"]) # type: ignore[index] - categorical = list(contract["categorical_features"]) # type: ignore[index] + feature_names = contract["feature_names"] + categorical = contract["categorical_features"] + n_features = contract["n_features"] + assert isinstance(feature_names, list) + assert isinstance(categorical, list) + assert isinstance(n_features, int) + assert all(isinstance(feature, str) for feature in feature_names) + assert all(isinstance(feature, str) for feature in categorical) expected_features = [ feature for feature in CATBOOST_FEATURES if feature not in EXPECTED_CONTRACT_OMISSIONS ] - assert int(contract["n_features"]) == 42 # type: ignore[index] + assert n_features == 42 assert len(feature_names) == 42 assert feature_names == expected_features assert categorical == CATEGORICAL_FEATURES @@ -42,7 +65,9 @@ def test_frozen_pd_contract_matches_current_catboost_feature_order() -> None: def test_feature_config_materializes_the_champion_feature_contract() -> None: contract = _load_contract() - feature_names = list(contract["feature_names"]) # type: ignore[index] + feature_names = contract["feature_names"] + assert isinstance(feature_names, list) + assert all(isinstance(feature, str) for feature in feature_names) frame = pd.DataFrame( { **{feature: [1, 2, 3] for feature in feature_names}, diff --git a/tests/test_features/test_schemas.py b/tests/test_features/test_schemas.py deleted file mode 100644 index 9fa37b7..0000000 --- a/tests/test_features/test_schemas.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Tests for the Pandera schemas in :mod:`src.features.schemas`. - -Both the dict-based ``DataFrameSchema`` and the class-based ``DataFrameModel`` -forms must accept canonical CRPTO outputs and reject the common violations -(non-monotone PD intervals, out-of-range probabilities). -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -import pandera.errors -import pytest - -from src.features import schemas as cs - -# --------------------------------------------------------------------------- -# Loan master -# --------------------------------------------------------------------------- - - -def _valid_loan_master(n: int = 8) -> pd.DataFrame: - rng = np.random.default_rng(0) - return pd.DataFrame( - { - "loan_amnt": rng.uniform(1000, 35_000, n), - "annual_inc": rng.uniform(30_000, 200_000, n), - "loan_to_income": rng.uniform(0.01, 1.0, n), - "dti": rng.uniform(0.5, 35.0, n), - "default_flag": rng.integers(0, 2, n), - "int_rate": rng.uniform(5.0, 25.0, n), - } - ) - - -def test_loan_master_accepts_canonical_frame() -> None: - df = _valid_loan_master() - out = cs.validate_loan_master(df) - assert len(out) == len(df) - - -def test_loan_master_rejects_invalid_default_flag() -> None: - df = _valid_loan_master() - df.loc[0, "default_flag"] = 7 # not in {0, 1} - with pytest.raises(pandera.errors.SchemaError): - cs.validate_loan_master(df) - - -# --------------------------------------------------------------------------- -# Prediction (dict-based + class-based) -# --------------------------------------------------------------------------- - - -def _valid_predictions(n: int = 8) -> pd.DataFrame: - rng = np.random.default_rng(1) - pd_point = rng.uniform(0.05, 0.4, n) - delta = rng.uniform(0.0, 0.05, n) - return pd.DataFrame( - { - "pd_point": pd_point, - "pd_low": np.clip(pd_point - delta, 0.0, 1.0), - "pd_high": np.clip(pd_point + delta, 0.0, 1.0), - } - ) - - -def test_prediction_schema_accepts_monotone_intervals() -> None: - df = _valid_predictions() - out = cs.validate_predictions(df) - assert len(out) == len(df) - - -def test_prediction_schema_rejects_inverted_bounds() -> None: - df = _valid_predictions() - df.loc[0, "pd_low"] = df.loc[0, "pd_high"] + 0.05 # invert - with pytest.raises(pandera.errors.SchemaError): - cs.validate_predictions(df) - - -def test_prediction_dataframe_model_accepts_monotone_intervals() -> None: - df = _valid_predictions() - out = cs.PredictionOutputModel.validate(df) - assert len(out) == len(df) - - -def test_prediction_dataframe_model_rejects_out_of_range_probabilities() -> None: - df = _valid_predictions() - df.loc[0, "pd_high"] = 1.5 - with pytest.raises(pandera.errors.SchemaError): - cs.PredictionOutputModel.validate(df) - - -# --------------------------------------------------------------------------- -# Conformal output -# --------------------------------------------------------------------------- - - -def _valid_conformal(n: int = 8) -> pd.DataFrame: - rng = np.random.default_rng(2) - y_pred = rng.uniform(0.05, 0.4, n) - width = rng.uniform(0.0, 0.1, n) - return pd.DataFrame( - { - "y_pred": y_pred, - "pd_low_90": np.clip(y_pred - width / 2, 0.0, 1.0), - "pd_high_90": np.clip(y_pred + width / 2, 0.0, 1.0), - "grade": rng.choice(["A", "B", "C"], n), - "width_90": width, - } - ) - - -def test_conformal_schema_accepts_canonical_frame() -> None: - df = _valid_conformal() - out = cs.validate_conformal_output(df) - assert len(out) == len(df) - - -def test_conformal_dataframe_model_round_trip() -> None: - df = _valid_conformal() - out = cs.ConformalOutputModel.validate(df) - assert len(out) == len(df) - - -def test_conformal_schema_rejects_negative_width() -> None: - df = _valid_conformal() - df.loc[0, "width_90"] = -0.05 - with pytest.raises(pandera.errors.SchemaError): - cs.validate_conformal_output(df) diff --git a/tests/test_features/test_tabprep_challenger.py b/tests/test_features/test_tabprep_challenger.py deleted file mode 100644 index 1c74761..0000000 --- a/tests/test_features/test_tabprep_challenger.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd - -from src.features.feature_engineering import TARGET -from src.features.tabprep_challenger import ( - TabPrepChallengerTransformer, - TabPrepVariantConfig, - resolve_tabprep_categorical_features, - resolve_tabprep_input_features, -) - - -def _fixture_frame(n_rows: int = 80) -> pd.DataFrame: - rng = np.random.default_rng(123) - dti = np.linspace(4.0, 34.0, n_rows) - grade = np.where(dti > 20, "C", "A") - purpose = np.where(np.arange(n_rows) % 3 == 0, "debt_consolidation", "credit_card") - target = ((dti > 22) | (purpose == "debt_consolidation")).astype(int) - return pd.DataFrame( - { - "id": np.arange(n_rows), - "issue_d": pd.date_range("2016-01-01", periods=n_rows, freq="MS"), - "loan_amnt": np.linspace(1_000.0, 20_000.0, n_rows), - "annual_inc": np.linspace(30_000.0, 120_000.0, n_rows), - "dti": dti, - "installment": np.linspace(75.0, 700.0, n_rows), - "open_acc": rng.integers(1, 12, size=n_rows), - "has_recent_inq": rng.integers(0, 2, size=n_rows), - "inq_last_12m__is_missing": rng.integers(0, 2, size=n_rows), - "grade": grade, - "purpose": purpose, - "verification_status": np.where( - np.arange(n_rows) % 2 == 0, "Verified", "Source Verified" - ), - "term": np.where(np.arange(n_rows) % 2 == 0, "36", "60"), - "loan_status": np.where(target == 1, "Charged Off", "Fully Paid"), - "url": [f"https://example.invalid/{idx}" for idx in range(n_rows)], - TARGET: target, - } - ) - - -def _feature_config() -> dict[str, object]: - return { - "CHALLENGER_FEATURE_POOL_V2": [ - "id", - "loan_amnt", - "annual_inc", - "dti", - "installment", - "open_acc", - "has_recent_inq", - "inq_last_12m__is_missing", - "grade", - "purpose", - "verification_status", - "term", - "loan_status", - "url", - ], - "CATEGORICAL_FEATURES": ["grade", "purpose", "verification_status", "term"], - "INTERACTION_FEATURES": [], - } - - -def _small_variant() -> TabPrepVariantConfig: - return TabPrepVariantConfig( - name="unit", - max_generated_features=24, - arithmetic_features=8, - groupby_features=5, - target_encoding_features=3, - interaction_encoding_features=4, - rsfc_features=4, - max_numeric_base_features=5, - max_groupby_numeric_features=4, - max_categorical_base_features=3, - max_scoring_rows=80, - n_oof_folds=4, - min_group_support=2, - ) - - -def test_resolve_tabprep_input_features_filters_forbidden_columns() -> None: - frame = _fixture_frame() - - features = resolve_tabprep_input_features(frame, feature_config=_feature_config()) - - assert "loan_status" not in features - assert "url" not in features - assert "id" not in features - assert "verification_status" in features - assert "grade" in features - assert "dti" in features - - -def test_tabprep_transformer_is_deterministic_and_split_aligned() -> None: - train = _fixture_frame() - test = _fixture_frame(20).assign(grade="Z") - features = resolve_tabprep_input_features(train, feature_config=_feature_config()) - categorical = resolve_tabprep_categorical_features(features, feature_config=_feature_config()) - - first = TabPrepChallengerTransformer( - variant=_small_variant(), - input_features=features, - categorical_features=categorical, - random_state=7, - ) - first_train = first.fit_transform(train, train[TARGET], issue_dates=train["issue_d"]) - first_test = first.transform(test) - - second = TabPrepChallengerTransformer( - variant=_small_variant(), - input_features=features, - categorical_features=categorical, - random_state=7, - ) - second_train = second.fit_transform(train, train[TARGET], issue_dates=train["issue_d"]) - second_test = second.transform(test) - - assert list(first_train.columns) == list(first_test.columns) - assert first_train.shape[1] > 0 - assert not np.isinf(first_train.to_numpy(dtype=float)).any() - assert not np.isinf(first_test.to_numpy(dtype=float)).any() - pd.testing.assert_frame_equal(first_train, second_train) - pd.testing.assert_frame_equal(first_test, second_test) - - -def test_transform_ignores_target_column_in_new_splits() -> None: - train = _fixture_frame() - features = resolve_tabprep_input_features(train, feature_config=_feature_config()) - categorical = resolve_tabprep_categorical_features(features, feature_config=_feature_config()) - transformer = TabPrepChallengerTransformer( - variant=_small_variant(), - input_features=features, - categorical_features=categorical, - random_state=11, - ) - transformer.fit_transform(train, train[TARGET], issue_dates=train["issue_d"]) - - cal_a = _fixture_frame(20) - cal_b = cal_a.copy() - cal_a[TARGET] = 0 - cal_b[TARGET] = 1 - - pd.testing.assert_frame_equal(transformer.transform(cal_a), transformer.transform(cal_b)) - - -def test_generated_manifest_contains_provenance_without_forbidden_sources() -> None: - train = _fixture_frame() - features = resolve_tabprep_input_features(train, feature_config=_feature_config()) - categorical = resolve_tabprep_categorical_features(features, feature_config=_feature_config()) - transformer = TabPrepChallengerTransformer( - variant=_small_variant(), - input_features=features, - categorical_features=categorical, - random_state=5, - ) - transformer.fit_transform(train, train[TARGET], issue_dates=train["issue_d"]) - - manifest = transformer.feature_manifest() - - assert {"feature", "generator", "operation", "source_features"}.issubset(manifest.columns) - assert not manifest["source_features"].str.contains("loan_status|url|id").any() diff --git a/tests/test_ijds_active_claim_sync.py b/tests/test_ijds_active_claim_sync.py index 29c95c0..1230c98 100644 --- a/tests/test_ijds_active_claim_sync.py +++ b/tests/test_ijds_active_claim_sync.py @@ -1,153 +1,463 @@ -"""Drift guard for the active calibration-selected IJDS policy.""" +"""Drift guards for the active IJDS evidence and manuscript surfaces.""" from __future__ import annotations -import csv +import hashlib import json +import re from pathlib import Path from typing import Any import pytest +from scripts.build_ijds_submission_tex import render_submission_tex + REPO = Path(__file__).resolve().parents[1] -TABLES = REPO / "reports/crpto/tables" -RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" -GOVERNANCE = ( - REPO / "models/experiments/champion_reopen" / RUN_TAG / "portfolio/ijds_policy_governance.json" -) -SUMMARY = GOVERNANCE.with_name("calibration_selected_policy_summary.json") +EVIDENCE = REPO / "reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json" +RUN = "ijds-binary-geometry-frontier-v4-2026-07-15-v5" +COMMIT = "e2bba580a0b07c145bd64ff61440973d6e31349b" SURFACES = ( REPO / "paper/CRPTO_ijds.qmd", REPO / "paper/supplement_ijds.qmd", REPO / "paper/submission/CRPTO_ijds_submission.tex", ) -TABLE_STEMS = ( - "crpto_tableA35_exact_alpha_grid", - "crpto_tableA36_calibration_policy_selector", - "crpto_tableA37_calibration_selected_temporal_evaluation", - "crpto_tableA38_calibration_selected_grade_audit", - "crpto_tableA39_calibration_selected_bootstrap", - "crpto_tableA40_calibration_selected_point_baseline", -) def _json(path: Path) -> dict[str, Any]: - assert path.is_file(), f"Missing active governance: {path}" + assert path.is_file(), path return json.loads(path.read_text(encoding="utf-8")) -def _rows(stem: str) -> list[dict[str, str]]: - with (TABLES / f"{stem}.csv").open(encoding="utf-8") as handle: - return list(csv.DictReader(handle)) - - -def _surface_text(path: Path) -> str: - return path.read_text(encoding="utf-8").replace(r"\$", "$").replace("{,}", ",") - - -def test_active_governance_locks_simple_policy_and_selector() -> None: - payload = _json(GOVERNANCE) - summary = _json(SUMMARY) - policy = payload["selected_policy"] - selector = payload["selection_protocol"] - - assert payload["status"] == "active_ijds_policy" - assert payload["run_tag"] == RUN_TAG - assert payload["generated_at_utc"] == summary["generated_at_utc"] - assert policy["policy_mode"] == "blended_uncertainty" - assert policy["risk_tolerance"] == pytest.approx(0.17) - assert policy["gamma"] == pytest.approx(0.50) - assert policy["uncertainty_aversion"] == pytest.approx(0.0) - assert policy["min_budget_utilization"] == pytest.approx(0.0) - assert selector["min_budget_utilization"] == pytest.approx(0.999) - assert selector["n_total"] == 9 - assert selector["n_eligible"] == 5 - assert selector["outcome_columns_used"] == 0 - assert selector["statistical_assumption_columns_used"] == 0 - assert selector["endpoint_budget_cap"] == pytest.approx(0.28) - assert selector["selector_forbidden_columns_present"] == [] - assert selector["calibration_metadata"]["target_alpha"] == pytest.approx(0.10) - assert selector["calibration_metadata"]["selection_period"] == "2017-11" - assert selector["calibration_metadata"]["selection_rows"] == 14943 - assert selector["calibration_metadata"]["audit_period"] == "2017-12" - assert selector["calibration_metadata"]["audit_rows"] == 20695 - assert selector["calibration_metadata"]["outcomes_isolated_until_post_selection_audit"] is True - assert "_outcome" not in selector["selector_input_columns"] - assert selector["endpoint_cap_stability"]["cap_lower_inclusive"] == pytest.approx( - 0.25903604939435104 - ) - assert selector["endpoint_cap_stability"]["cap_upper_exclusive"] == pytest.approx( - 0.29049078888716334 - ) - assert selector["calibration_audit"]["same_policy_selected"] is True - assert selector["calibration_audit"]["selected_policy"][ - "weighted_miscoverage" - ] == pytest.approx(0.124925) - assert payload["exact_alpha_reference_replay"]["pass"] is True - - -def test_active_tables_agree_with_governance() -> None: - payload = _json(GOVERNANCE) - full = payload["full_oot"] - selected = next( - row - for row in _rows("crpto_tableA40_calibration_selected_point_baseline") - if row["policy"] == "Calibration-selected 50/50 CRPTO" +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _normalize(text: str) -> str: + value = text.lower() + for old, new in { + r"\$": "$", + r"\%": "%", + r"\_": "_", + "{,}": ",", + "{": "", + "}": "", + "`": "", + }.items(): + value = value.replace(old, new) + return re.sub(r"\s+", " ", value) + + +def test_active_evidence_locks_v4_lineage_and_claim_boundary() -> None: + evidence = _json(EVIDENCE) + + assert evidence["status"] == "active_ijds_v5_endpoint_reason_audited_paper_facing_evidence" + assert evidence["run_tag"] == RUN + assert evidence["protocol_commit"] == COMMIT + assert evidence["claim_boundary"] == { + "previously_inspected_archive": True, + "confirmatory": False, + "prospective": False, + "causal": False, + "selected_set_validity": False, + "policy_winner": False, + "nested_scopes_are_independent_replications": False, + } + assert evidence["protected_stages_run"] == [] + assert evidence["protected_artifacts_written"] == [] + + +def test_active_design_is_exact() -> None: + evidence = _json(EVIDENCE) + design = evidence["design"] + + assert design == { + "primary_oot_candidates": 376890, + "primary_oot_resolved": 364814, + "primary_oot_unresolved": 12076, + "residual_windows": 8, + "learners": 5, + "v4_detailed_coverage_learners": 2, + "credit_control_learners": 5, + "portfolio_learners": 1, + "taxonomy_diagnostics": [1, 2, 5, 10], + "policies": 9, + "v4_policies_are_supporting_not_closed_family": True, + "oot_months": 15, + "development_months": 11, + "two_ruler_gamma_grid": [0.0, 0.25, 0.5, 0.75, 1.0], + "two_ruler_primary_contrast": "gamma_1_minus_gamma_0", + "two_ruler_interior_coordinates": [0.25, 0.5, 0.75], + "two_ruler_tracks": 6, + "frontier_caps": 3067, + "development_support_lower": pytest.approx(0.0555726278946077), + "development_support_upper": pytest.approx(0.09999720664228194), + "evaluation_endpoint": "terminal_default_reconstructed_as_observable_by_2020-09-30", + "archive_is_verified_point_in_time_snapshot": False, + } + + +def test_full_data_contract_credit_controls_and_coverage_are_exact() -> None: + evidence = _json(EVIDENCE) + data = evidence["data_contract"] + controls = evidence["credit_risk_controls"] + coverage = evidence["coverage"] + + assert data["raw_rows"] == 2925493 + assert data["valid_loan_rows"] == 2925492 + assert data["term36_rows_all_dates"] == 2060077 + assert data["term60_rows_all_dates"] == 865415 + assert data["active_design_rows"] == 640543 + assert data["raw_schema_columns"] == 142 + assert data["eligible_raw_features"] == 30 + assert data["declared_coverage_exceptions"] == 2 + assert data["coverage_exceptions_requiring_sensitivity"] == 2 + assert data["late_schema_features"] == 48 + assert data["sampling"] == "none_all_eligible_rows_within_each_declared_temporal_role" + + assert controls["all_five_all_eight_upper_below_nominal"] is True + assert controls["controls_enter_portfolio_optimization"] is False + assert controls["model_or_feature_selected_from_oot"] is False + assert controls["scorecard_superiority_claim_authorized"] is False + rows = {row["learner"]: row for row in controls["rows"]} + assert set(rows) == { + "catboost_platt", + "numeric_logistic_platt", + "catboost_monotonic_platt", + "woe_scorecard_platform_platt", + "woe_scorecard_borrower_platt", + } + assert all(row["windows_upper_below_0_90"] == 8 for row in rows.values()) + assert rows["catboost_monotonic_platt"]["roc_auc"] == pytest.approx(0.6519537792141734) + assert rows["woe_scorecard_borrower_platt"]["coverage_upper_max"] == pytest.approx( + 0.8977261269866539 ) - alpha = next( - row - for row in _rows("crpto_tableA35_exact_alpha_grid") - if row["selected_for_policy"] == "True" + assert controls["calibration"]["optimizer_success_rows"] == 30 + assert controls["calibration"]["all_primary_oot_mean_calibration_error_negative"] is True + assert controls["calibration"]["all_primary_oot_slopes_below_one"] is True + assert controls["woe_iv"]["optbinning_problems"] == 45 + assert controls["woe_iv"]["all_optimal"] is True + assert controls["temporal_shift"]["primary_oot_score_psi"][ + "woe_scorecard_borrower_platt" + ] == pytest.approx(0.07233216453444681) + assert coverage["catboost_all_eight_upper_below_nominal"] is True + assert coverage["logistic_all_eight_upper_below_nominal"] is True + assert coverage["catboost_bound_min"] == pytest.approx(0.8424845445620738) + assert coverage["catboost_bound_max"] == pytest.approx(0.8825970442304121) + assert coverage["logistic_bound_min"] == pytest.approx(0.8500305128817427) + assert coverage["logistic_bound_max"] == pytest.approx(0.8962217092520364) + rows = coverage["rows"] + assert len(rows) == 16 + assert {row["learner"] for row in rows} == { + "catboost_platt", + "numeric_logistic_platt", + } + assert {row["window_id"] for row in rows} == { + f"w{index:02d}_" + suffix + for index, suffix in enumerate( + ( + "2012m01_m06", + "2012m02_m07", + "2012m03_m08", + "2012m04_m09", + "2012m05_m10", + "2012m06_m11", + "2012m07_m12", + "2012m08_2013m01", + ), + start=1, + ) + } + + +def test_phase_transition_and_portfolio_boundary_are_exact() -> None: + evidence = _json(EVIDENCE) + phase = evidence["binary_phase_transition"] + portfolio = evidence["portfolio"] + + assert phase["stratum"] == 2 + assert phase["w7_fit_prevalence"] == pytest.approx(0.10170349131388093) + assert phase["w8_fit_prevalence"] == pytest.approx(0.0971465213209362) + assert phase["w7_residual_quantile"] == pytest.approx(0.8884345991499274) + assert phase["w8_residual_quantile"] == pytest.approx(0.1118010883671265) + assert phase["w7_mean_width"] == pytest.approx(0.9842633701640714) + assert phase["w8_mean_width"] == pytest.approx(0.2076312400549422) + assert phase["w8_oot_coverage_bound"] == pytest.approx([0.8225359596189609, 0.8547066934861538]) + lag = phase["label_lag_sensitivity"] + assert lag["admissible_lags_months"] == [0, 3, 6] + assert lag["nonadmissible_lags_months"] == [8, 12] + assert lag["w7_to_w8_threshold_crossing_at_all_admissible_lags"] is True + assert lag["crossing_disappears_outside_locked_retention_scope"] is True + assert lag["causal_interpretation_authorized"] is False + + assert portfolio["c2_cells"] == 1080 + assert portfolio["c2_match_residual_abs_max"] < 1e-16 + assert portfolio["c2_point_minus_guardrail_objective_min"] > -1e-5 + assert portfolio["broad_stress_all_envelopes_cross_zero"] is True + assert portfolio["broad_stress_cells"] == 216 + assert portfolio["w8_development_all_envelopes_cross_zero"] is True + counts = { + (row["metric"], row["direction"]): row["cells"] + for row in portfolio["development_direction_counts"] + } + assert counts == { + ("funded_miscoverage", "crosses_zero"): 45, + ("funded_miscoverage", "guardrail_higher"): 27, + ("standardized_payoff", "crosses_zero"): 66, + ("standardized_payoff", "guardrail_lower"): 6, + ("terminal_default", "crosses_zero"): 72, + } + tie = portfolio["evaluated_point_cap_solver_stability"] + assert tie["point_cap_rows"] == 7297 + assert tie["near_zero_bases"] == 0 + assert tie["tie_sensitive_rows"] == 0 + assert tie["continuous_frontier_uniqueness_claim"] is False + + +def test_two_ruler_diagnostic_is_finite_complete_and_nonselective() -> None: + challenger = _json(EVIDENCE)["decision_challenger"] + + assert challenger["scope"] == "finite_two_ruler_three_interior_coordinate_diagnostic" + assert challenger["continuous_frontier_claim"] is False + assert challenger["tracks_are_independent_replications"] is False + assert challenger["endpoint_contrast"] == "gamma_1_minus_gamma_0" + assert challenger["counts"] == { + "evaluated_portfolios": 6240, + "joined_funded_rows": 622455, + "window_endpoint_contrasts": 48, + "monthly_endpoint_contrasts": 720, + "metric_direction_cells": 144, + "outcome_audit_rows": 8, + } + assert challenger["primary_oot_unresolved"] == 12076 + assert challenger["manifest"]["sha256"] == ( + "9ee55a2522349c8520f308bc69273774dd48964847dfd340b78a7be46474cd7f" ) - assert float(selected["realized_return"]) == pytest.approx(full["realized_return"]) - assert float(selected["weighted_outcome"]) == pytest.approx(full["weighted_default_rate"]) - assert float(selected["weighted_miscoverage"]) == pytest.approx(full["weighted_miscoverage"]) - assert float(selected["endpoint_budget"]) == pytest.approx(full["endpoint_budget"]) - assert float(selected["markov_loss_threshold"]) == pytest.approx(full["markov_loss_threshold"]) - assert float(alpha["target_alpha"]) == pytest.approx(0.10) - assert float(alpha["empirical_coverage"]) == pytest.approx(0.9348356081757077) - - -def test_active_a35_to_a40_exist_in_csv_and_tex() -> None: - missing = [ - f"{stem}.{suffix}" - for stem in TABLE_STEMS - for suffix in ("csv", "tex") - if not (TABLES / f"{stem}.{suffix}").is_file() - ] - assert not missing, "Missing active IJDS evidence: " + ", ".join(missing) - - -def test_active_manuscript_surfaces_share_numeric_anchors() -> None: - payload = _json(GOVERNANCE) - full = payload["full_oot"] - contrast = payload["point_pd_contrast"] - anchors = ( - f"${full['realized_return']:,.2f}", - f"{full['weighted_default_rate']:.6f}", - f"{full['weighted_miscoverage']:.6f}", - f"{full['Gamma_CP']:.6f}", - f"{full['Gamma_residual']:.6f}", - f"{full['endpoint_budget']:.6f}", - f"{full['observed_accounting_bound']:.6f}", - f"{full['markov_loss_threshold']:.6f}", - f"{contrast['realized_return']:,.2f}", - f"{100 * contrast['selected_return_cost_pct']:.3f}", - f"{100 * contrast['selected_default_reduction']:.4f}", - "0.28", - "0.124925", - "$163,421.14", + rows = {(row["ruler"], row["coordinate"]): row for row in challenger["rows"]} + assert set(rows) == { + (ruler, coordinate) + for ruler in ("objective_matched", "normalized_score") + for coordinate in (0.25, 0.5, 0.75) + } + quarter = rows[("objective_matched", 0.25)] + assert quarter["active_months_per_window_min"] == 4 + assert quarter["active_months_per_window_max"] == 4 + assert quarter["payoff_bound_usd_lower_min"] == pytest.approx(-9134.339201705214) + assert quarter["payoff_bound_usd_upper_max"] == pytest.approx(5603.660798333496) + assert quarter["default_bound_pp_upper_max"] == pytest.approx(0.12654340602615935) + assert quarter["payoff_direction_pattern"] == "crosses_zero:8" + assert quarter["default_direction_pattern"] == "crosses_zero:8" + assert quarter["miscoverage_direction_pattern"] == "crosses_zero:8" + + half = rows[("objective_matched", 0.5)] + assert half["payoff_bound_usd_upper_max"] < 0.0 + assert half["default_bound_pp_lower_min"] > 0.0 + assert half["miscoverage_bound_pp_lower_min"] > 0.0 + + three_quarters = rows[("objective_matched", 0.75)] + assert three_quarters["payoff_direction_pattern"] == "gamma_1_lower:1;crosses_zero:7" + assert three_quarters["default_direction_pattern"] == "gamma_1_higher:1;crosses_zero:7" + assert three_quarters["miscoverage_direction_pattern"] == "gamma_1_higher:8" + + normalized = [row for key, row in rows.items() if key[0] == "normalized_score"] + assert all(row["payoff_bound_usd_upper_max"] < 0.0 for row in normalized[:2]) + assert normalized[2]["payoff_direction_pattern"] == "gamma_1_lower:7;crosses_zero:1" + assert all(row["default_bound_pp_lower_min"] > 0.0 for row in normalized) + assert all(row["miscoverage_bound_pp_lower_min"] > 0.0 for row in normalized) + + repetition = challenger["objective_matched_coordinate_025_repetition"] + assert repetition["allocations_identical_across_windows_to_cents"] is True + assert repetition["changed_loan_month_positions_min"] == 44 + assert repetition["changed_loan_month_positions_max"] == 44 + assert repetition["one_way_turnover_usd_min"] == pytest.approx(155937.26968238514) + + interpretation = challenger["interpretation"] + assert interpretation["normalized_score_equalizes_opportunity_cost"] is False + assert interpretation["preferred_gamma"] is None + assert interpretation["preferred_ruler"] is None + assert interpretation["preferred_coordinate"] is None + assert interpretation["policy_winner"] is None + + +def test_endpoint_availability_sensitivity_is_complete_and_nonselective() -> None: + sensitivity = _json(EVIDENCE)["sensitivity"]["evaluation_endpoint_availability"] + + assert sensitivity["charged_off_lags_months"] == [0, 3, 6, 8, 12] + assert sensitivity["endpoint_or_result_selected"] is False + assert sensitivity["allocation_refit"] is False + assert sensitivity["six_month_endpoint_reconciles_to_active_evaluation"] is True + assert sensitivity["fit_label_lag_crossed_factorially"] is False + assert len(sensitivity["rows"]) == 5 + assert {row["charged_off_lag_months"] for row in sensitivity["rows"]} == { + 0, + 3, + 6, + 8, + 12, + } + + +def test_portfolio_structure_sensitivity_is_complete_and_nonselective() -> None: + sensitivity = _json(EVIDENCE)["sensitivity"]["portfolio_structure"] + + assert sensitivity["scenario_count"] == 36 + assert sensitivity["complete_cartesian_grid"] is True + assert sensitivity["scenario_or_result_selected"] is False + assert sensitivity["baseline_reconciles_to_active_evaluation"] is True + assert sensitivity["every_scenario_has_adverse_default_and_miscoverage_cells"] is True + assert sensitivity["minimum_adverse_default_cells_per_scenario"] == 17 + assert sensitivity["minimum_adverse_miscoverage_cells_per_scenario"] == 21 + assert sensitivity["universally_favorable_scenarios"] == 0 + assert sensitivity["universally_adverse_scenarios"] == 0 + assert sensitivity["scenarios_with_any_favorable_payoff_cell"] == 26 + assert sensitivity["scenarios_with_any_favorable_default_cell"] == 20 + assert sensitivity["scenarios_with_any_favorable_miscoverage_cell"] == 20 + assert sensitivity["portfolios_per_scenario"] == 1440 + assert sensitivity["purpose_cap_binding_share_by_cap"] == { + "0.20": 1.0, + "0.25": 1.0, + "0.30": 1.0, + "1.00": 0.0, + } + assert sensitivity["maximum_loan_weight_by_budget"] == { + "500000": 0.08, + "1000000": 0.04, + "2000000": 0.02, + } + assert len(sensitivity["rows"]) == 36 + + +def test_endpoint_reasons_missingness_and_second_origin_are_bounded() -> None: + evidence = _json(EVIDENCE) + endpoint = evidence["evaluation_endpoint"] + assert endpoint["reason_census_partitions_primary_candidates"] is True + assert endpoint["primary_oot_nonterminal_or_unresolved_status"] == 11551 + assert endpoint["primary_oot_terminal_after_cutoff"] == 47 + assert endpoint["primary_oot_terminal_availability_date_missing"] == 478 + assert endpoint["missingness_mechanism_identified"] is False + + missingness = evidence["sensitivity"]["missingness_encoding"] + assert missingness["all_three_all_eight_upper_below_nominal"] is True + assert missingness["model_or_encoding_selected"] is False + assert missingness["missingness_mechanism_identified"] is False + assert missingness["portfolio_claim_authorized"] is False + assert len(missingness["rows"]) == 3 + + rolling = evidence["sensitivity"]["rolling_origin"] + assert rolling["origin_count"] == 2 + assert rolling["window_cells"] == 16 + assert rolling["all_sixteen_upper_below_nominal"] is True + assert rolling["model_or_origin_selected"] is False + assert rolling["independent_replication_claim_authorized"] is False + + +def test_fit_label_completion_and_allocation_granularity_are_bounded() -> None: + evidence = _json(EVIDENCE) + fit = evidence["sensitivity"]["fit_label_completion"] + assert fit["unavailable_fit_labels_total"] == 215 + assert fit["unavailable_fit_labels_by_split"] == { + "pd_development": 41, + "probability_calibration": 24, + "conformal_fit": 150, + } + assert fit["all_scenarios_all_windows_upper_below_nominal"] is True + assert fit["w7_w8_crossing_scenarios"] == 3 + assert fit["w7_w8_crossing_in_all_scenarios"] is False + assert fit["scenarios_are_sharp_bounds_over_all_label_assignments"] is False + assert fit["scenario_or_result_selected"] is False + assert len(fit["rows"]) == 4 + + granularity = evidence["sensitivity"]["allocation_granularity"] + assert granularity["portfolios"] == 1440 + assert granularity["tracks"] == 96 + assert granularity["source_rows"] == 143175 + assert granularity["changed_rows"] == 2985 + assert granularity["cash_share_max"] < 3.4e-5 + assert granularity["default_rate_perturbation_abs_max"] < 1.3e-5 + assert granularity["integer_policy_or_reoptimization_claim_authorized"] is False + + +def test_evidence_manifest_hashes_every_active_output() -> None: + evidence = _json(EVIDENCE) + + assert { + "active_source_registry", + "evidence_builder", + "source_registry_loader", + "claim_ledger_contract", + "claim_ledger_loader", + "endpoint_availability_sensitivity/summary", + "endpoint_availability_sensitivity/loader", + "portfolio_structure_sensitivity/summary", + "portfolio_structure_sensitivity/loader", + "robustness_sensitivities/loader", + "artifact_descriptor_helper", + "outcome_free/source_protocol_freeze", + "two_ruler/outcome_free/freeze", + "credit_controls/freeze", + }.issubset(evidence["source_artifacts"]) + assert evidence["paper_artifacts"] + for descriptor in ( + *evidence["source_artifacts"].values(), + *evidence["paper_artifacts"].values(), + ): + path = REPO / descriptor["path"] + assert path.is_file(), path + assert path.stat().st_size == descriptor["bytes"] + assert _sha256(path) == descriptor["sha256"] + + +def test_manuscript_surfaces_share_v4_claims_and_retire_old_headlines() -> None: + active = ( + "376,890", + "364,814", + "12,076", + "307,842", + "56,972", + "478", + "6,240", + "9,134.34", + "5,603.66", + "14,738", + "155,937.27", + "44 loan-month positions", + "0.1017", + "0.0971", + "0.8884", + "0.1118", + "215", + "0.884669", + "2,985", + "0.001284", + "3,067", + "216", + "72", + "0.884332", + "0.874768", + "status-indexed", + "selected-set", + ) + retired = ( + "0.879647", + "0.845072", + "0.870973", + "7 of 9", + "5 of 9", + "selected guardrail", + "all nine policies are co-primary", + "$179,327.59", + "active v3", + "endpoint-recovery direction reconciliation", ) for surface in SURFACES: - text = _surface_text(surface) - missing = [anchor for anchor in anchors if anchor not in text] - assert not missing, f"{surface.name} missing active anchors: {missing}" + text = _normalize(surface.read_text(encoding="utf-8")) + assert not [token for token in active if _normalize(token) not in text], surface + assert not [token for token in retired if _normalize(token) in text], surface -def test_retired_headline_numbers_do_not_appear_in_active_surfaces() -> None: - retired = ("0.345084", "50,010", "27,508", "capped_blended_uncertainty") - for surface in SURFACES: - text = _surface_text(surface) - present = [token for token in retired if token in text] - assert not present, f"{surface.name} retains retired claims: {present}" +def test_official_tex_is_deterministically_generated_from_qmd() -> None: + assert render_submission_tex(check=True) diff --git a/tests/test_ijds_anonymity.py b/tests/test_ijds_anonymity.py new file mode 100644 index 0000000..3b56ad8 --- /dev/null +++ b/tests/test_ijds_anonymity.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from scripts.check_publication_integrity import ( + REVIEWER_FORBIDDEN_LITERALS, + REVIEWER_FORBIDDEN_PATTERNS, + REVIEWER_SURFACES, +) + + +def test_reviewer_surfaces_exclude_identity_and_searchable_fingerprints() -> None: + for path in REVIEWER_SURFACES: + text = path.read_text(encoding="utf-8") + normalized = text.lower() + + for token in REVIEWER_FORBIDDEN_LITERALS: + assert token not in normalized, f"{path.name} exposes {token!r}" + for label, pattern in REVIEWER_FORBIDDEN_PATTERNS: + assert pattern.search(text) is None, f"{path.name} exposes {label}" diff --git a/tests/test_ijds_audit/test_allocation_granularity.py b/tests/test_ijds_audit/test_allocation_granularity.py new file mode 100644 index 0000000..60547d8 --- /dev/null +++ b/tests/test_ijds_audit/test_allocation_granularity.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.ijds_audit.allocation_granularity import ( + floor_allocations_to_lot, + granularity_contrast_bounds, +) + + +def _allocations() -> pd.DataFrame: + rows: list[dict[str, object]] = [] + for period, first in (("2016-04", 62.0), ("2016-05", 57.0)): + for identifier, exposure, amount, probability in ( + (f"{period}-a", first, 100.0, 0.10), + (f"{period}-b", 100.0 - first, 100.0, 0.20), + ): + rows.append( + { + "id": identifier, + "window_id": "w01", + "role": "primary_oot", + "period": period, + "policy_label": "candidate", + "candidate_id": "candidate", + "comparator_rule": "objective_matched", + "frontier_ruler": "objective_matched", + "frontier_coordinate": 0.25, + "frontier_cap": 0.20, + "gamma": 1.0, + "loan_amnt": amount, + "exposure": exposure, + "expected_payoff_rate": 0.05, + "expected_payoff_contribution": exposure * 0.05, + "pd_point": probability, + "pd_effective": probability, + "conformal_lower": 0.0, + "conformal_upper": 0.8, + "contractual_rate": 0.12, + "purpose": "debt_consolidation", + "scenario_purpose_cap": 1.0, + } + ) + return pd.DataFrame(rows) + + +def test_floor_allocations_retains_cash_and_preserves_constraints() -> None: + rounded, audit = floor_allocations_to_lot( + _allocations(), + lot_size=25.0, + committed_budget=100.0, + ) + + assert set(rounded["exposure"]) == {25.0, 50.0} + assert audit["cash_residual"].tolist() == [25.0, 25.0] + assert audit["changed_positions"].tolist() == [2, 2] + assert rounded.groupby("period", observed=True)["weight"].sum().tolist() == [0.75, 0.75] + + +def test_granularity_bounds_use_committed_capital_including_cash() -> None: + source = _allocations() + rounded, _ = floor_allocations_to_lot( + source, + lot_size=25.0, + committed_budget=100.0, + ) + outcomes = pd.DataFrame( + { + "id": source["id"].astype("string"), + "snapshot_default": pd.Series([0, 1, pd.NA, 0], dtype="Int8"), + } + ) + + bounds = granularity_contrast_bounds( + source, + rounded, + outcomes, + committed_budget=100.0, + periods=("2016-04", "2016-05"), + lgd=0.45, + ).iloc[0] + + assert bounds["policy_a_capital"] == pytest.approx(150.0) + assert bounds["policy_a_normalization_capital"] == 200.0 + assert bounds["cash_residual_total"] == pytest.approx(50.0) + assert ( + bounds["weighted_default_difference_lower"] <= (bounds["weighted_default_difference_upper"]) + ) + + +def test_floor_allocations_rejects_nonpositive_lot() -> None: + with pytest.raises(ValueError, match="Lot size"): + floor_allocations_to_lot( + _allocations(), + lot_size=0.0, + committed_budget=100.0, + ) diff --git a/tests/test_ijds_audit/test_claim_ledger.py b/tests/test_ijds_audit/test_claim_ledger.py new file mode 100644 index 0000000..aac6b19 --- /dev/null +++ b/tests/test_ijds_audit/test_claim_ledger.py @@ -0,0 +1,120 @@ +"""Tests for the nonnumeric IJDS claim contract.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from src.ijds_audit.claim_ledger import ( + load_claim_ledger, + materialize_claim_ledger, + resolve_json_pointer, +) + + +def _write_ledger(tmp_path: Path, *, expected: str = "ready") -> Path: + (tmp_path / "body.md").write_text("\n", encoding="utf-8") + (tmp_path / "registry.md").write_text( + "\n\n", + encoding="utf-8", + ) + ledger = tmp_path / "claims.yaml" + ledger.write_text( + f"""schema_version: test.1 +status: active_ijds_claim_contract +surfaces: + body: body.md + registry: registry.md +claims: + - id: test.empirical + status: active + kind: empirical + rule: equals + result_pointer: /result/status + expected: {expected} + lineages: [test.evaluation] + scope: unit_test + forbidden_inference: [causal] + surfaces: + required: [body, registry] + allowed: [body, registry] + - id: test.theorem + status: active + kind: theorem + rule: documented + lineages: [] + scope: algebra + forbidden_inference: [empirical_universality] + surfaces: + required: [registry] + allowed: [registry] +""", + encoding="utf-8", + ) + return ledger + + +def test_claim_ledger_materializes_without_numeric_duplication(tmp_path: Path) -> None: + ledger = _write_ledger(tmp_path) + + result = materialize_claim_ledger( + ledger, + evidence={"result": {"status": "ready"}}, + repo_root=tmp_path, + ) + + assert result["numeric_authority"] == "parent_evidence_manifest_only" + assert [claim["id"] for claim in result["claims"]] == ["test.empirical", "test.theorem"] + assert result["claims"][0]["result"] == "ready" + assert result["claims"][1]["result"] is None + + +def test_claim_ledger_rejects_failed_rule_or_missing_marker(tmp_path: Path) -> None: + ledger = _write_ledger(tmp_path) + with pytest.raises(RuntimeError, match="failed"): + materialize_claim_ledger( + ledger, + evidence={"result": {"status": "not-ready"}}, + repo_root=tmp_path, + ) + + (tmp_path / "body.md").write_text("no marker\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="missing markers"): + materialize_claim_ledger( + ledger, + evidence={"result": {"status": "ready"}}, + repo_root=tmp_path, + ) + + +def test_claim_ledger_rejects_numeric_expected_result(tmp_path: Path) -> None: + ledger = _write_ledger(tmp_path, expected="1.5") + text = ledger.read_text(encoding="utf-8").replace("expected: 1.5", "expected: 1.5") + ledger.write_text(text, encoding="utf-8") + + with pytest.raises(ValueError, match="duplicates a numeric result"): + load_claim_ledger(ledger) + + +def test_json_pointer_handles_escaped_object_keys() -> None: + document = {"a/b": {"tilde~key": False}} + assert resolve_json_pointer(document, "/a~1b/tilde~0key") is False + + +def test_active_claim_ledger_matches_generated_evidence() -> None: + root = Path(__file__).resolve().parents[2] + evidence = json.loads( + (root / "reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json").read_text( + encoding="utf-8" + ) + ) + + expected = materialize_claim_ledger( + root / "configs/ijds_claim_ledger.yaml", + evidence=evidence, + repo_root=root, + ) + + assert evidence["claim_ledger"] == expected diff --git a/tests/test_ijds_audit/test_credit_controls.py b/tests/test_ijds_audit/test_credit_controls.py new file mode 100644 index 0000000..bac3f34 --- /dev/null +++ b/tests/test_ijds_audit/test_credit_controls.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import pickle +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from optbinning import BinningProcess +from sklearn.linear_model import LogisticRegression + +from src.ijds_audit.config import load_credit_control_config +from src.ijds_audit.credit_controls import ( + WOELogisticModel, + credit_prediction_metrics, + scorecard_binning_table, +) + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "configs/experiments/ijds_credit_risk_controls_2026-07-13_v1.yaml" +V2_CONFIG = ROOT / "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2.yaml" +V2B_CONFIG = ROOT / "configs/experiments/ijds_credit_risk_controls_2026-07-13_v2b.yaml" + + +def test_credit_control_config_is_closed_and_uses_all_rows() -> None: + config = load_credit_control_config(CONFIG) + controls = config["credit_risk_controls"] + + assert controls["selection_from_oot"] is False + assert controls["portfolio_optimization"] is False + assert controls["sampling"] == "none_all_eligible_rows" + assert len(controls["co_primary_models"]) == 5 + assert not set(controls["platform_signal_features"]).intersection( + controls["scorecards"]["borrower"]["features"] + ) + + +def test_credit_control_v2_imports_the_exact_v1b_freeze() -> None: + config = load_credit_control_config(V2_CONFIG) + resume = config["resume_credit_control_freeze"] + + assert resume["source_run_tag"].endswith("v1b") + assert resume["source_protocol_commit"] == "1776cbf8b201ae5b92756e5ea397a403d6cc7c9f" + assert resume["source_freeze_sha256"] == ( + "da4805e644bcf5decfbb0a67c0c81a5b9dd61f3ab2e17d3dc5264100e7eb4d35" + ) + + +def test_credit_control_v2b_requires_exact_v2_coverage() -> None: + config = load_credit_control_config(V2B_CONFIG) + recovery = config["evaluation_recovery"] + + assert recovery["calibration_solver"] == "sklearn_unpenalized_lbfgs" + assert recovery["require_exact_coverage_equivalence"] is True + assert recovery["coverage_reference"]["sha256"] == ( + "956bdc9880c80cebc1f48fc2cdf57688dfff7dddd3939b4fd972413d83039767" + ) + + +def test_credit_control_config_rejects_platform_signal_in_borrower_scorecard( + tmp_path: Path, +) -> None: + payload = CONFIG.read_text(encoding="utf-8").replace( + 'extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml"', + f'extends: "{(CONFIG.parent / "ijds_binary_geometry_frontier_v4_2026-07-12.yaml").as_posix()}"', + ) + payload = payload.replace( + ' borrower:\n name: "woe_scorecard_borrower_platt"\n features:', + ' borrower:\n name: "woe_scorecard_borrower_platt"\n features:\n - "int_rate"', + ) + path = tmp_path / "invalid.yaml" + path.write_text(payload, encoding="utf-8") + + with pytest.raises(ValueError, match="borrower scorecard"): + load_credit_control_config(path) + + +def test_woe_model_handles_unknown_missing_and_parquet_serialization(tmp_path: Path) -> None: + generator = np.random.default_rng(7) + features = pd.DataFrame( + { + "numeric": generator.normal(size=400), + "category": generator.choice(["a", "b", "c"], size=400), + } + ) + labels = ( + features["numeric"] + + features["category"].eq("c").astype(float) * 0.8 + + generator.normal(size=400) + > 0 + ).astype(int) + process = BinningProcess( + ["numeric", "category"], + categorical_variables=["category"], + min_n_bins=2, + max_n_bins=5, + min_bin_size=0.05, + n_jobs=1, + ).fit(features, labels) + transformed = process.transform( + features, + metric="woe", + metric_missing=0, + metric_special=0, + ) + estimator = LogisticRegression(max_iter=1000).fit(transformed, labels) + model = WOELogisticModel( + name="synthetic", + features=("numeric", "category"), + categorical_features=("category",), + binning_process=process, + logistic_regression=estimator, + ) + restored = pickle.loads(pickle.dumps(model)) + probe = pd.DataFrame( + { + "numeric": [0.0, np.nan], + "category": ["unseen", None], + } + ) + + probabilities = restored.predict_proba(probe) + assert probabilities.shape == (2, 2) + assert np.isfinite(probabilities).all() + assert np.allclose(probabilities.sum(axis=1), 1.0) + binning_table = scorecard_binning_table(restored) + binning_table.to_parquet(tmp_path / "woe_binning_table.parquet", index=False) + assert pd.api.types.is_numeric_dtype(binning_table["WoE"]) + + +def test_credit_prediction_metrics_report_discrimination_and_calibration() -> None: + generator = np.random.default_rng(19) + probability = np.linspace(0.01, 0.60, 5000) + labels = generator.binomial(1, probability) + + metrics = credit_prediction_metrics(labels, probability) + + assert metrics["gini"] == pytest.approx(2.0 * metrics["roc_auc"] - 1.0) + assert 0.0 <= metrics["ks"] <= 1.0 + assert 0.0 <= metrics["average_precision"] <= 1.0 + assert np.isfinite(metrics["calibration_intercept"]) + assert np.isfinite(metrics["calibration_slope"]) + assert metrics["calibration_optimizer_success"] is True diff --git a/tests/test_ijds_audit/test_endpoint_recovery.py b/tests/test_ijds_audit/test_endpoint_recovery.py new file mode 100644 index 0000000..afdb519 --- /dev/null +++ b/tests/test_ijds_audit/test_endpoint_recovery.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd +import pytest + +from src.ijds_audit.endpoint_recovery import reconcile_from_json_reference +from src.utils.isolated_experiment import relative_artifact_descriptor + + +def _reference(tmp_path: Path) -> tuple[dict[str, object], pd.DataFrame]: + reference = pd.DataFrame({"id": ["a", "b"], "metric": [1.0, 2.0]}) + parquet = tmp_path / "reference.parquet" + reference.to_parquet(parquet, index=False) + manifest = tmp_path / "manifest.json" + manifest.write_text( + json.dumps( + {"artifacts": {"result": relative_artifact_descriptor(parquet, repo_root=tmp_path)}} + ), + encoding="utf-8", + ) + return relative_artifact_descriptor(manifest, repo_root=tmp_path), reference + + +def test_endpoint_recovery_allows_appended_diagnostics(tmp_path: Path) -> None: + manifest, reference = _reference(tmp_path) + current = reference.assign(identification_width=[0.1, 0.2]) + + audit = reconcile_from_json_reference( + {"result": current}, + reference_json=manifest, + artifact_section="artifacts", + repo_root=tmp_path, + ) + + assert audit["status"] == "exact_reference_column_equivalence_verified" + assert audit["frames"]["result"]["appended_columns"] == ["identification_width"] + + +def test_endpoint_recovery_rejects_scientific_metric_drift(tmp_path: Path) -> None: + manifest, reference = _reference(tmp_path) + current = reference.copy() + current.loc[0, "metric"] += 1.0e-12 + + with pytest.raises(AssertionError): + reconcile_from_json_reference( + {"result": current}, + reference_json=manifest, + artifact_section="artifacts", + repo_root=tmp_path, + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + + +def test_endpoint_recovery_accepts_and_records_machine_float_drift(tmp_path: Path) -> None: + manifest, reference = _reference(tmp_path) + current = reference.copy() + current.loc[0, "metric"] = float(current.loc[0, "metric"]) + 2.0e-15 + + audit = reconcile_from_json_reference( + {"result": current}, + reference_json=manifest, + artifact_section="artifacts", + repo_root=tmp_path, + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + + assert audit["status"] == "reference_column_equivalence_verified_with_float_tolerance" + assert audit["equivalence"] == { + "non_float_columns_exact": True, + "float_atol": 5.0e-14, + "float_rtol": 5.0e-14, + } + assert audit["frames"]["result"]["float_drift"]["metric"]["maximum_absolute"] > 0.0 + + +def test_endpoint_recovery_keeps_nonfloat_columns_exact_under_float_tolerance( + tmp_path: Path, +) -> None: + manifest, reference = _reference(tmp_path) + current = reference.copy() + current.loc[0, "id"] = "changed" + + with pytest.raises(AssertionError): + reconcile_from_json_reference( + {"result": current}, + reference_json=manifest, + artifact_section="artifacts", + repo_root=tmp_path, + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) diff --git a/tests/test_ijds_audit/test_endpoint_sensitivity.py b/tests/test_ijds_audit/test_endpoint_sensitivity.py new file mode 100644 index 0000000..fafa623 --- /dev/null +++ b/tests/test_ijds_audit/test_endpoint_sensitivity.py @@ -0,0 +1,55 @@ +"""Tests for endpoint-availability reconstruction and summaries.""" + +from __future__ import annotations + +import pandas as pd + +from src.ijds_audit.endpoint_sensitivity import ( + endpoint_census, + rebuild_archive_outcomes, + summarize_coverage_sensitivity, +) + + +def test_rebuild_archive_outcomes_changes_only_lagged_charge_off_resolution() -> None: + universe = pd.DataFrame( + { + "id": pd.Series(["good", "bad"], dtype="string"), + "loan_status": ["Fully Paid", "Charged Off"], + "last_pymnt_d": ["Jan-2020", "Jan-2020"], + "design_split": ["primary_oot", "primary_oot"], + "issue_d": pd.to_datetime(["2016-04-01", "2016-04-01"]), + } + ) + lag0 = rebuild_archive_outcomes( + universe, + evaluation_cutoff="2020-06-30", + charged_off_lag_months=0, + ) + lag12 = rebuild_archive_outcomes( + universe, + evaluation_cutoff="2020-06-30", + charged_off_lag_months=12, + ) + assert lag0["snapshot_default"].tolist() == [0, 1] + assert lag12["snapshot_default"].iloc[0] == 0 + assert pd.isna(lag12["snapshot_default"].iloc[1]) + census = endpoint_census(lag12, lag_months=12) + assert census.loc[0, "resolved_rows"] == 1 + assert census.loc[0, "unresolved_rows"] == 1 + + +def test_coverage_summary_retains_each_lag_learner_and_role() -> None: + coverage = pd.DataFrame( + { + "charged_off_lag_months": [0, 0, 12, 12], + "learner": ["m", "m", "m", "m"], + "role": ["primary_oot"] * 4, + "window_id": ["w1", "w2", "w1", "w2"], + "coverage_lower": [0.8, 0.82, 0.79, 0.80], + "coverage_upper": [0.88, 0.89, 0.90, 0.91], + } + ) + summary = summarize_coverage_sensitivity(coverage) + assert summary["windows"].tolist() == [2, 2] + assert summary["coverage_upper_max"].tolist() == [0.89, 0.91] diff --git a/tests/test_ijds_audit/test_evaluation_outcome_contracts.py b/tests/test_ijds_audit/test_evaluation_outcome_contracts.py new file mode 100644 index 0000000..0f34ef2 --- /dev/null +++ b/tests/test_ijds_audit/test_evaluation_outcome_contracts.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pandas as pd +import pytest + +from src.ijds_audit.config import load_credit_control_config, load_v4_config +from src.ijds_audit.evaluation import ( + RESOLUTION_CHARGED_OFF_BY_CUTOFF, + RESOLUTION_FULLY_PAID_BY_CUTOFF, + RESOLUTION_NONTERMINAL, + RESOLUTION_TERMINAL_AFTER_CUTOFF, + RESOLUTION_TERMINAL_DATE_MISSING, + build_archive_outcomes, + comparator_envelopes, + endpoint_resolution_audit, + evaluate_frozen_portfolios, + temporal_coverage_audit, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def _portfolio_inputs() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict[str, Any]]: + records = pd.DataFrame( + { + "window_id": ["w01"], + "role": ["primary_oot"], + "period": ["2016-04"], + "policy_label": ["policy"], + "comparator_rule": ["guardrail"], + "n_candidates": [2], + "robust_guardrail": [False], + "total_allocated": [100.0], + } + ) + allocations = pd.DataFrame( + { + "id": pd.Series(["funded-null"], dtype="string"), + "window_id": ["w01"], + "role": ["primary_oot"], + "period": ["2016-04"], + "policy_label": ["policy"], + "comparator_rule": ["guardrail"], + "contractual_rate": [0.10], + "conformal_lower": [0.0], + "conformal_upper": [0.8], + "exposure": [100.0], + "weight": [1.0], + } + ) + outcomes = pd.DataFrame( + { + "id": pd.Series(["funded-null", "unfunded"], dtype="string"), + "snapshot_default": pd.Series([pd.NA, 0], dtype="Int8"), + "snapshot_resolution": pd.Series(["right_censored", "fully_paid"], dtype="string"), + "role": pd.Series(["primary_oot", "primary_oot"], dtype="string"), + "period": pd.Series(["2016-04", "2016-04"], dtype="string"), + } + ) + config: dict[str, Any] = { + "payoff": {"lgd": 0.45}, + "policy": {"budget": 100.0}, + } + return records, allocations, outcomes, config + + +def _coverage_inputs() -> tuple[pd.DataFrame, pd.DataFrame]: + scores = pd.DataFrame( + { + "id": pd.Series(["a", "b"], dtype="string"), + "issue_d": pd.to_datetime(["2016-04-01", "2016-04-15"]), + "design_split": pd.Series(["primary_oot", "primary_oot"], dtype="string"), + } + ) + outcomes = pd.DataFrame( + { + "id": pd.Series(["a", "b"], dtype="string"), + "snapshot_default": pd.Series([0, pd.NA], dtype="Int8"), + "role": pd.Series(["primary_oot", "primary_oot"], dtype="string"), + "period": pd.Series(["2016-04", "2016-04"], dtype="string"), + } + ) + return scores, outcomes + + +def test_archive_endpoint_reasons_separate_missing_and_post_cutoff_dates() -> None: + universe = pd.DataFrame( + { + "id": pd.Series(["paid", "charged", "missing", "late", "current"], dtype="string"), + "terminal_default": pd.Series([0, 1, 1, 0, pd.NA], dtype="Int8"), + "label_available_at": pd.to_datetime( + ["2020-01-31", "2020-02-29", None, "2020-10-31", None] + ), + "design_split": pd.Series(["primary_oot"] * 5, dtype="string"), + "issue_d": pd.to_datetime(["2016-04-01"] * 5), + } + ) + + outcomes = build_archive_outcomes(universe, evaluation_cutoff="2020-09-30") + + assert outcomes["snapshot_default"].tolist()[:2] == [0, 1] + assert outcomes["snapshot_default"].isna().tolist()[2:] == [True, True, True] + assert outcomes["snapshot_resolution"].tolist() == [ + RESOLUTION_FULLY_PAID_BY_CUTOFF, + RESOLUTION_CHARGED_OFF_BY_CUTOFF, + RESOLUTION_TERMINAL_DATE_MISSING, + RESOLUTION_TERMINAL_AFTER_CUTOFF, + RESOLUTION_NONTERMINAL, + ] + audit = endpoint_resolution_audit(outcomes, roles=("primary_oot",)) + assert audit["candidate_rows"].sum() == 5 + assert audit["resolved_rows"].sum() == 2 + assert audit["unresolved_rows"].sum() == 3 + + +def test_v5_endpoint_recovery_configs_lock_machine_tolerance_reconciliation() -> None: + binary = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-15_v5.yaml" + ) + credit = load_credit_control_config( + ROOT / "configs/experiments/ijds_credit_risk_controls_2026-07-15_v5.yaml" + ) + + assert binary["run_tag"].endswith("2026-07-15-v5") + assert credit["run_tag"].endswith("2026-07-15-v5") + recovery = binary["endpoint_reason_recovery"] + assert recovery["require_exact_non_float_reference_equivalence"] is True + assert recovery["float_atol"] == 5.0e-14 + assert recovery["float_rtol"] == 5.0e-14 + assert credit["endpoint_reason_recovery"]["artifact_section"] == "evaluation_artifacts" + + +def test_evaluate_frozen_portfolios_preserves_nullable_outcome_fact() -> None: + records, allocations, outcomes, config = _portfolio_inputs() + + evaluated, joined = evaluate_frozen_portfolios( + records, + allocations, + outcomes, + config=config, + ) + + assert pd.isna(joined.loc[0, "snapshot_default"]) + assert evaluated.loc[0, "n_unresolved_candidates"] == 1 + assert evaluated.loc[0, "n_unresolved_positive_exposure"] == 1 + + +def test_evaluate_frozen_portfolios_rejects_duplicate_solve_keys() -> None: + records, allocations, outcomes, config = _portfolio_inputs() + records = pd.concat([records, records], ignore_index=True) + + with pytest.raises(ValueError, match="duplicate evaluation keys"): + evaluate_frozen_portfolios(records, allocations, outcomes, config=config) + + +def test_comparator_envelopes_rejects_duplicate_support_keys() -> None: + support = pd.DataFrame( + { + "window_id": ["w01", "w01"], + "paired_policy_id": ["policy", "policy"], + "support_lower": [0.05, 0.05], + "support_upper": [0.12, 0.12], + } + ) + + with pytest.raises(ValueError, match="duplicate policy-window keys"): + comparator_envelopes( + pd.DataFrame(), + support, + broad_lower=0.05, + broad_upper=0.12, + ) + + +@pytest.mark.parametrize("invalid_count", [1.5, float("inf"), -1]) +def test_evaluate_frozen_portfolios_rejects_invalid_declared_candidate_count( + invalid_count: float, +) -> None: + records, allocations, outcomes, config = _portfolio_inputs() + records["n_candidates"] = records["n_candidates"].astype(float) + records.loc[0, "n_candidates"] = invalid_count + + with pytest.raises(RuntimeError, match="invalid candidate census"): + evaluate_frozen_portfolios(records, allocations, outcomes, config=config) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("missing", "missing=1, extra=0"), + ("extra", "missing=0, extra=1"), + ("duplicate", "duplicate IDs"), + ], +) +def test_evaluate_frozen_portfolios_rejects_outcome_census_mutations( + mutation: str, + message: str, +) -> None: + records, allocations, outcomes, config = _portfolio_inputs() + if mutation == "missing": + outcomes = outcomes.loc[~outcomes["id"].eq("unfunded")].copy() + elif mutation == "extra": + extra = outcomes.iloc[[1]].assign(id="extra") + outcomes = pd.concat([outcomes, extra], ignore_index=True) + else: + outcomes = pd.concat([outcomes, outcomes.iloc[[1]]], ignore_index=True) + + with pytest.raises(RuntimeError, match=message): + evaluate_frozen_portfolios(records, allocations, outcomes, config=config) + + +@pytest.mark.parametrize( + ("column", "value"), + [("role", "policy_development"), ("period", "2016-05")], +) +def test_evaluate_frozen_portfolios_rejects_outcome_metadata_mismatch( + column: str, + value: str, +) -> None: + records, allocations, outcomes, config = _portfolio_inputs() + outcomes.loc[outcomes["id"].eq("funded-null"), column] = value + + with pytest.raises(RuntimeError, match=f"Outcome {column} disagrees"): + evaluate_frozen_portfolios(records, allocations, outcomes, config=config) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("missing", "missing=1, extra=0"), + ("extra", "missing=0, extra=1"), + ("duplicate", "duplicate IDs"), + ], +) +def test_temporal_coverage_rejects_outcome_census_mutations( + mutation: str, + message: str, +) -> None: + scores, outcomes = _coverage_inputs() + if mutation == "missing": + outcomes = outcomes.loc[~outcomes["id"].eq("b")].copy() + elif mutation == "extra": + extra = outcomes.iloc[[0]].assign(id="extra") + outcomes = pd.concat([outcomes, extra], ignore_index=True) + else: + outcomes = pd.concat([outcomes, outcomes.iloc[[0]]], ignore_index=True) + + with pytest.raises(RuntimeError, match=message): + temporal_coverage_audit( + scores, + outcomes, + {}, + pd.DataFrame(), + roles=("primary_oot",), + ) + + +@pytest.mark.parametrize( + ("column", "value"), + [("role", "policy_development"), ("period", "2016-05")], +) +def test_temporal_coverage_rejects_outcome_metadata_mismatch( + column: str, + value: str, +) -> None: + scores, outcomes = _coverage_inputs() + outcomes.loc[outcomes["id"].eq("a"), column] = value + + with pytest.raises(RuntimeError, match=f"Outcome {column} disagrees"): + temporal_coverage_audit( + scores, + outcomes, + {}, + pd.DataFrame(), + roles=("primary_oot",), + ) diff --git a/tests/test_ijds_audit/test_fit_label_sensitivity.py b/tests/test_ijds_audit/test_fit_label_sensitivity.py new file mode 100644 index 0000000..7e7cc3b --- /dev/null +++ b/tests/test_ijds_audit/test_fit_label_sensitivity.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.ijds_audit.fit_label_sensitivity import ( + FIT_LABEL_SCENARIOS, + apply_fit_label_scenario, + summarize_fit_label_coverage, +) + + +def _universe() -> pd.DataFrame: + return pd.DataFrame( + { + "design_split": [ + "pd_development", + "pd_development", + "probability_calibration", + "conformal_fit", + "primary_oot", + ], + "terminal_default": pd.Series([0, 1, 1, 0, pd.NA], dtype="Int8"), + "label_available": [True, False, False, False, False], + } + ) + + +@pytest.mark.parametrize( + ("scenario", "expected"), + [ + ("observed_only", [0, 1, 1, 0]), + ("all_unavailable_nondefault", [0, 0, 0, 0]), + ("all_unavailable_default", [0, 1, 1, 1]), + ("hindsight_terminal", [0, 1, 1, 0]), + ], +) +def test_fit_label_scenario_changes_only_unavailable_fitting_rows( + scenario: str, + expected: list[int], +) -> None: + completed, audit = apply_fit_label_scenario(_universe(), scenario=scenario) + + assert completed.loc[:3, "terminal_default"].astype(int).tolist() == expected + assert pd.isna(completed.loc[4, "terminal_default"]) + assert bool(completed.loc[4, "label_available"]) is False + expected_completed = 0 if scenario == "observed_only" else 3 + assert int(audit["completed_rows"].sum()) == expected_completed + + +def test_fit_label_scenario_rejects_unknown_name() -> None: + with pytest.raises(ValueError, match="Unknown"): + apply_fit_label_scenario(_universe(), scenario="selected_after_outcomes") + + +def test_fit_label_coverage_summary_requires_complete_grid() -> None: + rows = [] + for scenario in FIT_LABEL_SCENARIOS: + for window in ("w01", "w02"): + rows.append( + { + "fit_label_scenario": scenario, + "window_id": window, + "conformal_group": -1, + "coverage_lower": 0.84, + "coverage_upper": 0.88, + "mean_width": 0.4, + } + ) + coverage = pd.DataFrame(rows) + + summary = summarize_fit_label_coverage( + coverage, + window_ids=("w01", "w02"), + nominal_coverage=0.90, + ) + + assert len(summary) == len(FIT_LABEL_SCENARIOS) + assert bool(summary["all_windows_upper_below_nominal"].all()) + + with pytest.raises(RuntimeError, match="incomplete"): + summarize_fit_label_coverage( + coverage.iloc[:-1], + window_ids=("w01", "w02"), + nominal_coverage=0.90, + ) diff --git a/tests/test_ijds_audit/test_grid_contracts.py b/tests/test_ijds_audit/test_grid_contracts.py new file mode 100644 index 0000000..4825796 --- /dev/null +++ b/tests/test_ijds_audit/test_grid_contracts.py @@ -0,0 +1,141 @@ +"""Mutation tests for paper-facing Cartesian evidence contracts.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.ijds_audit.grid_contracts import ( + require_exact_frame, + require_exact_grid, + require_finite, + require_unique_row, + require_unique_value, +) + + +def _grid() -> pd.DataFrame: + return pd.DataFrame( + { + "learner": ["a", "a", "b", "b"], + "window": [1, 2, 1, 2], + "value": [0.1, 0.2, 0.3, 0.4], + "constant": [7, 7, 7, 7], + } + ) + + +def test_exact_grid_is_order_invariant_and_keyed() -> None: + frame = _grid().sample(frac=1.0, random_state=7) + + require_exact_grid( + frame, + domains={"learner": ("a", "b"), "window": (1, 2)}, + label="test grid", + ) + row = require_unique_row(frame, key={"learner": "b", "window": 2}, label="test row") + assert row["value"] == pytest.approx(0.4) + assert require_unique_value(frame, "constant", label="test constant") == 7 + + +def test_exact_grid_rejects_duplicate_replacing_missing_key() -> None: + mutated = pd.concat([_grid().iloc[:-1], _grid().iloc[[0]]], ignore_index=True) + + with pytest.raises(RuntimeError, match="duplicate grid keys"): + require_exact_grid( + mutated, + domains={"learner": ("a", "b"), "window": (1, 2)}, + label="test grid", + ) + + +def test_exact_grid_rejects_missing_or_extra_key() -> None: + with pytest.raises(RuntimeError, match="grid changed"): + require_exact_grid( + _grid().iloc[:-1], + domains={"learner": ("a", "b"), "window": (1, 2)}, + label="test grid", + ) + + extra = pd.concat( + [_grid(), pd.DataFrame({"learner": ["c"], "window": [1], "value": [0.5]})], + ignore_index=True, + ) + with pytest.raises(RuntimeError, match="grid changed"): + require_exact_grid( + extra, + domains={"learner": ("a", "b"), "window": (1, 2)}, + label="test grid", + ) + + +def test_finite_and_unique_value_contracts_reject_silent_corruption() -> None: + nonfinite = _grid() + nonfinite.loc[1, "value"] = np.inf + with pytest.raises(RuntimeError, match="nonfinite"): + require_finite(nonfinite, ("value",), label="test values") + + nonconstant = _grid() + nonconstant.loc[1, "constant"] = 8 + with pytest.raises(RuntimeError, match="no unique"): + require_unique_value(nonconstant, "constant", label="test constant") + + +def test_exact_frame_tolerance_applies_only_to_nonkey_floats() -> None: + expected = pd.DataFrame({"key": [1.0], "value": [0.3], "label": ["a"]}) + rounded = expected.copy() + rounded.loc[0, "value"] += 2.0e-16 + + require_exact_frame( + rounded, + expected, + keys=("key",), + label="roundoff", + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + + changed_key = rounded.copy() + changed_key.loc[0, "key"] += 2.0e-16 + with pytest.raises(RuntimeError, match="does not reconcile exactly"): + require_exact_frame( + changed_key, + expected, + keys=("key",), + label="changed key", + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + + changed_label = rounded.copy() + changed_label.loc[0, "label"] = "b" + with pytest.raises(RuntimeError, match="does not reconcile exactly"): + require_exact_frame( + changed_label, + expected, + keys=("key",), + label="changed label", + float_atol=5.0e-14, + float_rtol=5.0e-14, + ) + + +def test_exact_frame_allows_only_named_reference_columns() -> None: + actual = pd.DataFrame({"key": [1], "value": [0.3]}) + expected = actual.assign(width=[0.0]) + + require_exact_frame( + actual, + expected, + keys=("key",), + label="declared extension", + allowed_expected_extra_columns=("width",), + ) + with pytest.raises(RuntimeError, match="columns differ"): + require_exact_frame( + actual, + expected, + keys=("key",), + label="undeclared extension", + ) diff --git a/tests/test_ijds_audit/test_lag_sensitivity.py b/tests/test_ijds_audit/test_lag_sensitivity.py new file mode 100644 index 0000000..13ceef4 --- /dev/null +++ b/tests/test_ijds_audit/test_lag_sensitivity.py @@ -0,0 +1,52 @@ +"""Tests for the frozen-score label-lag sensitivity.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from src.ijds_audit.lag_sensitivity import build_label_lag_phase_sensitivity +from src.models.binary_conformal_guardrail import fit_binary_outcome_recipe + + +def test_label_lag_sensitivity_preserves_frozen_taxonomy() -> None: + score = np.linspace(0.02, 0.40, 20) + outcome = np.array([0] * 16 + [1] * 4) + frozen = fit_binary_outcome_recipe( + score, + outcome, + alpha=0.10, + n_groups=1, + bin_edges=(0.0, 1.0), + taxonomy_provenance="test", + taxonomy_method="fixed_test", + ) + universe = pd.DataFrame( + { + "loan_status": ["Fully Paid"] * 16 + ["Charged Off"] * 4, + "last_pymnt_d": ["Jan-2012"] * 20, + "design_split": ["conformal_fit"] * 20, + "issue_d": pd.to_datetime(["2012-01-01"] * 20), + } + ) + config = { + "source": {"information_cutoff": "2016-03-31"}, + "conformal": {"canonical_groups": 1, "alpha": 0.10}, + "lag_sensitivity": { + "phase_stratum": 0, + "minimum_monthly_label_retention": 0.99, + }, + "residual_specification": { + "windows": [{"id": "w01", "start": "2012-01-01", "end": "2012-06-30"}] + }, + } + result = build_label_lag_phase_sensitivity( + universe, + score, + {"w01": {1: frozen}}, + config, + lag_months=[0], + ) + assert result.loc[0, "retained_rows"] == 20 + assert result.loc[0, "phase_prevalence"] == 0.20 + assert bool(result.loc[0, "passes_locked_retention"]) diff --git a/tests/test_ijds_audit/test_missingness_sensitivity.py b/tests/test_ijds_audit/test_missingness_sensitivity.py new file mode 100644 index 0000000..75ce683 --- /dev/null +++ b/tests/test_ijds_audit/test_missingness_sensitivity.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +from src.ijds_audit.missingness_sensitivity import ( + build_missingness_variant, + load_missingness_config, +) +from src.ijds_audit.prediction import PreparedData + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "configs/experiments/ijds_missingness_sensitivity_2026-07-15_v3.yaml" + + +def _prepared() -> PreparedData: + universe = pd.DataFrame( + { + "mths_since_last_delinq": [None, 12.0, 24.0], + "pub_rec_bankruptcies": [None, 0.0, 2.0], + } + ) + features = pd.DataFrame( + { + "delinq_recency": [999.0, 12.0, 24.0], + "has_bankruptcy": [0, 0, 1], + "income": [1.0, 2.0, 3.0], + "purpose": ["a", "b", "c"], + } + ) + return PreparedData( + universe=universe, + features=features, + numeric_features=("delinq_recency", "has_bankruptcy", "income"), + categorical_features=("purpose",), + source_inventory={}, + availability_audit=pd.DataFrame(), + monthly_residual_availability=pd.DataFrame(), + ) + + +def test_missingness_protocol_is_closed_and_no_selection() -> None: + config = load_missingness_config(CONFIG, repo_root=ROOT) + assert [item["id"] for item in config["specifications"]] == [ + "catboost_platt", + "catboost_missing_indicators_platt", + "catboost_native_missing_platt", + ] + assert config["evaluation"]["no_model_selection"] is True + assert config["evaluation"]["no_portfolio_optimization"] is True + assert config["protocol_tag"].endswith("2026-07-15-v3") + assert config["specifications"][2]["added_numeric_features"] == [ + "delinq_recency_native", + "has_bankruptcy_native", + ] + + +def test_explicit_indicator_variant_retains_active_mappings() -> None: + variant = build_missingness_variant(_prepared(), variant="explicit_indicators") + assert variant.features["delinq_recency"].tolist() == [999.0, 12.0, 24.0] + assert variant.features["delinq_recency_missing"].tolist() == [1, 0, 0] + assert variant.features["bankruptcy_count_missing"].tolist() == [1, 0, 0] + + +def test_native_variant_preserves_feature_semantics_and_nan() -> None: + variant = build_missingness_variant(_prepared(), variant="native_missing") + assert "delinq_recency" not in variant.features + assert "has_bankruptcy" not in variant.features + assert pd.isna(variant.features.loc[0, "delinq_recency_native"]) + assert pd.isna(variant.features.loc[0, "has_bankruptcy_native"]) + assert variant.features.loc[1, "delinq_recency_native"] == 12.0 + assert variant.features["has_bankruptcy_native"].iloc[1:].tolist() == [0.0, 1.0] + assert "bankruptcy_count_native" not in variant.features diff --git a/tests/test_ijds_audit/test_publication_generation.py b/tests/test_ijds_audit/test_publication_generation.py new file mode 100644 index 0000000..13f6d16 --- /dev/null +++ b/tests/test_ijds_audit/test_publication_generation.py @@ -0,0 +1,155 @@ +"""Contracts for transactional publication evidence generation.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from src.ijds_audit import publication_generation +from src.ijds_audit.publication_generation import ( + PUBLICATION_IMPLEMENTATION_PATHS, + promote_publication_generation, + publication_implementation_descriptors, + staged_artifact_descriptor, + staged_output_path, +) + +REPO = Path(__file__).resolve().parents[2] + + +def test_implementation_inventory_binds_every_acceptance_dependency() -> None: + required = { + "active_source_registry", + "claim_ledger_contract", + "publication_targets_contract", + "evidence_builder", + "publication_integrity_checker", + "paper_pdf_auditor", + "publication_generation_helper", + "v4_config_loader", + "grid_contracts", + "endpoint_availability_sensitivity/loader", + "portfolio_structure_sensitivity/loader", + "robustness_sensitivities/loader", + "claim_ledger_loader", + "source_registry_loader", + "artifact_descriptor_helper", + "pipeline_runtime_helper", + } + assert set(PUBLICATION_IMPLEMENTATION_PATHS) == required + descriptors = publication_implementation_descriptors(REPO) + assert set(descriptors) == required + for descriptor in descriptors.values(): + assert (REPO / descriptor["path"]).is_file() + assert descriptor["bytes"] > 0 + assert len(descriptor["sha256"]) == 64 + + +def test_staged_descriptor_uses_canonical_target_path(tmp_path: Path) -> None: + repo = tmp_path + transaction = repo / ".transaction" + target = repo / "reports/tables/table.csv" + staged = staged_output_path(transaction, target, repo_root=repo) + staged.write_text("value\n1\n", encoding="utf-8") + + descriptor = staged_artifact_descriptor(staged, target, repo_root=repo) + + assert descriptor["path"] == "reports/tables/table.csv" + assert descriptor["bytes"] == staged.stat().st_size + assert len(descriptor["sha256"]) == 64 + + +def test_promotion_replaces_manifest_after_every_artifact( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path + transaction = repo / ".transaction" + targets = [repo / "reports/b.csv", repo / "reports/a.csv"] + artifacts: dict[Path, Path] = {} + for index, target in enumerate(targets): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"old-{index}", encoding="utf-8") + staged = staged_output_path(transaction, target, repo_root=repo) + staged.write_text(f"new-{index}", encoding="utf-8") + artifacts[target] = staged + manifest = repo / "reports/evidence.json" + manifest.write_text("old-manifest", encoding="utf-8") + staged_manifest = staged_output_path(transaction, manifest, repo_root=repo) + staged_manifest.write_text("new-manifest", encoding="utf-8") + + real_replace = os.replace + calls: list[tuple[Path, Path]] = [] + + def recording_replace( + source: str | os.PathLike[str], + destination: str | os.PathLike[str], + ) -> None: + calls.append((Path(source).resolve(), Path(destination).resolve())) + real_replace(source, destination) + + monkeypatch.setattr(publication_generation.os, "replace", recording_replace) + promoted = promote_publication_generation( + artifacts, + staged_manifest=staged_manifest, + manifest_target=manifest, + repo_root=repo, + transaction_root=transaction, + ) + + assert promoted[-1] == manifest.resolve() + assert calls[-1][1] == manifest.resolve() + assert [target for _, target in calls[:-1]] == sorted( + (target.resolve() for target in targets), + key=Path.as_posix, + ) + assert manifest.read_text(encoding="utf-8") == "new-manifest" + assert {target.read_text(encoding="utf-8") for target in targets} == {"new-0", "new-1"} + + +def test_failed_manifest_promotion_restores_previous_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path + transaction = repo / ".transaction" + targets = [repo / "reports/a.csv", repo / "reports/b.csv"] + artifacts: dict[Path, Path] = {} + for index, target in enumerate(targets): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"old-{index}", encoding="utf-8") + staged = staged_output_path(transaction, target, repo_root=repo) + staged.write_text(f"new-{index}", encoding="utf-8") + artifacts[target] = staged + manifest = repo / "reports/evidence.json" + manifest.write_text("old-manifest", encoding="utf-8") + staged_manifest = staged_output_path(transaction, manifest, repo_root=repo) + staged_manifest.write_text("new-manifest", encoding="utf-8") + + real_replace = os.replace + injected = False + + def fail_once_on_manifest( + source: str | os.PathLike[str], + destination: str | os.PathLike[str], + ) -> None: + nonlocal injected + if Path(source).resolve() == staged_manifest.resolve() and not injected: + injected = True + raise OSError("injected manifest promotion failure") + real_replace(source, destination) + + monkeypatch.setattr(publication_generation.os, "replace", fail_once_on_manifest) + with pytest.raises(OSError, match="injected manifest promotion failure"): + promote_publication_generation( + artifacts, + staged_manifest=staged_manifest, + manifest_target=manifest, + repo_root=repo, + transaction_root=transaction, + ) + + assert manifest.read_text(encoding="utf-8") == "old-manifest" + assert [target.read_text(encoding="utf-8") for target in targets] == ["old-0", "old-1"] diff --git a/tests/test_ijds_audit/test_publication_integrity_scope.py b/tests/test_ijds_audit/test_publication_integrity_scope.py new file mode 100644 index 0000000..53f0bc5 --- /dev/null +++ b/tests/test_ijds_audit/test_publication_integrity_scope.py @@ -0,0 +1,10 @@ +"""Scope boundaries for active publication-integrity scanning.""" + +from __future__ import annotations + +from scripts.check_publication_integrity import ACTIVE_EDITORIAL_SURFACES, REPO + + +def test_historical_extraction_manifest_is_not_an_active_editorial_surface() -> None: + assert REPO / "EXTRACTION_MANIFEST.md" not in ACTIVE_EDITORIAL_SURFACES + assert REPO / "docs/research/active_claims_2026-07-14.md" in ACTIVE_EDITORIAL_SURFACES diff --git a/tests/test_ijds_audit/test_publication_sources.py b/tests/test_ijds_audit/test_publication_sources.py new file mode 100644 index 0000000..67d8c70 --- /dev/null +++ b/tests/test_ijds_audit/test_publication_sources.py @@ -0,0 +1,435 @@ +"""Tests for the active evidence source registry.""" + +from __future__ import annotations + +import hashlib +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from src.ijds_audit.publication_sources import ( + active_lineage_run_tags, + load_source_registry, + load_verified_source_registry, +) +from src.utils.artifact_descriptor import relative_artifact_descriptor + +ROOT = Path(__file__).resolve().parents[2] + +FREEZE_TAG = "binary-freeze-v1" +EVALUATION_TAG = "binary-evaluation-v2" +DIAGNOSTIC_TAG = "raw-data-audit-v1" +SENSITIVITY_TAG = "endpoint-sensitivity-v1" +DVC_ROOTS = ("data/processed", "models") + + +def _protocol_identity( + run_tag: str, + *, + paper_role: str, + dvc_tracked: bool, + commit: str, +) -> dict[str, Any]: + return { + "run_tag": run_tag, + "protocol_tag": f"protocol/{run_tag}", + "protocol_commit": commit, + "scientific_uv_lock_sha256": "a" * 64, + "paper_role": paper_role, + "dvc_tracked": dvc_tracked, + } + + +def _explicit_payload(root: Path) -> dict[str, Any]: + source = root / "evidence" / "source.json" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text('{"status": "fixture"}\n', encoding="utf-8") + tracked_tags = (FREEZE_TAG, EVALUATION_TAG, SENSITIVITY_TAG) + return { + "schema_version": "test-explicit-v1", + "status": "active_ijds_paper_evidence_source_registry", + "lineages": { + "binary_geometry": { + "outcome_free": _protocol_identity( + FREEZE_TAG, + paper_role="outcome_free", + dvc_tracked=True, + commit="1" * 40, + ), + "evaluation": _protocol_identity( + EVALUATION_TAG, + paper_role="evaluation", + dvc_tracked=True, + commit="2" * 40, + ), + } + }, + "diagnostics": { + "raw_data_audit": { + "run_tag": DIAGNOSTIC_TAG, + "status": "complete_fixture_diagnostic", + "paper_role": "diagnostic", + "dvc_tracked": False, + } + }, + "sensitivities": { + "endpoint_availability": _protocol_identity( + SENSITIVITY_TAG, + paper_role="assumption_sensitivity", + dvc_tracked=True, + commit="3" * 40, + ) + }, + "dvc_pointers": [ + f"{prefix}/experiments/ijds_audit/{run_tag}.dvc" + for run_tag in tracked_tags + for prefix in DVC_ROOTS + ], + "sources": { + "fixture": relative_artifact_descriptor(source, repo_root=root), + }, + } + + +def _valid_pointer_payload(run_tag: str) -> dict[str, Any]: + return { + "outs": [ + { + "md5": f"{'a' * 32}.dir", + "size": 128, + "nfiles": 2, + "hash": "md5", + "path": run_tag, + } + ] + } + + +def _write_yaml(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + +def _write_registry(root: Path, payload: dict[str, Any]) -> Path: + path = root / "configs" / "ijds_active_evidence_sources.yaml" + _write_yaml(path, payload) + return path + + +def _materialize_registry(root: Path, payload: dict[str, Any]) -> Path: + for pointer in payload["dvc_pointers"]: + pointer_path = root / pointer + _write_yaml(pointer_path, _valid_pointer_payload(pointer_path.stem)) + return _write_registry(root, payload) + + +def test_active_evidence_registry_verifies_every_source() -> None: + payload, sources = load_verified_source_registry( + ROOT / "configs/ijds_active_evidence_sources.yaml", + repo_root=ROOT, + ) + assert payload["schema_version"] == "2026-07-15.4" + assert set(sources) == { + "v4_config", + "v4_summary", + "v4_receipt", + "two_ruler_manifest", + "credit_summary", + "credit_receipt", + "raw_data_audit", + "label_lag_sensitivity", + "solver_tie_audit", + "endpoint_sensitivity_summary", + "structural_sensitivity_config", + "structural_sensitivity_freeze", + "structural_sensitivity_summary", + "rolling_origin_summary", + "rolling_origin_receipt", + "missingness_summary", + "missingness_receipt", + "fit_label_completion_freeze", + "fit_label_completion_summary", + "allocation_granularity_freeze", + "allocation_granularity_summary", + } + assert len(payload["dvc_pointers"]) == 31 + assert payload["lineages"]["binary_geometry"]["evaluation"]["run_tag"].endswith("2026-07-15-v5") + + +def test_active_registry_returns_all_dvc_run_tags_in_causal_config_order() -> None: + payload = load_source_registry(ROOT / "configs/ijds_active_evidence_sources.yaml") + + assert active_lineage_run_tags(payload) == ( + "ijds-binary-geometry-frontier-v4-2026-07-12-v1", + "ijds-binary-geometry-frontier-v4-2026-07-15-v5", + "ijds-normalized-objective-frontier-2026-07-13-v1c", + "ijds-normalized-objective-frontier-2026-07-15-v5", + "ijds-credit-risk-controls-2026-07-13-v1b", + "ijds-credit-risk-controls-2026-07-15-v5", + "ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "ijds-portfolio-structure-sensitivity-2026-07-15-v6", + "ijds-rolling-origin-2017-2026-07-15-v4", + "ijds-missingness-sensitivity-2026-07-15-v3", + "ijds-fit-label-completion-sensitivity-2026-07-16-v2", + "ijds-allocation-granularity-sensitivity-2026-07-16-v3", + "ijds-binary-geometry-frontier-v4-2026-07-14-v3", + "ijds-normalized-objective-frontier-2026-07-14-v3", + "ijds-credit-risk-controls-2026-07-14-v3", + "ijds-portfolio-structure-sensitivity-2026-07-15-v5", + ) + + +def test_tracked_unit_can_declare_one_dvc_root(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + run_tag = "data-only-replay-dependency" + payload["replay_dependencies"] = { + "fixture": { + **_protocol_identity( + run_tag, + paper_role="non_evidence_replay_dependency", + dvc_tracked=True, + commit="4" * 40, + ), + "dvc_roots": ["data/processed"], + } + } + pointer = f"data/processed/experiments/ijds_audit/{run_tag}.dvc" + payload["dvc_pointers"].append(pointer) + registry_path = _materialize_registry(tmp_path, payload) + + loaded = load_source_registry(registry_path, repo_root=tmp_path) + + assert active_lineage_run_tags(loaded)[-1] == run_tag + + +def test_registry_rejects_dvc_roots_on_untracked_unit(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + payload["diagnostics"]["raw_data_audit"]["dvc_roots"] = ["data/processed"] + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(ValueError, match="requires dvc_tracked=true"): + load_source_registry(registry_path) + + +def test_explicit_paper_roles_and_dvc_tracking_control_pointer_contract(tmp_path: Path) -> None: + registry_path = _materialize_registry(tmp_path, _explicit_payload(tmp_path)) + + payload = load_source_registry(registry_path, repo_root=tmp_path) + + assert payload["sensitivities"]["endpoint_availability"]["paper_role"] == ( + "assumption_sensitivity" + ) + assert active_lineage_run_tags(payload) == ( + FREEZE_TAG, + EVALUATION_TAG, + SENSITIVITY_TAG, + ) + + +def test_registry_rejects_duplicate_run_tags_across_identity_sections(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + payload["diagnostics"]["raw_data_audit"]["run_tag"] = FREEZE_TAG + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(ValueError, match="run tags must be globally unique"): + load_source_registry(registry_path) + + +@pytest.mark.parametrize( + ("section", "group", "unit"), + [ + ("lineages", "binary_geometry", "outcome_free"), + ("diagnostics", None, "raw_data_audit"), + ("sensitivities", None, "endpoint_availability"), + ], +) +def test_registry_rejects_missing_run_tag_in_every_identity_section( + tmp_path: Path, + section: str, + group: str | None, + unit: str, +) -> None: + payload = _explicit_payload(tmp_path) + section_payload = payload[section] + identity = section_payload[group][unit] if group is not None else section_payload[unit] + del identity["run_tag"] + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(TypeError, match=r"Missing registry identity: .*run_tag"): + load_source_registry(registry_path) + + +def test_registry_rejects_incomplete_protocol_identity(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + del payload["sensitivities"]["endpoint_availability"]["protocol_commit"] + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(TypeError, match="protocol_commit"): + load_source_registry(registry_path) + + +def test_registry_requires_scientific_lock_for_protocol_identity(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + del payload["sensitivities"]["endpoint_availability"]["scientific_uv_lock_sha256"] + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(TypeError, match="scientific_uv_lock_sha256"): + load_source_registry(registry_path) + + +def test_registry_rejects_malformed_scientific_lock(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + payload["sensitivities"]["endpoint_availability"]["scientific_uv_lock_sha256"] = "ABC" + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(ValueError, match="64-character lowercase"): + load_source_registry(registry_path) + + +def test_registry_verifies_lock_from_protocol_commit(tmp_path: Path) -> None: + subprocess.run(["git", "init", "--quiet"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "Test"], cwd=tmp_path, check=True) + lock = b"version = 1\n" + (tmp_path / "uv.lock").write_bytes(lock) + subprocess.run(["git", "add", "uv.lock"], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "--quiet", "-m", "fixture"], cwd=tmp_path, check=True) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + payload = _explicit_payload(tmp_path) + for section in ("lineages", "diagnostics", "sensitivities"): + del payload[section] + run_tag = "fixture-protocol" + protocol_tag = f"protocol/{run_tag}" + subprocess.run(["git", "tag", protocol_tag], cwd=tmp_path, check=True) + payload["lineages"] = { + "fixture": { + **_protocol_identity( + run_tag, + paper_role="outcome_free", + dvc_tracked=True, + commit=commit, + ), + "scientific_uv_lock_sha256": hashlib.sha256(lock).hexdigest(), + } + } + payload["dvc_pointers"] = [f"{root}/experiments/ijds_audit/{run_tag}.dvc" for root in DVC_ROOTS] + registry_path = _materialize_registry(tmp_path, payload) + + load_source_registry(registry_path, repo_root=tmp_path) + + payload["lineages"]["fixture"]["scientific_uv_lock_sha256"] = "0" * 64 + registry_path = _write_registry(tmp_path, payload) + with pytest.raises(RuntimeError, match="commit contains"): + load_source_registry(registry_path, repo_root=tmp_path) + + +def test_registry_rejects_null_explicit_dvc_tracking(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + payload["diagnostics"]["raw_data_audit"]["dvc_tracked"] = None + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(TypeError, match="dvc_tracked must be boolean"): + load_source_registry(registry_path) + + +@pytest.mark.parametrize("field", ["paper_role", "dvc_tracked"]) +def test_registry_rejects_partially_explicit_identity(tmp_path: Path, field: str) -> None: + payload = _explicit_payload(tmp_path) + del payload["sensitivities"]["endpoint_availability"][field] + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(TypeError, match="require both paper_role and dvc_tracked"): + load_source_registry(registry_path) + + +@pytest.mark.parametrize("mutation", ["missing", "unexpected"]) +def test_registry_rejects_missing_or_unexpected_dvc_pointers( + tmp_path: Path, + mutation: str, +) -> None: + payload = _explicit_payload(tmp_path) + if mutation == "missing": + payload["dvc_pointers"].pop() + else: + payload["dvc_pointers"].append(f"models/experiments/ijds_audit/{DIAGNOSTIC_TAG}.dvc") + registry_path = _write_registry(tmp_path, payload) + + with pytest.raises(ValueError, match="do not match"): + load_source_registry(registry_path) + + +def test_registry_rejects_pointer_out_path_mismatch(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + registry_path = _materialize_registry(tmp_path, payload) + pointer_path = tmp_path / payload["dvc_pointers"][0] + pointer_payload = _valid_pointer_payload(pointer_path.stem) + pointer_payload["outs"][0]["path"] = "different-run" + _write_yaml(pointer_path, pointer_payload) + + with pytest.raises(ValueError, match="does not match run directory"): + load_source_registry(registry_path, repo_root=tmp_path) + + +def test_registry_normalizes_pointer_out_path_before_matching(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + registry_path = _materialize_registry(tmp_path, payload) + pointer_path = tmp_path / payload["dvc_pointers"][0] + pointer_payload = _valid_pointer_payload(pointer_path.stem) + pointer_payload["outs"][0]["path"] = f"./{pointer_path.stem}/" + _write_yaml(pointer_path, pointer_payload) + + loaded = load_source_registry(registry_path, repo_root=tmp_path) + + assert loaded["schema_version"] == "test-explicit-v1" + + +@pytest.mark.parametrize( + ("mutation", "expected_message"), + [ + ("outs", "exactly one out"), + ("md5", "md5 must be"), + ("size", "size must be a non-negative integer"), + ("nfiles", "nfiles must be a non-negative integer"), + ], +) +def test_registry_rejects_malformed_pointer_structure( + tmp_path: Path, + mutation: str, + expected_message: str, +) -> None: + payload = _explicit_payload(tmp_path) + registry_path = _materialize_registry(tmp_path, payload) + pointer_path = tmp_path / payload["dvc_pointers"][0] + pointer_payload = _valid_pointer_payload(pointer_path.stem) + if mutation == "outs": + pointer_payload["outs"] = [] + elif mutation == "md5": + pointer_payload["outs"][0]["md5"] = "not-an-md5" + elif mutation == "size": + pointer_payload["outs"][0]["size"] = True + else: + pointer_payload["outs"][0]["nfiles"] = -1 + _write_yaml(pointer_path, pointer_payload) + + with pytest.raises((TypeError, ValueError), match=expected_message): + load_source_registry(registry_path, repo_root=tmp_path) + + +def test_registry_rejects_malformed_pointer_yaml(tmp_path: Path) -> None: + payload = _explicit_payload(tmp_path) + registry_path = _materialize_registry(tmp_path, payload) + pointer_path = tmp_path / payload["dvc_pointers"][0] + pointer_path.write_text("outs:\n- md5: [\n", encoding="utf-8") + + with pytest.raises(ValueError, match="Malformed active DVC pointer YAML"): + load_source_registry(registry_path, repo_root=tmp_path) diff --git a/tests/test_ijds_audit/test_raw_data_audit.py b/tests/test_ijds_audit/test_raw_data_audit.py new file mode 100644 index 0000000..9ccda4a --- /dev/null +++ b/tests/test_ijds_audit/test_raw_data_audit.py @@ -0,0 +1,64 @@ +"""Tests for the full-archive IJDS data-contract audit.""" + +from __future__ import annotations + +import pandas as pd + +from src.ijds_audit.raw_data_audit import _feature_contract, classify_raw_column + + +def test_post_outcome_columns_are_never_model_eligible() -> None: + role, reason = classify_raw_column("total_pymnt", active_required=["total_pymnt"]) + assert role == "post_outcome_or_servicing" + assert "never eligible" in reason + + +def test_late_joint_application_fields_have_explicit_role() -> None: + role, reason = classify_raw_column("sec_app_fico_range_low", active_required=[]) + assert role == "joint_application_origination" + assert "introduced late" in reason + + +def test_active_origination_field_is_distinct_from_metadata() -> None: + role, _ = classify_raw_column("fico_range_low", active_required=["fico_range_low"]) + assert role == "active_protocol_input" + + +def test_feature_contract_applies_threshold_and_declared_exceptions() -> None: + coverage = pd.DataFrame( + { + "feature": ["ordinary", "nullable"] * 4, + "cohort": [ + "pd_development", + "pd_development", + "probability_calibration", + "probability_calibration", + "conformal_fit", + "conformal_fit", + "primary_oot", + "primary_oot", + ], + "coverage": [0.96, 0.30, 0.97, 0.35, 0.98, 0.40, 0.99, 0.50], + } + ) + rules = { + "minimum_fitting_feature_coverage": 0.95, + "late_feature_fitting_coverage": 0.50, + "late_feature_primary_coverage": 0.80, + "active_feature_coverage_exceptions": { + "nullable": { + "type": "structural", + "missingness_semantics": "explicit sentinel", + "requires_sensitivity": True, + } + }, + } + contract = _feature_contract( + coverage, + columns=["ordinary", "nullable"], + active_required=["ordinary", "nullable"], + rules=rules, + ).set_index("feature") + assert contract.loc["ordinary", "decision"] == "eligible" + assert contract.loc["nullable", "decision"] == "eligible_with_declared_coverage_exception" + assert bool(contract.loc["nullable", "requires_sensitivity"]) diff --git a/tests/test_ijds_audit/test_robustness_evidence.py b/tests/test_ijds_audit/test_robustness_evidence.py new file mode 100644 index 0000000..2b14e1b --- /dev/null +++ b/tests/test_ijds_audit/test_robustness_evidence.py @@ -0,0 +1,113 @@ +"""Contracts for fit-label completion and allocation-granularity evidence.""" + +from __future__ import annotations + +from pathlib import Path + +from src.ijds_audit.publication_sources import load_verified_source_registry +from src.ijds_audit.robustness_evidence import ( + allocation_granularity_publication_table, + fit_label_completion_publication_table, + load_allocation_granularity_evidence, + load_fit_label_completion_evidence, +) + +REPO = Path(__file__).resolve().parents[2] +REGISTRY = REPO / "configs/ijds_active_evidence_sources.yaml" + + +def _registry(): + return load_verified_source_registry(REGISTRY, repo_root=REPO) + + +def test_active_fit_label_completion_is_complete_unselected_and_hash_verified() -> None: + registry, sources = _registry() + evidence = load_fit_label_completion_evidence( + sources["fit_label_completion_summary"], + freeze_path=sources["fit_label_completion_freeze"], + identity=registry["sensitivities"]["fit_label_completion"], + repo_root=REPO, + ) + + assert evidence.findings == { + "coverage_cells": 32, + "phase_cells": 32, + "all_scenarios_all_windows_upper_below_nominal": True, + "w7_w8_crossing_by_scenario": { + "observed_only": True, + "all_unavailable_nondefault": True, + "all_unavailable_default": False, + "hindsight_terminal": True, + }, + "w7_w8_crossing_scenarios": 3, + "w7_w8_crossing_in_all_scenarios": False, + "unavailable_fit_labels_by_split": { + "pd_development": 41, + "probability_calibration": 24, + "conformal_fit": 150, + }, + "unavailable_fit_labels_total": 215, + } + assert evidence.freeze["evaluation_outcome_columns_passed_to_fitting"] == [] + assert set(evidence.outcome_free_artifacts) == { + "scores", + "fit_audit", + "scenario_audit", + "recipes", + } + + +def test_fit_label_publication_table_reports_all_declared_corners() -> None: + registry, sources = _registry() + evidence = load_fit_label_completion_evidence( + sources["fit_label_completion_summary"], + freeze_path=sources["fit_label_completion_freeze"], + identity=registry["sensitivities"]["fit_label_completion"], + repo_root=REPO, + ) + table = fit_label_completion_publication_table(evidence) + + assert len(table) == 4 + assert table["fit_label_scenario"].is_unique + assert table["windows_upper_below_nominal"].eq(8).all() + assert table["coverage_upper_max"].lt(0.90).all() + assert int(table["w7_w8_stratum2_crossing"].sum()) == 3 + + +def test_active_allocation_granularity_is_complete_unselected_and_hash_verified() -> None: + registry, sources = _registry() + evidence = load_allocation_granularity_evidence( + sources["allocation_granularity_summary"], + freeze_path=sources["allocation_granularity_freeze"], + identity=registry["sensitivities"]["allocation_granularity"], + repo_root=REPO, + ) + + assert evidence.findings["tracks"] == 96 + assert evidence.findings["portfolios"] == 1440 + assert evidence.findings["changed_rows"] == 2985 + assert evidence.findings["cash_share_max"] < 3.4e-5 + assert evidence.findings["payoff_rate_perturbation_abs_max"] < 3.5e-6 + assert evidence.findings["default_rate_perturbation_abs_max"] < 1.3e-5 + assert evidence.findings["miscoverage_rate_perturbation_abs_max"] < 1.2e-5 + assert evidence.freeze["outcome_columns_passed_to_rounding"] == [] + assert set(evidence.outcome_free_artifacts) == { + "rounded_allocations", + "rounded_solve_records", + "granularity_audit", + } + + +def test_allocation_granularity_publication_table_is_one_complete_row() -> None: + registry, sources = _registry() + evidence = load_allocation_granularity_evidence( + sources["allocation_granularity_summary"], + freeze_path=sources["allocation_granularity_freeze"], + identity=registry["sensitivities"]["allocation_granularity"], + repo_root=REPO, + ) + table = allocation_granularity_publication_table(evidence) + + assert len(table) == 1 + assert int(table.iloc[0]["tracks"]) == 96 + assert int(table.iloc[0]["source_rows"]) == 143175 diff --git a/tests/test_ijds_audit/test_sensitivity_evidence.py b/tests/test_ijds_audit/test_sensitivity_evidence.py new file mode 100644 index 0000000..f03dda7 --- /dev/null +++ b/tests/test_ijds_audit/test_sensitivity_evidence.py @@ -0,0 +1,116 @@ +"""Contracts for paper-facing endpoint sensitivity evidence.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from src.ijds_audit.sensitivity_evidence import ( + _validate_endpoint_frames, + endpoint_publication_table, + load_endpoint_sensitivity_evidence, +) + +ROOT = Path(__file__).resolve().parents[2] +SUMMARY = ( + ROOT + / "models/experiments/ijds_audit" + / "ijds-endpoint-availability-sensitivity-2026-07-14-v1" + / "endpoint_sensitivity_summary.json" +) +IDENTITY = { + "run_tag": "ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_tag": "protocol/ijds-endpoint-availability-sensitivity-2026-07-14-v1", + "protocol_commit": "8865f1cfbd387576bdf805f3e52f030261e4b717", +} + + +def _load(): + return load_endpoint_sensitivity_evidence( + SUMMARY, + identity=IDENTITY, + repo_root=ROOT, + reference_coverage=pd.read_parquet( + ROOT + / "data/processed/experiments/ijds_audit" + / "ijds-credit-risk-controls-2026-07-14-v3/evaluation/temporal_coverage.parquet" + ), + reference_two_ruler=pd.read_parquet( + ROOT + / "data/processed/experiments/ijds_audit" + / "ijds-normalized-objective-frontier-2026-07-14-v3" + / "evaluation/window_endpoint_contrasts.parquet" + ), + reference_envelopes=pd.read_parquet( + ROOT + / "data/processed/experiments/ijds_audit" + / "ijds-binary-geometry-frontier-v4-2026-07-14-v3" + / "evaluation/comparator_envelopes.parquet" + ), + ) + + +def test_endpoint_sensitivity_is_complete_and_reconciles_exactly() -> None: + evidence = _load() + + assert evidence.reconciliation == { + "charged_off_lag_months": 6, + "coverage_cells_exact": 120, + "two_ruler_contrasts_exact": 48, + "exact_support_envelopes_exact": 648, + "byte_value_equal_after_lag_column_removed": True, + } + table = endpoint_publication_table(evidence).set_index("charged_off_lag_months") + assert table.index.tolist() == [0, 3, 6, 8, 12] + assert table.loc[0, "primary_resolved"] == 364_861 + assert table.loc[6, "primary_resolved"] == 364_814 + assert table.loc[8, "primary_unresolved"] == 12_320 + assert table.loc[12, "primary_unresolved"] == 13_602 + assert table.loc[0, "coverage_upper_max"] == pytest.approx(0.8976411849455659) + assert table.loc[6, "coverage_upper_max"] == pytest.approx(0.8977258526333881) + assert table.loc[8, "coverage_upper_max"] == pytest.approx(0.8981507095313236) + assert table.loc[12, "coverage_upper_max"] == pytest.approx(0.9004108276571989) + assert table.loc[12, "coverage_upper_below_0_90_cells"] == 39 + assert table.loc[12, "coverage_upper_at_or_above_0_90_cells"] == 1 + assert table["broad_stress_exact_frontier_standardized_payoff_crosses_zero_cells"].eq(72).all() + assert table["broad_stress_exact_frontier_terminal_default_crosses_zero_cells"].eq(72).all() + assert table["broad_stress_exact_frontier_funded_miscoverage_crosses_zero_cells"].eq(72).all() + + +def test_endpoint_grid_validation_is_order_invariant() -> None: + evidence = _load() + shuffled = { + name: frame.sample(frac=1.0, random_state=20260715).reset_index(drop=True) + for name, frame in evidence.frames.items() + } + + _validate_endpoint_frames(shuffled) + + +def test_endpoint_grid_rejects_deleted_or_duplicate_cell() -> None: + evidence = _load() + deleted = {name: frame.copy() for name, frame in evidence.frames.items()} + deleted["coverage_cells"] = deleted["coverage_cells"].iloc[1:].reset_index(drop=True) + with pytest.raises(RuntimeError, match="grid changed"): + _validate_endpoint_frames(deleted) + + duplicated = {name: frame.copy() for name, frame in evidence.frames.items()} + duplicated["two_ruler_directions"] = pd.concat( + [duplicated["two_ruler_directions"].iloc[1:], duplicated["two_ruler_directions"].iloc[[1]]], + ignore_index=True, + ) + with pytest.raises(RuntimeError, match="duplicate grid keys"): + _validate_endpoint_frames(duplicated) + + +def test_endpoint_grid_rejects_vacuous_lag_subset() -> None: + evidence = _load() + frames = {name: frame.copy() for name, frame in evidence.frames.items()} + frames["coverage_summary"] = frames["coverage_summary"].loc[ + ~frames["coverage_summary"]["charged_off_lag_months"].eq(12) + ] + + with pytest.raises(RuntimeError, match="grid changed"): + _validate_endpoint_frames(frames) diff --git a/tests/test_ijds_audit/test_structural_checkpoint.py b/tests/test_ijds_audit/test_structural_checkpoint.py new file mode 100644 index 0000000..3db55db --- /dev/null +++ b/tests/test_ijds_audit/test_structural_checkpoint.py @@ -0,0 +1,79 @@ +"""Tests for structural-shard validation and zero-copy recovery.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pandas as pd +import pytest + +from src.ijds_audit.structural_checkpoint import ( + ARTIFACT_FILES, + EXPECTED_ROWS, + hardlink_structural_shard, + inspect_structural_shard, +) + +SCENARIO = "b0500k_p020_l025" + + +def _write_shard(root: Path, *, retry_slack: float = 1.0e-12) -> Path: + frontier = root / SCENARIO / "frontier" + frontier.mkdir(parents=True) + for name, filename in ARTIFACT_FILES.items(): + rows = EXPECTED_ROWS.get(name, 5) + frame = pd.DataFrame({"scenario_id": [SCENARIO] * rows}) + if name == "minimum_endpoint_diagnostics": + frame["minimum_endpoint_retried"] = [True] + [False] * (rows - 1) + frame["minimum_endpoint_retry_slack"] = [retry_slack] + [0.0] * (rows - 1) + frame["minimum_cap_residual"] = [retry_slack] + [0.0] * (rows - 1) + frame.to_parquet(frontier / filename, index=False) + return root / SCENARIO + + +def test_structural_shard_is_validated_and_hardlinked(tmp_path: Path) -> None: + source = _write_shard(tmp_path / "source") + inspection = inspect_structural_shard( + source, + scenario_id=SCENARIO, + retry_slacks=[1.0e-12, 1.0e-10], + cap_residual_tolerance=1.0e-8, + ) + + linked = hardlink_structural_shard( + inspection, + destination_root=tmp_path / "destination" / SCENARIO, + ) + + assert inspection.minimum_endpoint_retries == 1 + assert inspection.rows["solve_records"] == 1_440 + assert all(os.path.samefile(inspection.paths[name], path) for name, path in linked.items()) + + +def test_structural_shard_rejects_undeclared_retry_slack(tmp_path: Path) -> None: + source = _write_shard(tmp_path / "source", retry_slack=1.0e-9) + + with pytest.raises(RuntimeError, match="undeclared retry slack"): + inspect_structural_shard( + source, + scenario_id=SCENARIO, + retry_slacks=[1.0e-12], + cap_residual_tolerance=1.0e-8, + ) + + +def test_structural_shard_rejects_outcome_columns(tmp_path: Path) -> None: + source = _write_shard(tmp_path / "source") + allocation_path = source / "frontier" / ARTIFACT_FILES["allocations"] + frame = pd.read_parquet(allocation_path) + frame["snapshot_default"] = 0 + frame.to_parquet(allocation_path, index=False) + + with pytest.raises(RuntimeError, match="has outcomes"): + inspect_structural_shard( + source, + scenario_id=SCENARIO, + retry_slacks=[1.0e-12, 1.0e-10], + cap_residual_tolerance=1.0e-8, + ) diff --git a/tests/test_ijds_audit/test_structural_evidence.py b/tests/test_ijds_audit/test_structural_evidence.py new file mode 100644 index 0000000..8275bdf --- /dev/null +++ b/tests/test_ijds_audit/test_structural_evidence.py @@ -0,0 +1,83 @@ +"""Contracts for complete-grid portfolio-structure evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd + +from src.ijds_audit.publication_sources import load_verified_source_registry +from src.ijds_audit.structural_evidence import ( + load_structural_sensitivity_evidence, + structural_publication_table, +) + +REPO = Path(__file__).resolve().parents[2] +REGISTRY = REPO / "configs/ijds_active_evidence_sources.yaml" + + +def _load_active_structural_evidence(): + registry, sources = load_verified_source_registry(REGISTRY, repo_root=REPO) + two_ruler = json.loads(sources["two_ruler_manifest"].read_text(encoding="utf-8")) + reference = pd.read_parquet( + two_ruler["evaluation_artifacts"]["window_endpoint_contrasts"]["path"] + ) + return load_structural_sensitivity_evidence( + sources["structural_sensitivity_summary"], + freeze_path=sources["structural_sensitivity_freeze"], + config_path=sources["structural_sensitivity_config"], + identity=registry["sensitivities"]["portfolio_structure"], + repo_root=REPO, + reference_two_ruler=reference, + ) + + +def test_active_structural_evidence_is_complete_hash_verified_and_unselected() -> None: + evidence = _load_active_structural_evidence() + + assert evidence.summary["scenario_count"] == 36 + assert all(value is None for value in evidence.summary["selection"].values()) + assert evidence.summary["baseline_reconciliation_maxima"] == { + "realized_payoff_difference_lower": 0.0, + "realized_payoff_difference_upper": 0.0, + "weighted_default_difference_lower": 0.0, + "weighted_default_difference_upper": 0.0, + "weighted_miscoverage_difference_lower": 0.0, + "weighted_miscoverage_difference_upper": 0.0, + } + assert evidence.findings["complete_cartesian_grid"] is True + assert evidence.findings["universally_favorable_scenarios"] == 0 + assert evidence.findings["universally_adverse_scenarios"] == 0 + assert evidence.findings["portfolios_per_scenario"] == 1440 + assert evidence.findings["purpose_cap_binding_share_by_cap"] == { + "0.20": 1.0, + "0.25": 1.0, + "0.30": 1.0, + "1.00": 0.0, + } + assert evidence.findings["frontier_constraint_binding_share_by_budget"] == { + "500000": 1.0, + "1000000": 1.0, + "2000000": 1.0, + } + assert evidence.findings["maximum_loan_weight_by_budget"] == { + "500000": 0.08, + "1000000": 0.04, + "2000000": 0.02, + } + + +def test_structural_publication_table_reports_every_scenario_and_direction() -> None: + table = structural_publication_table(_load_active_structural_evidence()) + + assert len(table) == 36 + assert table["scenario_id"].is_unique + assert table["activity_portfolios"].eq(1440).all() + assert table["activity_frontier_constraint_binding_share"].eq(1.0).all() + for metric in ("standardized_payoff", "funded_default", "funded_binary_miscoverage"): + columns = [ + column for column in table if column.startswith(metric) and column.endswith("_cells") + ] + assert len(columns) == 4 + assert table[columns].sum(axis=1).eq(48).all() diff --git a/tests/test_ijds_audit/test_structural_sensitivity.py b/tests/test_ijds_audit/test_structural_sensitivity.py new file mode 100644 index 0000000..261b783 --- /dev/null +++ b/tests/test_ijds_audit/test_structural_sensitivity.py @@ -0,0 +1,19 @@ +"""Tests for the complete structural-sensitivity grid.""" + +from __future__ import annotations + +from src.ijds_audit.structural_sensitivity import declared_scenarios + + +def test_declared_structural_grid_is_complete_and_has_one_baseline() -> None: + config = { + "structural_grid": { + "budgets": [500_000.0, 1_000_000.0, 2_000_000.0], + "purpose_caps": [0.20, 0.25, 0.30, 1.00], + "lgds": [0.25, 0.45, 0.65], + } + } + scenarios = declared_scenarios(config) + assert len(scenarios) == 36 + assert len({item["scenario_id"] for item in scenarios}) == 36 + assert sum(item["is_baseline"] for item in scenarios) == 1 diff --git a/tests/test_ijds_audit_core.py b/tests/test_ijds_audit_core.py new file mode 100644 index 0000000..2131781 --- /dev/null +++ b/tests/test_ijds_audit_core.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import yaml + +from src.evaluation.coverage_transport import binary_miscoverage_bounds +from src.evaluation.standardized_credit_payoff import expected_objective_coefficients +from src.ijds_audit.config import load_v4_config +from src.ijds_audit.evaluation import build_archive_outcomes +from src.ijds_audit.geometry import ( + BOTH, + EMPTY, + ONE_ONLY, + ZERO_ONLY, + binary_set_codes, + constant_score_population_phase, + summarize_binary_geometry, +) +from src.ijds_audit.portfolio import ( + PointPortfolioSession, + c2_cap, + enumerate_basis_breakpoints, + solve_point_portfolio, + verify_c2_dominance, +) +from src.ijds_audit.protocol import expand_frontier_for_window +from src.ijds_audit.simulation import run_factorial_simulation +from src.models.binary_conformal_guardrail import fit_binary_outcome_recipe +from src.optimization.portfolio_model import solve_portfolio_highspy_native + +ROOT = Path(__file__).resolve().parents[1] + + +def test_v4_config_is_closed_and_complete() -> None: + config = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + ) + assert len(config["residual_specification"]["windows"]) == 8 + assert config["design"]["policy_development_start"] == "2013-02-01" + + recovery = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12_v2.yaml" + ) + assert recovery["run_tag"].endswith("-v2") + assert recovery["resume_outcome_free"]["source_run_tag"].endswith("-v1") + + endpoint_recovery = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + ) + contract = endpoint_recovery["target"]["evaluation_outcome_contract"] + assert endpoint_recovery["run_tag"].endswith("-v3") + assert contract["mode"] == "conservative_terminal_status_reconstruction" + assert contract["archive_is_verified_point_in_time_snapshot"] is False + + +def test_v4_config_rejects_unknown_critical_keys(tmp_path: Path) -> None: + config = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-14_v3.yaml" + ) + config["policy"]["budegt"] = config["policy"]["budget"] + path = tmp_path / "typo.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + with pytest.raises(KeyError, match="budegt"): + load_v4_config(path) + + +def test_binary_set_codes_and_summary() -> None: + lower = np.array([0.1, 0.0, 0.2, 0.0]) + upper = np.array([0.9, 0.8, 1.0, 1.0]) + np.testing.assert_array_equal( + binary_set_codes(lower, upper), + np.array([EMPTY, ZERO_ONLY, ONE_ONLY, BOTH]), + ) + summary = summarize_binary_geometry(lower, upper) + assert summary["set_empty_count"] == 1 + assert summary["set_both_count"] == 1 + assert summary["width_q50"] == pytest.approx(0.8) + + +def test_binary_coverage_bounds_respect_empty_singleton_and_both_sets() -> None: + outcomes = np.full(4, np.nan) + lower = np.array([0.1, 0.0, 0.2, 0.0]) + upper = np.array([0.9, 0.8, 1.0, 1.0]) + miss_low, miss_high = binary_miscoverage_bounds(outcomes, lower, upper) + assert 1.0 - miss_high.mean() == pytest.approx(0.25) + assert 1.0 - miss_low.mean() == pytest.approx(0.75) + + +def test_reconstructed_endpoint_censors_terminal_status_after_cutoff() -> None: + universe = pd.DataFrame( + { + "id": pd.Series(["a", "b", "c"], dtype="string"), + "terminal_default": pd.Series([0, 1, pd.NA], dtype="Int8"), + "label_available_at": pd.to_datetime(["2020-09-01", "2020-10-01", None]), + "design_split": ["primary_oot"] * 3, + "issue_d": pd.to_datetime(["2016-04-01"] * 3), + } + ) + outcomes = build_archive_outcomes(universe, evaluation_cutoff="2020-09-30") + assert outcomes["snapshot_default"].tolist() == [0, pd.NA, pd.NA] + assert outcomes["snapshot_resolution"].tolist() == [ + "fully_paid_by_reconstructed_cutoff", + "terminal_after_reconstructed_cutoff", + "nonterminal_or_unresolved_status", + ] + + +def test_constant_score_phase_transition() -> None: + low_prevalence = constant_score_population_phase(score=0.08, prevalence=0.09, alpha=0.10) + high_prevalence = constant_score_population_phase(score=0.08, prevalence=0.11, alpha=0.10) + assert low_prevalence.discrete_set == "{0}" + assert low_prevalence.coverage == pytest.approx(0.91) + assert high_prevalence.discrete_set == "{0,1}" + assert high_prevalence.coverage == pytest.approx(1.0) + + +def test_exact_budget_point_lp_and_c2_dominance() -> None: + frame = pd.DataFrame( + { + "loan_amnt": [60.0, 50.0, 40.0, 30.0], + "purpose": ["a", "a", "b", "b"], + } + ) + point = np.array([0.03, 0.07, 0.10, 0.14]) + rate = np.array([0.10, 0.14, 0.18, 0.22]) + objective = expected_objective_coefficients(point, rate, lgd=0.45) + guard_exposure = np.array([60.0, 0.0, 40.0, 0.0]) + cap = c2_cap(guard_exposure, point) + solution = solve_point_portfolio( + frame, + point_score=point, + objective_rate=objective, + budget=100.0, + risk_cap=cap, + purpose_cap=0.6, + ) + assert solution.total_allocated == pytest.approx(100.0) + assert solution.weighted_point_score <= cap + 1e-10 + diagnostics = verify_c2_dominance( + guardrail_exposure=guard_exposure, + point_solution=solution, + point_score=point, + objective_rate=objective, + ) + assert diagnostics["point_minus_guardrail_objective"] >= -1e-5 + assert solution.basis_cap_lower <= cap <= solution.basis_cap_upper + + breakpoints = enumerate_basis_breakpoints( + frame, + point_score=point, + objective_rate=objective, + budget=100.0, + purpose_cap=0.6, + lower_cap=0.06, + upper_cap=0.13, + ) + assert breakpoints[0] == pytest.approx(0.06) + assert breakpoints[-1] == pytest.approx(0.13) + assert tuple(sorted(breakpoints)) == breakpoints + + +def test_exact_budget_formulation_matches_legacy_inequalities() -> None: + frame = pd.DataFrame( + { + "loan_amnt": [55.0, 45.0, 35.0, 25.0, 20.0], + "purpose": ["a", "b", "a", "b", "c"], + } + ) + point = np.array([0.025, 0.052, 0.081, 0.113, 0.147]) + rate = np.array([0.09, 0.13, 0.17, 0.20, 0.24]) + objective = expected_objective_coefficients(point, rate, lgd=0.45) + cap = 0.075 + exact = solve_point_portfolio( + frame, + point_score=point, + objective_rate=objective, + budget=100.0, + risk_cap=cap, + purpose_cap=0.55, + ) + legacy = solve_portfolio_highspy_native( + loans=frame, + pd_point=point, + pd_low=point, + pd_high=point, + lgd=np.full(len(frame), 0.45), + int_rates=rate, + total_budget=100.0, + max_concentration=0.55, + max_portfolio_pd=cap, + robust=False, + min_budget_utilization=1.0, + objective_rate_override=objective, + threads=1, + ) + assert exact.objective_value == pytest.approx(legacy["objective_value"], abs=1e-8) + np.testing.assert_allclose( + exact.allocation_fraction, + np.asarray(legacy["allocation_vector"], dtype=float), + rtol=0.0, + atol=1e-8, + ) + + session = PointPortfolioSession( + frame, + point_score=point, + objective_rate=objective, + budget=100.0, + purpose_cap=0.55, + ) + first = session.solve(cap) + session.solve(0.10) + repeated = session.solve(cap) + assert repeated.objective_value == pytest.approx(first.objective_value, abs=1e-8) + np.testing.assert_allclose(repeated.allocation_fraction, first.allocation_fraction, atol=1e-8) + + +def test_factorial_simulation_smoke() -> None: + config = load_v4_config( + ROOT / "configs/experiments/ijds_binary_geometry_frontier_v4_2026-07-12.yaml" + ) + config["simulation"].update( + repetitions=3, + sample_size=500, + score_shift_grid=[0.0], + prevalence_shift_grid=[0.0], + taxonomy_groups_grid=[1], + censoring_rate_grid=[0.0], + ) + repetitions, summary = run_factorial_simulation(config) + assert len(repetitions) == 3 + assert len(summary) == 1 + assert np.isfinite(repetitions["point_minus_guardrail_objective"]).all() + + +def test_shared_frontier_replaces_placeholder_endpoints() -> None: + scores = pd.DataFrame( + { + "id": pd.Series(["a", "b", "c", "d"], dtype="string"), + "design_split": ["primary_oot"] * 4, + "pd_catboost_platt": [0.1, 0.2, 0.3, 0.4], + } + ) + recipe = fit_binary_outcome_recipe( + np.array([0.1, 0.2, 0.3, 0.4]), + np.array([0, 0, 1, 1]), + alpha=0.1, + n_groups=1, + bin_edges=(0.0, 1.0), + ) + shared = pd.DataFrame( + { + "id": pd.Series(["a", "c"], dtype="string"), + "conformal_lower": [np.nan, np.nan], + "conformal_upper": [np.nan, np.nan], + "conformal_group": [np.nan, np.nan], + } + ) + expanded = expand_frontier_for_window(shared, scores, recipe, window_id="window") + assert {"conformal_lower", "conformal_upper", "conformal_group"}.issubset(expanded) + assert not any(column.endswith(("_x", "_y")) for column in expanded.columns) + assert expanded["conformal_lower"].notna().all() diff --git a/tests/test_ijds_normalized_objective_frontier.py b/tests/test_ijds_normalized_objective_frontier.py new file mode 100644 index 0000000..7a45f24 --- /dev/null +++ b/tests/test_ijds_normalized_objective_frontier.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +import copy +import itertools +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from scripts.experiments.run_ijds_normalized_objective_frontier import prepare_output_paths +from src.ijds_audit.portfolio import PointPortfolioSession, PointPortfolioSolution +from src.ijds_challengers.config import load_frontier_config +from src.ijds_challengers.frontier import ( + ObjectiveFloorPortfolioSession, + common_objective_target, + normalized_exposure_distance, + normalized_score_cap, + solve_glop_portfolio, +) +from src.ijds_challengers.normalized_frontier import ( + _solve_minimum_endpoint, + _solve_objective_optimum, +) + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "configs/experiments/ijds_normalized_objective_frontier_2026-07-13_v1c.yaml" + + +def _point_solution() -> PointPortfolioSolution: + return PointPortfolioSolution( + allocation_fraction=np.array([1.0]), + exposure=np.array([1.0]), + objective_value=0.1, + weighted_point_score=0.2, + total_allocated=1.0, + basis_cap_lower=0.0, + basis_cap_upper=1.0, + simplex_iterations=1, + ) + + +class _BoundarySession: + def __init__(self, errors: list[RuntimeError | None]) -> None: + self.errors = errors + self.calls: list[float] = [] + + def solve(self, risk_cap: float) -> PointPortfolioSolution: + self.calls.append(float(risk_cap)) + index = len(self.calls) - 1 + error = self.errors[index] if index < len(self.errors) else None + if error is not None: + raise error + return _point_solution() + + +@pytest.mark.parametrize( + "message", + [ + "Point LP is not optimal: Infeasible.", + "Point LP did not fill its budget: 500000.001.", + ], +) +def test_minimum_endpoint_retries_only_known_boundary_failures(message: str) -> None: + session = _BoundarySession([RuntimeError(message)]) + + solution, slack = _solve_minimum_endpoint( + session, + minimum_score=0.2, + retry_slacks=[1.0e-12], + ) + + assert solution.objective_value == 0.1 + assert slack == 1.0e-12 + assert session.calls == [0.2, 0.2 + 1.0e-12] + + +def test_minimum_endpoint_does_not_mask_unrecognized_failure() -> None: + session = _BoundarySession([RuntimeError("HiGHS failed unexpectedly.")]) + + with pytest.raises(RuntimeError, match="unexpectedly"): + _solve_minimum_endpoint(session, minimum_score=0.2, retry_slacks=[1.0e-12, 1.0e-10]) + + assert session.calls == [0.2] + + +def test_minimum_endpoint_uses_closed_retry_ladder_for_unknown_status() -> None: + unknown = RuntimeError("Point LP is not optimal: Unknown.") + session = _BoundarySession([unknown, unknown]) + + solution, slack = _solve_minimum_endpoint( + session, + minimum_score=0.2, + retry_slacks=[1.0e-12, 1.0e-10], + ) + + assert solution.objective_value == 0.1 + assert slack == 1.0e-10 + assert session.calls == [0.2, 0.2 + 1.0e-12, 0.2 + 1.0e-10] + + +def _small_menu() -> tuple[pd.DataFrame, np.ndarray, np.ndarray]: + frame = pd.DataFrame( + { + "id": ["a", "b", "c", "d"], + "loan_amnt": [1.0, 1.0, 1.0, 1.0], + "purpose": ["a", "a", "b", "b"], + } + ) + score = np.array([0.03, 0.08, 0.12, 0.19]) + objective = np.array([0.04, 0.11, 0.07, 0.15]) + return frame, score, objective + + +def _enumerated_optimum( + *, + score: np.ndarray, + objective: np.ndarray, + mode: str, + threshold: float, +) -> tuple[float, float]: + equality = np.ones((1, 4)) + equality_rhs = np.array([2.0]) + inequalities: list[np.ndarray] = [] + rhs: list[float] = [] + for index in range(4): + upper = np.zeros(4) + upper[index] = 1.0 + inequalities.append(upper) + rhs.append(1.0) + lower = np.zeros(4) + lower[index] = -1.0 + inequalities.append(lower) + rhs.append(0.0) + inequalities.extend([np.array([1.0, 1.0, 0.0, 0.0]), np.array([0.0, 0.0, 1.0, 1.0])]) + rhs.extend([1.5, 1.5]) + if mode == "normalized_score": + inequalities.append(score.copy()) + rhs.append(float(threshold) * 2.0) + else: + inequalities.append(-objective.copy()) + rhs.append(-float(threshold)) + matrix = np.vstack(inequalities) + bound = np.asarray(rhs) + feasible: list[np.ndarray] = [] + for active in itertools.combinations(range(len(matrix)), 3): + system = np.vstack([equality, matrix[list(active)]]) + if np.linalg.matrix_rank(system) < 4: + continue + candidate = np.linalg.solve(system, np.concatenate([equality_rhs, bound[list(active)]])) + if bool(np.all(matrix @ candidate <= bound + 1e-9)): + feasible.append(candidate) + assert feasible + if mode == "normalized_score": + optimum = max(feasible, key=lambda value: float(value @ objective)) + else: + optimum = min(feasible, key=lambda value: float(value @ score)) + return float(optimum @ objective), float(optimum @ score / 2.0) + + +def test_frontier_config_is_locked_and_outcome_free() -> None: + config = load_frontier_config(CONFIG) + assert config["frontier"]["gamma_grid"] == [0.0, 0.25, 0.5, 0.75, 1.0] + assert config["frontier"]["coordinate_grid"] == [0.25, 0.5, 0.75] + assert config["claim_boundary"]["outcome_columns_passed"] == [] + assert config["frontier"]["rulers"]["primary"] == "objective_matched" + assert config["frontier"]["objective_optimum"]["diagnostic"] == ( + "nonbasic_reduced_costs_plus_reversed_id_order" + ) + assert config["solver"]["budget_residual_tolerance_dollars"] == 1.0e-4 + + +def test_frontier_config_rejects_policy_winner(tmp_path: Path) -> None: + text = CONFIG.read_text(encoding="utf-8").replace( + "no_policy_winner: true", + "no_policy_winner: false", + ) + path = tmp_path / "invalid.yaml" + path.write_text(text, encoding="utf-8") + with pytest.raises(ValueError, match="claim boundary"): + load_frontier_config(path) + + +def test_frontier_output_paths_are_contained_and_immutable(tmp_path: Path) -> None: + config = copy.deepcopy(load_frontier_config(CONFIG)) + config["run_tag"] = "normalized-frontier-test" + paths = prepare_output_paths(config, repo_root=tmp_path) + assert paths.data_dir == ( + tmp_path / "data/processed/experiments/ijds_audit/normalized-frontier-test" + ) + assert paths.model_dir == (tmp_path / "models/experiments/ijds_audit/normalized-frontier-test") + with pytest.raises(FileExistsError, match="already exists"): + prepare_output_paths(config, repo_root=tmp_path) + + +def test_normalized_cap_is_positive_affine_invariant() -> None: + frame, score, objective = _small_menu() + budget = 2.0 + base_min = ObjectiveFloorPortfolioSession( + frame, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve() + base_objective = PointPortfolioSession( + frame, + point_score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(1.0) + transformed = 0.5 * score + 0.2 + transformed_min = ObjectiveFloorPortfolioSession( + frame, + score=transformed, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve() + coordinate = 0.5 + base_cap = normalized_score_cap( + minimum_score=base_min.weighted_score, + score_at_objective=float(base_objective.exposure @ score / budget), + coordinate=coordinate, + ) + transformed_cap = normalized_score_cap( + minimum_score=transformed_min.weighted_score, + score_at_objective=float(base_objective.exposure @ transformed / budget), + coordinate=coordinate, + ) + assert transformed_cap == pytest.approx(0.5 * base_cap + 0.2) + base_solution = PointPortfolioSession( + frame, + point_score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(base_cap) + transformed_solution = PointPortfolioSession( + frame, + point_score=transformed, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(transformed_cap) + assert ( + normalized_exposure_distance( + base_solution.exposure, + transformed_solution.exposure, + budget=budget, + ) + < 1e-10 + ) + + +def test_both_rulers_match_vertex_enumeration_and_glop() -> None: + frame, score, objective = _small_menu() + budget = 2.0 + minimum = ObjectiveFloorPortfolioSession( + frame, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve() + unconstrained = PointPortfolioSession( + frame, + point_score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(1.0) + cap = normalized_score_cap( + minimum_score=minimum.weighted_score, + score_at_objective=float(unconstrained.exposure @ score / budget), + coordinate=0.5, + ) + normalized = PointPortfolioSession( + frame, + point_score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(cap) + brute_objective, brute_score = _enumerated_optimum( + score=score, + objective=objective, + mode="normalized_score", + threshold=cap, + ) + glop = solve_glop_portfolio( + frame, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + mode="normalized_score", + threshold=cap, + ) + assert normalized.objective_value == pytest.approx(brute_objective, abs=1e-9) + assert normalized.weighted_point_score == pytest.approx(brute_score, abs=1e-9) + assert glop.objective_value == pytest.approx(brute_objective, abs=1e-9) + assert glop.weighted_score == pytest.approx(brute_score, abs=1e-9) + + lower, target = common_objective_target( + minimum_objectives=[minimum.objective_value, minimum.objective_value - 0.01], + objective_optimum=unconstrained.objective_value, + coordinate=0.5, + ) + assert lower == pytest.approx(minimum.objective_value) + matched = ObjectiveFloorPortfolioSession( + frame, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + ).solve(target) + brute_objective, brute_score = _enumerated_optimum( + score=score, + objective=objective, + mode="objective_matched", + threshold=target, + ) + glop = solve_glop_portfolio( + frame, + score=score, + objective_rate=objective, + budget=budget, + purpose_cap=0.75, + mode="objective_matched", + threshold=target, + ) + assert matched.objective_value == pytest.approx(brute_objective, abs=1e-9) + assert matched.weighted_score == pytest.approx(brute_score, abs=1e-9) + assert glop.objective_value == pytest.approx(brute_objective, abs=1e-9) + assert glop.weighted_score == pytest.approx(brute_score, abs=1e-9) + + +def test_objective_optimum_uses_basis_and_order_diagnostics() -> None: + config = load_frontier_config(CONFIG) + frame, score, objective = _small_menu() + optimum = _solve_objective_optimum( + frame, + point_score=score, + objective_rate=objective, + budget=2.0, + purpose_cap=0.75, + time_limit=30, + threads=1, + role="primary_oot", + period="2016-04", + optimum_config=config["frontier"]["objective_optimum"], + solver_config=config["solver"], + ) + assert optimum.diagnostics["near_zero_nonbasic_reduced_costs"] == 0 + assert optimum.diagnostics["reversed_id_exposure_distance"] < 1e-10 + + +def test_objective_optimum_rejects_alternate_optimum() -> None: + config = load_frontier_config(CONFIG) + frame, score, objective = _small_menu() + objective[:] = 0.1 + with pytest.raises(RuntimeError, match="near-zero nonbasic reduced cost"): + _solve_objective_optimum( + frame, + point_score=score, + objective_rate=objective, + budget=2.0, + purpose_cap=0.75, + time_limit=30, + threads=1, + role="primary_oot", + period="2016-04", + optimum_config=config["frontier"]["objective_optimum"], + solver_config=config["solver"], + ) diff --git a/tests/test_ijds_normalized_objective_frontier_v2.py b/tests/test_ijds_normalized_objective_frontier_v2.py new file mode 100644 index 0000000..ca303bc --- /dev/null +++ b/tests/test_ijds_normalized_objective_frontier_v2.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import copy +from pathlib import Path + +import pandas as pd +import pytest +import yaml + +from scripts.experiments.run_ijds_normalized_objective_frontier_v2 import ( + preflight_output_paths, + prepare_output_paths, +) +from src.ijds_challengers.evaluation import ( + build_endpoint_contrasts, + direction_from_bounds, + validate_outcome_alignment, +) +from src.ijds_challengers.evaluation_config import EXPECTED_FREEZE_SHA256, load_v2_config + +ROOT = Path(__file__).resolve().parents[1] +V3_CONFIG = ROOT / "configs/experiments/ijds_normalized_objective_frontier_2026-07-14_v3.yaml" +ACTIVE_CONFIG = ROOT / "configs/experiments/ijds_normalized_objective_frontier_2026-07-15_v5.yaml" + + +def test_active_config_locks_freeze_grid_and_no_selection() -> None: + config = load_v2_config(ACTIVE_CONFIG) + assert config["source_frontier"]["freeze"]["sha256"] == EXPECTED_FREEZE_SHA256 + assert config["evaluation"]["expected_window_contrasts"] == 48 + assert config["evaluation"]["expected_monthly_contrasts"] == 720 + assert config["evaluation"]["expected_metric_directions"] == 144 + assert all(bool(value) for value in config["claim_boundary"].values()) + + +def test_v3_reuses_v1c_freeze_with_reconstructed_endpoint() -> None: + config = load_v2_config(V3_CONFIG) + assert config["source_frontier"]["freeze"]["sha256"] == EXPECTED_FREEZE_SHA256 + assert config["parent"]["config"].endswith("2026-07-14_v3.yaml") + assert config["outcomes"]["endpoint_contract"].startswith( + "conservative_terminal_status_reconstruction" + ) + + +def test_v5_changes_only_endpoint_reason_taxonomy_and_recovery_implementation() -> None: + config = load_v2_config(ACTIVE_CONFIG) + assert config["parent"]["config"].endswith("2026-07-15_v5.yaml") + recovery = config["endpoint_reason_recovery"] + assert recovery["require_exact_non_float_reference_equivalence"] is True + assert recovery["float_atol"] == 5.0e-14 + assert recovery["float_rtol"] == 5.0e-14 + assert config["evaluation"] == load_v2_config(V3_CONFIG)["evaluation"] + + +def test_active_config_rejects_coordinate_selection(tmp_path: Path) -> None: + config = load_v2_config(ACTIVE_CONFIG) + config["claim_boundary"]["no_coordinate_selection"] = False + path = tmp_path / "invalid.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + with pytest.raises(ValueError, match="claim boundary"): + load_v2_config(path) + + +def test_active_output_paths_are_contained_and_immutable(tmp_path: Path) -> None: + config = copy.deepcopy(load_v2_config(ACTIVE_CONFIG)) + config["run_tag"] = "frontier-v2-test" + paths = prepare_output_paths(config, repo_root=tmp_path) + assert paths.data_dir == tmp_path / "data/processed/experiments/ijds_audit/frontier-v2-test" + assert paths.model_dir == tmp_path / "models/experiments/ijds_audit/frontier-v2-test" + with pytest.raises(FileExistsError, match="already exists"): + prepare_output_paths(config, repo_root=tmp_path) + + +def test_active_output_preflight_does_not_create_directories(tmp_path: Path) -> None: + config = copy.deepcopy(load_v2_config(ACTIVE_CONFIG)) + config["run_tag"] = "frontier-v2-preflight-test" + paths = preflight_output_paths(config, repo_root=tmp_path) + assert not paths.data_dir.exists() + assert not paths.model_dir.exists() + paths.model_dir.mkdir(parents=True) + with pytest.raises(FileExistsError, match="already exists"): + preflight_output_paths(config, repo_root=tmp_path) + + +@pytest.mark.parametrize( + ("lower", "upper", "tolerance", "expected"), + [ + (0.2, 0.3, 1e-6, "gamma_1_higher"), + (-0.3, -0.2, 1e-6, "gamma_1_lower"), + (-0.2, 0.3, 1e-6, "crosses_zero"), + (-1e-8, 1e-8, 1e-6, "exact_zero"), + ], +) +def test_direction_classification( + lower: float, + upper: float, + tolerance: float, + expected: str, +) -> None: + assert direction_from_bounds(lower, upper, tolerance=tolerance) == expected + + +def test_outcome_alignment_requires_exact_role_and_period() -> None: + config = copy.deepcopy(load_v2_config(ACTIVE_CONFIG)) + config["evaluation"]["evaluated_roles"] = ["primary_oot"] + config["evaluation"]["expected_candidate_counts"] = {"primary_oot": 2} + allocations = pd.DataFrame( + { + "id": ["a", "b"], + "role": ["primary_oot", "primary_oot"], + "period": ["2016-04", "2016-04"], + } + ) + outcomes = pd.DataFrame( + { + "id": ["a", "b"], + "role": ["primary_oot", "primary_oot"], + "period": ["2016-04", "2016-05"], + "snapshot_default": pd.Series([0, 1], dtype="Int8"), + "snapshot_resolution": ["fully_paid", "charged_off"], + } + ) + with pytest.raises(RuntimeError, match="role/period"): + validate_outcome_alignment(allocations, outcomes, config=config) + + +def test_endpoint_contrast_uses_common_unresolved_union() -> None: + config = copy.deepcopy(load_v2_config(ACTIVE_CONFIG)) + config["evaluation"].update( + rulers=["normalized_score"], + coordinates=[0.5], + expected_primary_months=2, + ) + rows: list[dict[str, object]] = [] + for period, outcome in (("2016-04", 0.0), ("2016-05", float("nan"))): + for label, exposures in ( + ("normalized_score_g100_c050", (70.0, 30.0)), + ("normalized_score_g000_c050", (30.0, 70.0)), + ): + for loan_id, exposure, rate in zip( + (f"{period}-a", f"{period}-b"), + exposures, + (0.1, 0.2), + strict=True, + ): + rows.append( + { + "window_id": "w01", + "role": "primary_oot", + "period": period, + "policy_label": label, + "id": loan_id, + "exposure": exposure, + "contractual_rate": rate, + "conformal_lower": 0.0, + "conformal_upper": 0.8, + "snapshot_default": outcome, + "expected_payoff_contribution": exposure * 0.05, + } + ) + joined = pd.DataFrame(rows) + structure = pd.DataFrame( + { + "window_id": ["w01", "w01"], + "role": ["primary_oot", "primary_oot"], + "period": ["2016-04", "2016-05"], + "ruler": ["normalized_score", "normalized_score"], + "coordinate": [0.5, 0.5], + "normalized_exposure_distance": [0.4, 0.4], + } + ) + window, monthly = build_endpoint_contrasts( + joined, + structure, + config=config, + lgd=1.0, + ) + assert len(window) == 1 + assert len(monthly) == 2 + assert window.loc[0, "unresolved_union_loans"] == 2 + assert ( + window.loc[0, "realized_payoff_difference_lower"] + <= window.loc[0, "realized_payoff_difference_upper"] + ) + + +def test_endpoint_contrast_rejects_inconsistent_loan_attributes() -> None: + config = copy.deepcopy(load_v2_config(ACTIVE_CONFIG)) + config["evaluation"].update( + rulers=["normalized_score"], + coordinates=[0.5], + expected_primary_months=1, + ) + joined = pd.DataFrame( + { + "window_id": ["w01", "w01"], + "id": ["loan-a", "loan-a"], + "role": ["primary_oot", "primary_oot"], + "period": ["2016-04", "2016-04"], + "policy_label": [ + "normalized_score_g100_c050", + "normalized_score_g000_c050", + ], + "exposure": [60.0, 40.0], + "contractual_rate": [0.10, 0.11], + "conformal_lower": [0.0, 0.0], + "conformal_upper": [0.8, 0.8], + "snapshot_default": [0.0, 0.0], + "expected_payoff_contribution": [3.0, 2.0], + } + ) + structure = pd.DataFrame( + { + "window_id": ["w01"], + "role": ["primary_oot"], + "period": ["2016-04"], + "ruler": ["normalized_score"], + "coordinate": [0.5], + "normalized_exposure_distance": [0.2], + } + ) + with pytest.raises(RuntimeError, match="attributes disagree"): + build_endpoint_contrasts( + joined, + structure, + config=config, + lgd=1.0, + ) diff --git a/tests/test_ijds_policy_support_tie_audit.py b/tests/test_ijds_policy_support_tie_audit.py new file mode 100644 index 0000000..70a8b78 --- /dev/null +++ b/tests/test_ijds_policy_support_tie_audit.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import copy +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from scripts.experiments.run_ijds_policy_support_tie_audit import ( + DEFAULT_CONFIG_PATH, + load_config, + prepare_output_paths, +) +from src.ijds_audit.policy_support import ( + build_cap_census, + classify_cap, + point_basis_diagnostics, +) +from src.ijds_audit.portfolio import PointPortfolioSession + +ROOT = Path(__file__).resolve().parents[1] +PARENT_PORTFOLIO = ( + ROOT + / "data/processed/experiments/ijds_audit" + / "ijds-binary-geometry-frontier-v4-2026-07-12-v1/portfolio" +) + + +def test_policy_support_config_is_locked_and_outcome_free() -> None: + config = load_config(DEFAULT_CONFIG_PATH) + assert config["protocol_status"] == "locked_outcome_free_structural_audit_before_execution" + assert config["family_audit"]["gamma_grid"] == [0.0, 0.25, 0.5, 0.75, 1.0] + assert config["claim_boundary"]["outcome_columns_passed"] == [] + allowed = config["source_ingest"]["allowed_raw_columns"] + assert allowed == ["id", "loan_amnt", "int_rate", "purpose"] + + +def test_config_rejects_policy_promotion(tmp_path: Path) -> None: + text = DEFAULT_CONFIG_PATH.read_text(encoding="utf-8").replace( + "no_policy_promotion: true", "no_policy_promotion: false" + ) + path = tmp_path / "broken.yaml" + path.write_text(text, encoding="utf-8") + with pytest.raises(ValueError, match="promotion must remain forbidden"): + load_config(path) + + +@pytest.mark.parametrize( + ("cap", "expected"), + [ + (0.04, "infeasible"), + (0.05, "minimum_boundary"), + (0.07, "decision_active"), + (0.10, "objective_boundary"), + (0.12, "objective_slack"), + ], +) +def test_cap_classification_has_explicit_boundaries(cap: float, expected: str) -> None: + assert ( + classify_cap( + cap, + minimum_feasible_score=0.05, + unconstrained_objective_score=0.10, + tolerance=1e-10, + ) + == expected + ) + + +def test_cap_classification_rejects_reversed_domain() -> None: + with pytest.raises(ValueError, match="below the feasible minimum"): + classify_cap( + 0.1, + minimum_feasible_score=0.2, + unconstrained_objective_score=0.1, + tolerance=1e-10, + ) + + +def test_parent_cap_census_is_complete_and_tolerance_deduplicated() -> None: + records = pd.read_parquet(PARENT_PORTFOLIO / "outcome_free_solve_records.parquet") + support = pd.read_parquet(PARENT_PORTFOLIO / "development_comparator_support.parquet") + frontier = pd.read_parquet(PARENT_PORTFOLIO / "exact_frontier_breakpoints.parquet") + periods = sorted(records.loc[records["role"].eq("primary_oot"), "period"].unique()) + census = build_cap_census( + records, + support, + frontier, + periods=periods, + broad_support=(0.05, 0.12), + tolerance=1e-10, + ) + assert len(census) == 7_297 + assert census["period"].nunique() == 15 + assert int(census["is_named_c0"].sum()) == 45 + assert int(census["is_named_c1"].sum()) == 1_080 + assert int(census["is_named_c2"].sum()) == 1_079 + assert int(census["is_period_basis_breakpoint"].sum()) == 2_952 + assert (census["cluster_cap_max"] - census["cluster_cap_min"]).max() <= 1e-10 + + +def test_basis_diagnostics_reconcile_a_small_point_lp() -> None: + frame = pd.DataFrame( + { + "loan_amnt": [60.0, 50.0, 40.0, 30.0, 20.0], + "purpose": ["a", "a", "b", "b", "c"], + } + ) + point = np.array([0.03, 0.06, 0.09, 0.12, 0.15]) + objective = np.array([0.031, 0.081, 0.044, 0.103, 0.017]) + session = PointPortfolioSession( + frame, + point_score=point, + objective_rate=objective, + budget=100.0, + purpose_cap=0.6, + threads=1, + ) + solution = session.solve(0.09) + diagnostics = point_basis_diagnostics( + session, + solution, + dual_tolerance=1e-7, + primal_tolerance=1e-9, + ) + assert diagnostics["basis_valid"] is True + assert diagnostics["minimum_absolute_nonbasic_reduced_cost"] > 1e-7 + assert diagnostics["near_zero_nonbasic_reduced_costs"] == 0 + assert abs(diagnostics["objective_reconciliation_error"]) < 1e-12 + assert diagnostics["maximum_dual_sign_violation"] <= 1e-12 + + +def test_output_paths_are_contained_and_no_overwrite(tmp_path: Path) -> None: + config = copy.deepcopy(load_config(DEFAULT_CONFIG_PATH)) + config["run_tag"] = "policy-support-test" + paths = prepare_output_paths(config, repo_root=tmp_path) + assert paths.data_dir == ( + tmp_path / "data/processed/experiments/ijds_audit/policy-support-test" + ) + assert paths.model_dir == (tmp_path / "models/experiments/ijds_audit/policy-support-test") + with pytest.raises(FileExistsError, match="already exists"): + prepare_output_paths(config, repo_root=tmp_path) diff --git a/tests/test_ijds_policy_support_tie_evidence.py b/tests/test_ijds_policy_support_tie_evidence.py new file mode 100644 index 0000000..43eabcf --- /dev/null +++ b/tests/test_ijds_policy_support_tie_evidence.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pandas as pd +import pytest + +from scripts.build_ijds_policy_support_tie_evidence import EVIDENCE_PATH, MEMO_PATH, build +from src.utils.isolated_experiment import sha256_file + +ROOT = Path(__file__).resolve().parents[1] + + +def _evidence() -> dict[str, Any]: + payload = json.loads(EVIDENCE_PATH.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Policy-support evidence must be a JSON object.") + return payload + + +def test_policy_support_evidence_build_is_byte_idempotent() -> None: + build() + first = sha256_file(EVIDENCE_PATH), sha256_file(MEMO_PATH) + build() + assert (sha256_file(EVIDENCE_PATH), sha256_file(MEMO_PATH)) == first + + +def test_policy_support_evidence_is_outcome_free_and_not_promoted() -> None: + evidence = _evidence() + assert evidence["outcome_columns_passed"] == [] + assert evidence["active_claim_status"] == "not_active_until_family_redesign_decision" + assert evidence["protected_stages_run"] == [] + assert evidence["protected_artifacts_written"] == [] + + +def test_policy_family_endpoint_and_reconstruction_results() -> None: + family = _evidence()["results"]["family"] + assert family["rows"] == 3_120 + assert family["inherited_rows"] == 1_872 + assert family["inherited_infeasible"] == 0 + assert family["inherited_decision_active"] == 1_846 + assert family["inherited_slack"] == 26 + assert family["gamma_zero_objective_slack"] == 624 + assert family["gamma_one_decision_active"] == 624 + assert family["gamma_one_minus_075_objective_max"] < -500.0 + assert family["maximum_absolute_parent_score_difference"] < 3e-16 + assert family["maximum_absolute_parent_objective_difference"] < 1e-9 + + +def test_tie_census_corrects_the_exploratory_counts() -> None: + results = _evidence()["results"] + point = results["point_cap_census"] + order = results["order_sensitivity"] + assert point["rows"] == 7_297 + assert point["named_unique_cap_months"] == 2_204 + assert point["near_zero_bases"] == 0 + assert point["primal_degenerate_bases"] == 2_941 + assert point["minimum_absolute_nonbasic_reduced_cost"] == pytest.approx(0.00038757301081204787) + assert order["triggered_rows"] == 2_941 + assert order["tie_sensitive_rows"] == 0 + assert order["maximum_allocation_distance"] < 2e-14 + + +def test_policy_support_tables_keep_the_declared_domains() -> None: + tables = _evidence()["tables"] + family = pd.read_csv(ROOT / tables["family"]["path"]) + endpoint = pd.read_csv(ROOT / tables["gamma_endpoint"]["path"]) + support = pd.read_csv(ROOT / tables["comparator_support"]["path"]) + assert int(family["cells"].sum()) == 3_120 + assert int(endpoint["cells"].sum()) == 624 + assert endpoint["gamma_100_decision_active"].eq(endpoint["cells"]).all() + c0 = support.loc[support["source"].eq("named_c0")].iloc[0] + assert int(c0["cap_month_rows"]) == 45 + assert int(c0["objective_slack"]) == 45 diff --git a/tests/test_ijds_rolling_origin_protocol.py b/tests/test_ijds_rolling_origin_protocol.py new file mode 100644 index 0000000..2844980 --- /dev/null +++ b/tests/test_ijds_rolling_origin_protocol.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from src.ijds_audit.allocations import declared_menu_counts +from src.ijds_audit.config import load_v4_config + +ROOT = Path(__file__).resolve().parents[1] +CONFIGS = ROOT / "configs" / "experiments" + + +@pytest.mark.parametrize( + ("origin", "cutoff", "development_end", "calibration_start", "primary_start"), + [ + (2015, "2015-03-31", "2009-12-31", "2010-01-01", "2015-04"), + (2017, "2017-03-31", "2011-12-31", "2012-01-01", "2017-04"), + ], +) +def test_rolling_origin_configs_shift_the_complete_design( + origin: int, + cutoff: str, + development_end: str, + calibration_start: str, + primary_start: str, +) -> None: + config = load_v4_config(CONFIGS / f"ijds_rolling_origin_{origin}_2026-07-12.yaml") + assert config["source"]["information_cutoff"] == cutoff + assert config["design"]["development_end"] == development_end + assert config["design"]["probability_calibration_start"] == calibration_start + assert config["design"]["primary_oot_start_month"] == primary_start + assert config["design"]["primary_oot_end_month"] == f"{origin}-06" + assert config["rolling_origin"]["origin_year"] == origin + assert config["rolling_origin"]["outcome_based_origin_selection"] is False + assert config.get("resume_outcome_free") is None + assert declared_menu_counts(config) == (11, 3) + + +def test_rolling_origin_v4_imports_2017_freeze_and_uses_corrected_endpoint() -> None: + config = load_v4_config(CONFIGS / "ijds_rolling_origin_2017_2026-07-15_v4.yaml") + assert config["resume_outcome_free"]["source_run_tag"].endswith("2026-07-12-v2") + assert config["target"]["evaluation_outcome_contract"]["cutoff"] == "2020-09-30" + assert config["design"]["primary_oot_start_month"] == "2017-04" + assert config["design"]["primary_oot_end_month"] == "2017-06" + assert declared_menu_counts(config) == (11, 3) + assert config["protocol_tag"].endswith("2026-07-15-v4") + + +def test_original_v4_retains_its_fifteen_month_primary_horizon() -> None: + config = load_v4_config(CONFIGS / "ijds_binary_geometry_frontier_v4_2026-07-12.yaml") + assert declared_menu_counts(config) == (11, 15) + + +def test_window_validation_is_relative_to_declared_origin(tmp_path: Path) -> None: + source = CONFIGS / "ijds_rolling_origin_2015_2026-07-12.yaml" + payload = source.read_text(encoding="utf-8") + broken = payload.replace('start: "2011-02-01"', 'start: "2011-03-01"', 1) + path = tmp_path / "broken.yaml" + path.write_text( + broken.replace( + 'extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml"', + f'extends: "{(CONFIGS / "ijds_binary_geometry_frontier_v4_2026-07-12.yaml").as_posix()}"', + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="consecutive six-month window"): + load_v4_config(path) + + +def test_chronology_validator_rejects_a_post_cutoff_primary_gap(tmp_path: Path) -> None: + source = CONFIGS / "ijds_rolling_origin_2017_2026-07-12.yaml" + payload = source.read_text(encoding="utf-8") + broken = payload.replace( + 'primary_oot_start_month: "2017-04"', + 'primary_oot_start_month: "2017-05"', + ) + path = tmp_path / "broken.yaml" + path.write_text( + broken.replace( + 'extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml"', + f'extends: "{(CONFIGS / "ijds_binary_geometry_frontier_v4_2026-07-12.yaml").as_posix()}"', + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="information cutoff"): + load_v4_config(path) + + +def test_rolling_origin_validator_rejects_an_asymmetric_training_origin(tmp_path: Path) -> None: + source = CONFIGS / "ijds_rolling_origin_2015_2026-07-12.yaml" + payload = source.read_text(encoding="utf-8") + broken = payload.replace("origin_year: 2015", "origin_year: 2014") + path = tmp_path / "broken.yaml" + path.write_text( + broken.replace( + 'extends: "ijds_binary_geometry_frontier_v4_2026-07-12.yaml"', + f'extends: "{(CONFIGS / "ijds_binary_geometry_frontier_v4_2026-07-12.yaml").as_posix()}"', + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match=r"asymmetric source\.information_cutoff"): + load_v4_config(path) diff --git a/tests/test_ijds_v4_claim_sync.py b/tests/test_ijds_v4_claim_sync.py new file mode 100644 index 0000000..48c77a3 --- /dev/null +++ b/tests/test_ijds_v4_claim_sync.py @@ -0,0 +1,78 @@ +"""Structure and wording checks for the active V4 paper and its controls.""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +BODY = REPO / "paper/CRPTO_ijds.qmd" +SUPPLEMENT = REPO / "paper/supplement_ijds.qmd" +OFFICIAL = REPO / "paper/submission/CRPTO_ijds_submission.tex" + + +def test_body_and_generated_tex_share_architecture_citations_and_displays() -> None: + body = BODY.read_text(encoding="utf-8") + official = OFFICIAL.read_text(encoding="utf-8") + sections = ( + "Introduction", + "Related Work", + "Data and Locked Evaluation Design", + "Method", + "Audit Theory and Estimands", + "Results", + "Discussion", + "Limitations", + "Reproducibility", + "Conclusion", + ) + assert [body.index(f"# {section}") for section in sections] == sorted( + body.index(f"# {section}") for section in sections + ) + assert [official.index(rf"\section{{{section}}}") for section in sections] == sorted( + official.index(rf"\section{{{section}}}") for section in sections + ) + + body_citations = { + key + for key in re.findall(r"@([A-Za-z0-9_:-]+)", body) + if not key.startswith(("fig-", "tbl-", "eq-", "sec-")) + } + tex_citations: set[str] = set() + for group in re.findall(r"\\cite\w*\{([^}]+)\}", official): + tex_citations.update(key.strip() for key in group.split(",")) + assert body_citations == tex_citations + body_table_ids = set(re.findall(r"\{#(tbl-[A-Za-z0-9_-]+)\}", body)) + assert body_table_ids == {"tbl-protocol", "tbl-credit-controls", "tbl-two-ruler"} + assert official.count(r"\begin{longtable}") == len(body_table_ids) + assert body.count("{#fig-") == official.count(r"\begin{figure}") == 2 + + +def test_v4_wording_keeps_theory_and_empirical_scope_separate() -> None: + body = BODY.read_text(encoding="utf-8").lower() + supplement = SUPPLEMENT.read_text(encoding="utf-8").lower() + body_normalized = re.sub(r"\s+", " ", body) + supplement_normalized = re.sub(r"\s+", " ", supplement) + + for surface in (body_normalized, supplement_normalized): + assert "constant-score" in surface + assert re.search(r"varying scores|scores vary", surface) + assert "not a confidence interval" in surface + assert "not a deployable" in surface + assert "not independent replications" in surface + assert "not a prospective trial, preregistration, or causal estimate" in body_normalized + assert "not a causal identified set" in supplement + assert ( + "constant-score theorem identifies a mechanism rather than the varying-score empirical path" + ) in body_normalized + for surface in (body_normalized, supplement_normalized): + assert "simulation claim" not in surface + assert "no portfolio claim uses this simulation" not in surface + + +def test_review_surfaces_do_not_expose_exact_v4_identifiers() -> None: + for path in (BODY, SUPPLEMENT, OFFICIAL): + text = path.read_text(encoding="utf-8") + assert "ijds-binary-geometry-frontier-v4-2026-07-12" not in text + assert "60cdf298d965525cddaaf03abccd15ff805e1a15" not in text + assert "c2b3dc2d18c9fed80708682d5a0369c80c89643e2d28024418522d954ebe667c" not in text diff --git a/tests/test_inspect_ijds_pdfs.py b/tests/test_inspect_ijds_pdfs.py new file mode 100644 index 0000000..a7d164f --- /dev/null +++ b/tests/test_inspect_ijds_pdfs.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from scripts.inspect_ijds_pdfs import ( + BODY_QMD, + _load_abstract, + find_reference_start_page, + is_letter_size, + word_count, +) + + +def test_reference_heading_detection_is_one_based_and_standalone() -> None: + texts = ["Introduction\nReferences to prior work", "Results", "References\nA. Author"] + + assert find_reference_start_page(texts) == 3 + + +def test_letter_size_accepts_both_orientations_only() -> None: + assert is_letter_size(612.0, 792.0) + assert is_letter_size(792.0, 612.0) + assert not is_letter_size(595.0, 842.0) + + +def test_active_abstract_satisfies_ijds_length_and_paragraph_contract() -> None: + abstract = _load_abstract(BODY_QMD) + + assert word_count(abstract) <= 300 + assert "\n\n" not in abstract diff --git a/tests/test_models/test_binary_conformal_guardrail.py b/tests/test_models/test_binary_conformal_guardrail.py new file mode 100644 index 0000000..71d5c4b --- /dev/null +++ b/tests/test_models/test_binary_conformal_guardrail.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pytest + +from src.models.binary_conformal_guardrail import ( + BinaryOutcomeConformalRecipe, + assign_conformal_groups, + fit_binary_outcome_recipe, +) + + +def test_fixed_edges_are_not_relearned_from_residual_sample() -> None: + fixed_edges = (0.0, 0.2, 0.5, 1.0) + + first = fit_binary_outcome_recipe( + np.array([0.05, 0.10, 0.30, 0.40, 0.60, 0.90]), + np.array([0, 1, 0, 1, 0, 1]), + alpha=0.5, + bin_edges=fixed_edges, + taxonomy_provenance="pd_calibration_2011", + taxonomy_method="upstream_score_quantiles", + ) + second = fit_binary_outcome_recipe( + np.array([0.15, 0.19, 0.21, 0.49, 0.51, 0.99]), + np.array([1, 0, 1, 0, 1, 0]), + alpha=0.5, + bin_edges=fixed_edges, + taxonomy_provenance="pd_calibration_2011", + taxonomy_method="upstream_score_quantiles", + ) + + assert first.bin_edges == second.bin_edges == fixed_edges + assert first.taxonomy_provenance == "pd_calibration_2011" + assert first.taxonomy_method == "upstream_score_quantiles" + assert first.method == "fixed_taxonomy_split_mondrian_absolute_residual" + assert first.group_counts == second.group_counts == (2, 2, 2) + assert first.residual_quantiles != second.residual_quantiles + + +def test_fixed_edge_recipe_uses_exact_finite_sample_rank() -> None: + recipe = fit_binary_outcome_recipe( + np.array([0.1, 0.2, 0.3, 0.4]), + np.array([0, 0, 1, 1]), + alpha=0.4, + bin_edges=(0.0, 1.0), + ) + + # ceil((4 + 1) * (1 - 0.4)) = 3; sorted residuals are 0.1, 0.2, 0.6, 0.7. + assert recipe.raw_finite_sample_ranks == (3,) + assert recipe.finite_sample_ranks == (3,) + assert recipe.residual_quantiles == pytest.approx((0.6,)) + + +def test_bin_edges_must_be_strictly_increasing() -> None: + with pytest.raises(ValueError, match="strictly increasing"): + assign_conformal_groups(np.array([0.2]), (0.0, 0.5, 0.5, 1.0)) + + +def test_old_recipe_payload_deserializes_with_provenance_defaults() -> None: + payload: dict[str, Any] = { + "alpha": 0.1, + "requested_groups": 1, + "bin_edges": (0.0, 1.0), + "residual_quantiles": (0.4,), + "group_counts": (10,), + "finite_sample_ranks": (10,), + "raw_finite_sample_ranks": (10,), + } + + recipe = BinaryOutcomeConformalRecipe(**payload) + + assert recipe.taxonomy_provenance == "unspecified_legacy_recipe" + assert recipe.taxonomy_method == "unspecified_legacy_taxonomy" diff --git a/tests/test_models/test_calibration.py b/tests/test_models/test_calibration.py deleted file mode 100644 index e940b9d..0000000 --- a/tests/test_models/test_calibration.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Tests for src/models/calibration.py. - -Covers ECE computation, isotonic/Platt calibrators, and evaluation metrics. -""" - -from __future__ import annotations - -import pickle - -import numpy as np -import pytest - -from src.models.calibration import ( - LogitShiftCalibrator, - QuadraticLogitCalibrator, - TemperatureScalingCalibrator, - calibrate_isotonic, - evaluate_calibration, - expected_calibration_error, -) - -# --------------------------------------------------------------------------- -# expected_calibration_error -# --------------------------------------------------------------------------- - - -class TestExpectedCalibrationError: - def test_perfect_calibration_returns_zero(self): - # Probabilities exactly match true rates per bin - y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) - y_prob = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]) - ece = expected_calibration_error(y_true, y_prob, n_bins=10) - assert ece == pytest.approx(0.0, abs=0.01) - - def test_worst_calibration_is_high(self): - # All predictions = 0 but all labels = 1 - y_true = np.ones(100) - y_prob = np.zeros(100) - ece = expected_calibration_error(y_true, y_prob, n_bins=10) - assert ece > 0.5 - - def test_ece_in_unit_interval(self): - rng = np.random.default_rng(42) - y_true = rng.integers(0, 2, 500) - y_prob = rng.uniform(0, 1, 500) - ece = expected_calibration_error(y_true, y_prob) - assert 0.0 <= ece <= 1.0 - - def test_empty_bins_handled(self): - # Only values in [0, 0.1) bin - y_true = np.array([0, 0, 1]) - y_prob = np.array([0.01, 0.02, 0.05]) - ece = expected_calibration_error(y_true, y_prob, n_bins=10) - assert np.isfinite(ece) - - -# --------------------------------------------------------------------------- -# calibrate_isotonic -# --------------------------------------------------------------------------- - - -class TestCalibrateIsotonic: - def test_returns_isotonic_model(self): - rng = np.random.default_rng(42) - y_cal = rng.integers(0, 2, 200).astype(float) - proba_cal = rng.uniform(0, 1, 200) - model = calibrate_isotonic(y_cal, proba_cal) - assert hasattr(model, "predict") - - def test_predictions_in_zero_one(self): - rng = np.random.default_rng(42) - y_cal = rng.integers(0, 2, 200).astype(float) - proba_cal = rng.uniform(0, 1, 200) - model = calibrate_isotonic(y_cal, proba_cal) - preds = model.predict(np.linspace(0, 1, 50)) - assert preds.min() >= 0.0 - assert preds.max() <= 1.0 - - def test_monotonic_output(self): - rng = np.random.default_rng(42) - y_cal = rng.integers(0, 2, 300).astype(float) - proba_cal = rng.uniform(0, 1, 300) - model = calibrate_isotonic(y_cal, proba_cal) - inputs = np.linspace(0, 1, 100) - preds = model.predict(inputs) - # Isotonic regression should be non-decreasing - assert np.all(np.diff(preds) >= -1e-10) - - -# --------------------------------------------------------------------------- -# evaluate_calibration -# --------------------------------------------------------------------------- - - -class TestEvaluateCalibration: - def test_returns_ece_and_brier(self): - rng = np.random.default_rng(42) - y_true = rng.integers(0, 2, 200) - y_prob = rng.uniform(0, 1, 200) - metrics = evaluate_calibration(y_true, y_prob, name="test_model") - assert "ece" in metrics - assert "brier_score" in metrics - - def test_perfect_predictions_low_brier(self): - y_true = np.array([0, 0, 1, 1]) - y_prob = np.array([0.0, 0.0, 1.0, 1.0]) - metrics = evaluate_calibration(y_true, y_prob) - assert metrics["brier_score"] == pytest.approx(0.0) - - def test_brier_in_valid_range(self): - rng = np.random.default_rng(42) - y_true = rng.integers(0, 2, 500) - y_prob = rng.uniform(0, 1, 500) - metrics = evaluate_calibration(y_true, y_prob) - assert 0.0 <= metrics["brier_score"] <= 1.0 - - -class TestLogitShiftCalibrator: - def test_predictions_are_clipped_and_monotonic(self): - calibrator = LogitShiftCalibrator(delta=0.75) - scores = np.linspace(0.001, 0.999, 50) - preds = calibrator.transform(scores) - assert np.all(preds >= 0.0) - assert np.all(preds <= 1.0) - assert np.all(np.diff(preds) >= 0.0) - - def test_pickle_round_trip_preserves_outputs(self): - calibrator = LogitShiftCalibrator(delta=-0.25) - scores = np.array([0.05, 0.2, 0.5, 0.8], dtype=float) - restored = pickle.loads(pickle.dumps(calibrator)) - np.testing.assert_allclose(calibrator.predict(scores), restored.predict(scores)) - - -class TestExperimentalScoreCalibrators: - def test_temperature_scaling_predicts_bounded_probabilities(self): - y_true = np.array([0, 0, 0, 1, 1, 1]) - raw = np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95]) - calibrator = TemperatureScalingCalibrator().fit(raw, y_true) - - preds = calibrator.predict(np.linspace(0.01, 0.99, 20)) - - assert calibrator.temperature > 0.0 - assert np.all(preds >= 0.0) - assert np.all(preds <= 1.0) - - def test_quadratic_logit_pickle_round_trip_preserves_outputs(self): - y_true = np.array([0, 0, 0, 1, 1, 1, 1, 0]) - raw = np.array([0.04, 0.12, 0.28, 0.48, 0.62, 0.78, 0.92, 0.22]) - calibrator = QuadraticLogitCalibrator().fit(raw, y_true) - scores = np.linspace(0.05, 0.95, 10) - - restored = pickle.loads(pickle.dumps(calibrator)) - - np.testing.assert_allclose(calibrator.predict(scores), restored.predict(scores)) diff --git a/tests/test_models/test_calibrator_pickle_compat.py b/tests/test_models/test_calibrator_pickle_compat.py deleted file mode 100644 index d7f9111..0000000 --- a/tests/test_models/test_calibrator_pickle_compat.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Pickle compatibility tests for the frozen champion calibrator. - -These tests are the safety net for any future refactor of -``src/models/conformal.py``. They verify that ``models/pd_canonical_calibrator.pkl`` -can be loaded *with the current code on disk* and that its public surface -(predict / predict_proba) still produces sensible probabilities. - -If a refactor moves :class:`src.models.conformal.ProbabilityRegressor` or -its sibling adapters to a submodule without preserving the original -``__module__`` path, these tests will fail loudly. That is the desired -behaviour — the failure forces the refactor to add explicit pickle -compatibility shims (see ``docs/refactor/CONFORMAL_REFACTOR_PLAN.md``). -""" - -from __future__ import annotations - -from pathlib import Path - -import joblib -import numpy as np -import pytest - -CALIBRATOR_PATH = Path("models/pd_canonical_calibrator.pkl") - - -@pytest.fixture(scope="module") -def calibrator(): - if not CALIBRATOR_PATH.is_file(): - pytest.skip(f"{CALIBRATOR_PATH} not available locally — run `dvc pull` to fetch.") - return joblib.load(CALIBRATOR_PATH) - - -def test_calibrator_loads(calibrator) -> None: - """Pickle deserialisation must not raise. This is the canary.""" - assert calibrator is not None - - -def test_calibrator_module_paths_are_stable(calibrator) -> None: - """The classes inside the calibrator pickle must still live under - ``src.models.conformal`` (or a wrapper). Any refactor that changes - ``__module__`` without compat shims breaks reproducibility.""" - - def _module_of(obj) -> str: - return type(obj).__module__ - - seen_modules: set[str] = set() - - def _collect(node): - seen_modules.add(_module_of(node)) - for attr in dir(node): - if attr.startswith("_") and attr not in {"_estimator", "_wrapped", "_calibrator"}: - continue - try: - value = getattr(node, attr, None) - except Exception: - continue - if value is None: - continue - type_module = getattr(type(value), "__module__", "") - if type_module.startswith(("src.", "crpto.")): - seen_modules.add(type_module) - - _collect(calibrator) - - forbidden = {m for m in seen_modules if "._legacy" in m or "._private" in m} - assert not forbidden, ( - f"Calibrator references private/legacy submodules — pickle compat at risk: {forbidden}" - ) - - -def test_calibrator_predict_returns_probabilities(calibrator) -> None: - """Whatever public surface the calibrator exposes (predict / predict_proba / - transform) must produce values in [0, 1].""" - rng = np.random.default_rng(0) - sample = rng.uniform(0.0, 1.0, size=64) - sample_2d = sample.reshape(-1, 1) - - output = None - for method in ("predict", "predict_proba", "transform"): - fn = getattr(calibrator, method, None) - if not callable(fn): - continue - for candidate in (sample, sample_2d): - try: - result = fn(candidate) - except Exception: - continue - output = np.asarray(result, dtype=float) - break - if output is not None: - break - - assert output is not None, ( - "Calibrator does not expose any of predict/predict_proba/transform — " - "this would silently break ScoreCalibrator consumers." - ) - finite = output[np.isfinite(output)] - assert finite.size > 0 - assert np.all((finite >= -1e-9) & (finite <= 1.0 + 1e-9)), ( - f"Calibrator output outside [0, 1]: min={finite.min()}, max={finite.max()}" - ) - - -def test_calibrator_is_deterministic(calibrator) -> None: - """Two consecutive predict calls on the same input must return the same - values bit-for-bit. Calibrators are deterministic by design.""" - rng = np.random.default_rng(1) - sample = rng.uniform(0.0, 1.0, size=32) - - for method in ("predict", "predict_proba"): - fn = getattr(calibrator, method, None) - if not callable(fn): - continue - try: - a = np.asarray(fn(sample)) - b = np.asarray(fn(sample)) - except Exception: - try: - a = np.asarray(fn(sample.reshape(-1, 1))) - b = np.asarray(fn(sample.reshape(-1, 1))) - except Exception: - continue - np.testing.assert_array_equal(a, b) - return # one deterministic method is enough - - pytest.skip("Calibrator has no callable predict/predict_proba.") diff --git a/tests/test_models/test_conformal.py b/tests/test_models/test_conformal.py deleted file mode 100644 index 8fc40d6..0000000 --- a/tests/test_models/test_conformal.py +++ /dev/null @@ -1,546 +0,0 @@ -"""Unit tests for conformal prediction utilities.""" - -import numpy as np -import pandas as pd -import pytest - -from src.models.conformal import ( - ProbabilityRegressor, - _conformal_quantile, - apply_probability_calibrator, - build_mondrian_partition_labels, - conditional_coverage_by_group, - create_classification_sets, - create_classification_sets_mondrian, - create_cross_conformal_score_intervals, - create_pd_intervals_mondrian, - create_pd_intervals_mondrian_from_predictions, - summarize_prediction_sets, - validate_coverage, -) - -# ── ProbabilityRegressor ── - - -class FakeClassifier: - """Minimal classifier stub for testing.""" - - def predict_proba(self, X): - n = X.shape[0] if hasattr(X, "shape") else len(X) - probs = np.random.RandomState(42).random(n) - return np.column_stack([1 - probs, probs]) - - -def test_probability_regressor_wraps_classifier(): - clf = FakeClassifier() - reg = ProbabilityRegressor(clf) - X = pd.DataFrame({"a": [1, 2, 3]}) - preds = reg.predict(X) - assert preds.shape == (3,) - assert np.all(preds >= 0) - assert np.all(preds <= 1) - - -def test_probability_regressor_fit_is_noop(): - clf = FakeClassifier() - reg = ProbabilityRegressor(clf) - result = reg.fit(None, None) - assert result is reg - - -# ── _conformal_quantile ── - - -def test_conformal_quantile_returns_float(): - scores = np.array([0.1, 0.3, 0.5, 0.7, 0.9]) - q = _conformal_quantile(scores, alpha=0.1) - assert isinstance(q, float) - - -def test_conformal_quantile_empty_array(): - q = _conformal_quantile(np.array([]), alpha=0.1) - assert q == 0.0 - - -def test_conformal_quantile_high_coverage(): - """Lower alpha should give higher quantile.""" - scores = np.random.RandomState(42).random(100) - q_90 = _conformal_quantile(scores, alpha=0.1) - q_95 = _conformal_quantile(scores, alpha=0.05) - assert q_95 >= q_90 - - -# ── apply_probability_calibrator ── - - -def test_apply_calibrator_none_clips(): - scores = np.array([-0.1, 0.5, 1.2]) - result = apply_probability_calibrator(None, scores) - assert np.all(result >= 0) - assert np.all(result <= 1) - - -def test_apply_calibrator_isotonic(): - from sklearn.isotonic import IsotonicRegression - - iso = IsotonicRegression(y_min=0, y_max=1, out_of_bounds="clip") - iso.fit([0.1, 0.5, 0.9], [0, 0.5, 1]) - result = apply_probability_calibrator(iso, np.array([0.3, 0.7])) - assert result.shape == (2,) - assert np.all(result >= 0) - assert np.all(result <= 1) - - -# ── validate_coverage ── - - -def test_validate_coverage_perfect(): - """When all points are covered, coverage should be 1.0.""" - y_true = np.array([0.2, 0.5, 0.8]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - result = validate_coverage(y_true, y_intervals, alpha=0.1) - assert result["empirical_coverage"] == 1.0 - assert result["target_coverage"] == 0.9 - assert result["coverage_gap"] == pytest.approx(0.1, abs=0.01) - - -def test_validate_coverage_none(): - """When no points are covered, coverage should be 0.0.""" - y_true = np.array([0.5, 0.5, 0.5]) - y_intervals = np.array([[0.0, 0.1], [0.0, 0.1], [0.0, 0.1]]) - result = validate_coverage(y_true, y_intervals, alpha=0.1) - assert result["empirical_coverage"] == 0.0 - - -def test_validate_coverage_returns_all_keys(): - y_true = np.array([0.3, 0.7]) - y_intervals = np.array([[0.1, 0.5], [0.5, 0.9]]) - result = validate_coverage(y_true, y_intervals, alpha=0.1) - expected_keys = { - "empirical_coverage", - "target_coverage", - "coverage_gap", - "avg_interval_width", - "median_interval_width", - } - assert expected_keys.issubset(result.keys()) - - -def test_validate_coverage_width_positive(): - y_true = np.array([0.3, 0.7]) - y_intervals = np.array([[0.1, 0.5], [0.5, 0.9]]) - result = validate_coverage(y_true, y_intervals, alpha=0.1) - assert result["avg_interval_width"] > 0 - assert result["median_interval_width"] > 0 - - -# ── conditional_coverage_by_group ── - - -def test_conditional_coverage_groups(): - y_true = np.array([0.2, 0.8, 0.3, 0.9]) - y_intervals = np.array([[0.0, 0.5], [0.5, 1.0], [0.0, 0.5], [0.0, 0.5]]) - groups = pd.Series(["A", "A", "B", "B"]) - result = conditional_coverage_by_group(y_true, y_intervals, groups) - assert len(result) == 2 - assert "coverage" in result.columns - assert "avg_width" in result.columns - assert all(result["n"] == 2) - - -def test_conditional_coverage_handles_nans(): - y_true = np.array([0.5, 0.5]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0]]) - groups = pd.Series([None, "B"]) - result = conditional_coverage_by_group(y_true, y_intervals, groups) - assert len(result) == 2 # UNKNOWN + B - - -# ── conformal_metrics (from evaluation module) ── - - -def test_conformal_metrics_consistency(): - from src.evaluation.metrics import conformal_metrics - - rng = np.random.RandomState(42) - y_true = rng.random(100) - low = y_true - 0.3 - high = y_true + 0.3 - y_intervals = np.column_stack([low, high]) - result = conformal_metrics(y_true, y_intervals, alpha=0.1) - assert result["empirical_coverage"] == 1.0 # All covered with ±0.3 - assert result["avg_width"] == pytest.approx(0.6, abs=0.01) - assert result["coverage_gap"] == pytest.approx(0.1, abs=0.01) - - -def test_conformal_metrics_partial_coverage(): - from src.evaluation.metrics import conformal_metrics - - y_true = np.array([0.0, 0.5, 1.0, 0.0, 0.5, 1.0, 0.0, 0.5, 1.0, 0.0]) - low = np.full(10, 0.3) - high = np.full(10, 0.7) - y_intervals = np.column_stack([low, high]) - result = conformal_metrics(y_true, y_intervals, alpha=0.1) - assert 0 < result["empirical_coverage"] < 1 - assert result["avg_width"] == pytest.approx(0.4, abs=0.01) - - -# ── Edge Cases ── - - -def test_conformal_quantile_single_element(): - """Quantile on single-element array should return that element.""" - result = _conformal_quantile(np.array([0.5]), 0.9) - assert isinstance(result, float) - - -def test_validate_coverage_inverted_intervals(): - """Intervals where low > high should yield zero coverage.""" - y_true = np.array([0.5, 0.5]) - y_intervals = np.array([[0.8, 0.2], [0.9, 0.1]]) # inverted - result = validate_coverage(y_true, y_intervals, alpha=0.1) - assert result["empirical_coverage"] == 0.0 - - -def test_validate_coverage_with_nans_in_y_true(): - """NaN in y_true should not cause crash.""" - y_true = np.array([0.5, np.nan, 0.3]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - # Should not raise — coverage calculation handles NaN gracefully - result = validate_coverage(y_true, y_intervals, alpha=0.1) - assert "empirical_coverage" in result - - -def test_conditional_coverage_single_group(): - """One group should produce a single-row result.""" - y_true = np.array([0.2, 0.5, 0.8]) - y_intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - groups = pd.Series(["A", "A", "A"]) - result = conditional_coverage_by_group(y_true, y_intervals, groups) - assert len(result) == 1 - assert result["coverage"].iloc[0] == 1.0 - - -def test_build_mondrian_partition_labels_score_decile_returns_score_bands(): - y_prob_cal = np.linspace(0.01, 0.99, 20) - y_prob_eval = np.array([0.05, 0.55, 0.95]) - - group_cal, group_eval, meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal, - y_prob_eval=y_prob_eval, - partition="score_decile_mondrian", - ) - - assert len(group_cal) == len(y_prob_cal) - assert len(group_eval) == len(y_prob_eval) - assert meta["partition"] == "score_decile_mondrian" - assert meta["score_band_count"] >= 2 - assert all(label.startswith("score_q") for label in group_cal) - - -def test_build_mondrian_partition_labels_hybrid_falls_back_for_small_groups(): - y_prob_cal = np.linspace(0.01, 0.99, 12) - y_prob_eval = np.array([0.15, 0.85]) - base_groups_cal = pd.Series(["A"] * 10 + ["B"] * 2) - base_groups_eval = pd.Series(["A", "B"]) - - group_cal, group_eval, meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal, - y_prob_eval=y_prob_eval, - partition="grade_x_scoreband_mondrian", - base_groups_cal=base_groups_cal, - base_groups_eval=base_groups_eval, - n_score_bins=4, - min_group_size=5, - ) - - assert meta["partition"] == "grade_x_scoreband_mondrian" - assert meta["fallback_groups"] - assert "GLOBAL" in set(group_eval) - assert all(isinstance(label, str) for label in group_cal) - - -def test_build_mondrian_partition_labels_global_only_fallback_uses_global(): - y_prob_cal = np.linspace(0.01, 0.99, 10) - y_prob_eval = np.array([0.10, 0.90]) - base_groups_cal = pd.Series(["A"] * 8 + ["B"] * 2) - base_groups_eval = pd.Series(["A", "B"]) - - group_cal, group_eval, meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal, - y_prob_eval=y_prob_eval, - partition="grade_x_scoreband_mondrian", - base_groups_cal=base_groups_cal, - base_groups_eval=base_groups_eval, - n_score_bins=5, - min_group_size=20, - fallback_mode="global_only", - ) - - assert meta["fallback_mode"] == "global_only" - assert set(group_cal) == {"GLOBAL"} - assert set(group_eval) == {"GLOBAL"} - - -def test_summarize_prediction_sets_reports_ambiguity_metrics(): - y_true = np.array([0, 1, 1, 0]) - y_pred = np.array([0, 1, 1, 0]) - y_sets = np.array( - [ - [1, 0], # singleton negative - [0, 1], # singleton positive - [1, 1], # ambiguous - [0, 0], # empty - ] - ) - - result = summarize_prediction_sets(y_true, y_pred, y_sets) - - assert result["singleton_rate"] == pytest.approx(0.5) - assert result["ambiguity_rate"] == pytest.approx(0.25) - assert result["empty_set_rate"] == pytest.approx(0.25) - assert result["set_coverage"] == pytest.approx(0.75) - - -def test_create_classification_sets_margin_returns_valid_sets(): - clf = FakeBinaryClassifier(seed=7) - rng = np.random.RandomState(7) - X_cal = pd.DataFrame({"a": rng.random(120), "b": rng.random(120)}) - y_cal = pd.Series(rng.randint(0, 2, 120)) - X_test = pd.DataFrame({"a": rng.random(30), "b": rng.random(30)}) - - y_pred, y_sets = create_classification_sets( - classifier=clf, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=0.10, - method="margin", - ) - - assert y_pred.shape == (30,) - assert y_sets.shape == (30, 2) - assert np.isin(y_sets, [0, 1]).all() - - -def test_create_classification_sets_mondrian_supports_margin_with_fallback(): - clf = FakeBinaryClassifier(seed=9) - rng = np.random.RandomState(9) - X_cal = pd.DataFrame({"a": rng.random(40), "b": rng.random(40)}) - y_cal = pd.Series(rng.randint(0, 2, 40)) - X_test = pd.DataFrame({"a": rng.random(12), "b": rng.random(12)}) - group_cal = pd.Series(["A"] * 30 + ["B"] * 10) - group_test = pd.Series(["A"] * 6 + ["B"] * 6) - - y_pred, y_sets, diagnostics = create_classification_sets_mondrian( - classifier=clf, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - group_cal=group_cal, - group_test=group_test, - alpha=0.10, - method="margin", - min_group_size=25, - ) - - assert y_pred.shape == (12,) - assert y_sets.shape == (12, 2) - assert "B" in diagnostics["fallback_groups"] - - -def test_create_pd_intervals_mondrian_supports_score_scale_family(): - clf = FakeBinaryClassifier(seed=11) - rng = np.random.RandomState(11) - X_cal = pd.DataFrame({"a": rng.random(120), "b": rng.random(120)}) - y_cal = pd.Series(rng.randint(0, 2, 120)) - X_test = pd.DataFrame({"a": rng.random(24), "b": rng.random(24)}) - group_cal = pd.Series(["A"] * 60 + ["B"] * 60) - group_test = pd.Series(["A"] * 12 + ["B"] * 12) - - y_pred, y_intervals, diagnostics = create_pd_intervals_mondrian( - classifier=clf, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - group_cal=group_cal, - group_test=group_test, - alpha=0.10, - min_group_size=20, - score_scale_family="bernoulli_sqrt_clipped_0.02", - ) - - assert y_pred.shape == (24,) - assert y_intervals.shape == (24, 2) - assert diagnostics["score_scale_family"] == "bernoulli_sqrt_clipped_0.02" - - -def test_pd_intervals_mondrian_precomputed_predictions_match_classifier_route(): - clf = FakeClassifier() - x_cal = pd.DataFrame({"a": np.arange(8)}) - x_test = pd.DataFrame({"a": np.arange(4)}) - y_cal = pd.Series([0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0]) - group_cal = pd.Series(["A", "A", "B", "B", "B", "C", "C", "C"]) - group_test = pd.Series(["A", "B", "C", "D"]) - - y_pred_old, y_intervals_old, diagnostics_old = create_pd_intervals_mondrian( - classifier=clf, - X_cal=x_cal, - y_cal=y_cal, - X_test=x_test, - group_cal=group_cal, - group_test=group_test, - alpha=0.20, - min_group_size=2, - scaled_scores=True, - score_scale_family="bernoulli_sqrt", - log_summary=False, - ) - y_prob_cal = clf.predict_proba(x_cal)[:, 1] - y_prob_test = clf.predict_proba(x_test)[:, 1] - y_pred_new, y_intervals_new, diagnostics_new = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_prob_cal, - y_test_pred=y_prob_test, - y_cal=y_cal, - group_cal=group_cal, - group_test=group_test, - alpha=0.20, - min_group_size=2, - scaled_scores=True, - score_scale_family="bernoulli_sqrt", - log_summary=False, - ) - - np.testing.assert_allclose(y_pred_new, y_pred_old) - np.testing.assert_allclose(y_intervals_new, y_intervals_old) - assert diagnostics_new["fallback_groups"] == diagnostics_old["fallback_groups"] - assert diagnostics_new["group_cal_counts"] == diagnostics_old["group_cal_counts"] - - -def test_cross_conformal_score_intervals_output_shape(): - rng = np.random.RandomState(42) - y_cal = rng.binomial(1, 0.25, size=80) - y_prob_cal = rng.uniform(0.05, 0.8, size=80) - y_prob_test = rng.uniform(0.05, 0.8, size=20) - - y_pred, y_intervals = create_cross_conformal_score_intervals( - y_cal=y_cal, - y_prob_cal=y_prob_cal, - y_prob_test=y_prob_test, - alpha=0.1, - cv=3, - ) - - assert y_pred.shape == (20,) - assert y_intervals.shape == (20, 2) - assert np.all(y_pred >= 0.0) - assert np.all(y_pred <= 1.0) - assert np.all(y_intervals[:, 0] <= y_intervals[:, 1]) - assert np.all(y_intervals >= 0.0) - assert np.all(y_intervals <= 1.0) - - -# ── Residual Intervals (bootstrap-style benchmark) ── - - -def test_residual_intervals_output_shape(): - from src.models.conformal import create_residual_intervals - - clf = FakeClassifier() - X_cal = pd.DataFrame({"a": np.random.RandomState(0).random(100)}) - y_cal = pd.Series(np.random.RandomState(0).random(100)) - X_test = pd.DataFrame({"a": np.random.RandomState(1).random(20)}) - - y_pred, y_intervals = create_residual_intervals(clf, X_cal, y_cal, X_test, alpha=0.1) - assert y_pred.shape == (20,) - assert y_intervals.shape == (20, 2) - - -def test_residual_intervals_width_positive(): - from src.models.conformal import create_residual_intervals - - clf = FakeClassifier() - X_cal = pd.DataFrame({"a": np.random.RandomState(0).random(100)}) - y_cal = pd.Series(np.random.RandomState(0).random(100)) - X_test = pd.DataFrame({"a": np.random.RandomState(1).random(20)}) - - _, y_intervals = create_residual_intervals(clf, X_cal, y_cal, X_test, alpha=0.1) - widths = y_intervals[:, 1] - y_intervals[:, 0] - assert np.all(widths > 0) - - -def test_residual_intervals_narrower_with_higher_alpha(): - from src.models.conformal import create_residual_intervals - - clf = FakeClassifier() - X_cal = pd.DataFrame({"a": np.random.RandomState(0).random(200)}) - y_cal = pd.Series(np.random.RandomState(0).random(200)) - X_test = pd.DataFrame({"a": np.random.RandomState(1).random(30)}) - - _, iv_90 = create_residual_intervals(clf, X_cal, y_cal, X_test, alpha=0.10) - _, iv_50 = create_residual_intervals(clf, X_cal, y_cal, X_test, alpha=0.50) - - w_90 = (iv_90[:, 1] - iv_90[:, 0]).mean() - w_50 = (iv_50[:, 1] - iv_50[:, 0]).mean() - assert w_90 > w_50 # 90% intervals should be wider than 50% - - -# ── Venn-Abers ── - - -class FakeBinaryClassifier: - """Classifier stub that returns probabilities close to the true label.""" - - def __init__(self, seed=42): - self.rng = np.random.RandomState(seed) - self._classes = np.array([0, 1]) - - @property - def classes_(self): - return self._classes - - def predict_proba(self, X): - n = X.shape[0] if hasattr(X, "shape") else len(X) - probs = self.rng.random(n) * 0.5 + 0.25 # between 0.25 and 0.75 - return np.column_stack([1 - probs, probs]) - - def predict(self, X): - return (self.predict_proba(X)[:, 1] > 0.5).astype(int) - - -def test_venn_abers_output_shape(): - """Venn-Abers should return point predictions and p0/p1 arrays.""" - pytest.importorskip("venn_abers") - from src.models.conformal import create_pd_intervals_venn_abers - - clf = FakeBinaryClassifier(seed=42) - rng = np.random.RandomState(42) - n_cal, n_test = 200, 50 - X_cal = pd.DataFrame({"a": rng.random(n_cal), "b": rng.random(n_cal)}) - y_cal = pd.Series(rng.randint(0, 2, n_cal)) - X_test = pd.DataFrame({"a": rng.random(n_test), "b": rng.random(n_test)}) - - y_pred, p0, p1 = create_pd_intervals_venn_abers(clf, X_cal, y_cal, X_test) - assert y_pred.shape == (n_test,) - assert p0.shape == (n_test,) - assert p1.shape == (n_test,) - - -def test_venn_abers_bounds_valid(): - """p0 <= p1 and both in [0, 1].""" - pytest.importorskip("venn_abers") - from src.models.conformal import create_pd_intervals_venn_abers - - clf = FakeBinaryClassifier(seed=123) - rng = np.random.RandomState(123) - n_cal, n_test = 300, 80 - X_cal = pd.DataFrame({"a": rng.random(n_cal), "b": rng.random(n_cal)}) - y_cal = pd.Series(rng.randint(0, 2, n_cal)) - X_test = pd.DataFrame({"a": rng.random(n_test), "b": rng.random(n_test)}) - - y_pred, p0, p1 = create_pd_intervals_venn_abers(clf, X_cal, y_cal, X_test) - assert np.all(p0 >= 0) - assert np.all(p1 <= 1) - assert np.all(p0 <= p1) - assert np.all(y_pred >= p0) - assert np.all(y_pred <= p1) diff --git a/tests/test_models/test_conformal_alpha_grid.py b/tests/test_models/test_conformal_alpha_grid.py deleted file mode 100644 index 8864a2e..0000000 --- a/tests/test_models/test_conformal_alpha_grid.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from src.models.conformal_alpha_grid import ( - FrozenConformalRecipe, - alpha_interval_columns, - compute_exact_alpha_intervals, -) - - -def _payload() -> dict: - return { - "tuning_90_best": { - "partition": "grade", - "partition_probability_source": "calibrated", - "n_score_bins": 5, - "fallback_mode": "global_only", - "score_scale_family": "none", - "min_group_size": 1, - "alpha_target_90": 0.10, - "alpha_used_90": 0.095, - }, - "calibration_split": { - "calibration_fraction": 0.75, - "holdout_ratio": 0.20, - "random_state": 42, - }, - "group_coverage_multipliers": {"A": 1.05}, - "temporal_segment_multipliers": {}, - "global_rebalance": {"enabled": False, "applied": False}, - } - - -def test_recipe_scales_alpha_by_frozen_conservative_ratio() -> None: - recipe = FrozenConformalRecipe.from_results_payload(_payload()) - - assert recipe.used_alpha(0.01) == pytest.approx(0.0095) - assert alpha_interval_columns(0.01) == ( - "pd_low_alpha_0p010", - "pd_high_alpha_0p010", - ) - - -def test_recipe_rejects_narrowing_adjustments() -> None: - payload = _payload() - payload["global_rebalance"] = {"applied": True, "factor": 0.95} - - with pytest.raises(ValueError, match="widening"): - FrozenConformalRecipe.from_results_payload(payload) - - -def test_exact_alpha_intervals_use_frozen_partition_and_multiplier() -> None: - recipe = FrozenConformalRecipe.from_results_payload(_payload()) - y_cal = np.array([0.0, 1.0, 0.0, 1.0]) - p_cal = np.array([0.1, 0.8, 0.2, 0.7]) - p_eval = np.array([0.15, 0.75]) - groups_cal = pd.Series(["A", "A", "B", "B"]) - groups_eval = pd.Series(["A", "B"]) - - result = compute_exact_alpha_intervals( - recipe=recipe, - target_alpha=0.10, - y_cal=y_cal, - interval_probability_cal=p_cal, - interval_probability_eval=p_eval, - partition_probability_cal=p_cal, - partition_probability_eval=p_eval, - base_groups_cal=groups_cal, - base_groups_eval=groups_eval, - ) - - assert result.used_alpha == pytest.approx(0.095) - assert result.partition_labels.tolist() == ["A", "B"] - assert np.all(result.low <= result.point) - assert np.all(result.high >= result.point) - assert result.high[0] == pytest.approx(0.36) diff --git a/tests/test_models/test_conformal_artifact_properties.py b/tests/test_models/test_conformal_artifact_properties.py deleted file mode 100644 index 18d20f3..0000000 --- a/tests/test_models/test_conformal_artifact_properties.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Structural invariants of the frozen conformal artifact. - -The MAPIE 0.9 → 1.x migration in ``src/models/conformal.py`` is complete on -the code side (Codex commit ``c011b3d``). The remaining validation question -is: do the intervals stored in -``data/processed/conformal_intervals_mondrian.parquet`` still satisfy the -mathematical contract MAPIE guarantees? - -These tests **do not** re-run the conformal stage. They load the frozen -parquet and assert the properties any MAPIE-produced output must have. If -this file passes, the migration is internally consistent. A full bit-exact -drift validation against a fresh ``dvc repro crpto.conformal.intervals`` is -optional — see ``docs/refactor/MAPIE_MIGRATION_PLAN.md`` for the procedure. -""" - -from __future__ import annotations - -from pathlib import Path - -import numpy as np -import pandas as pd -import pytest - -ARTIFACT = Path("data/processed/conformal_intervals_mondrian.parquet") - -# Champion coverage targets (configs/crpto_conformal_policy.yaml). The -# observed coverage on the test set should be at least these minus a small -# finite-sample slack. -TARGET_90 = 0.90 -TARGET_95 = 0.95 -COVERAGE_SLACK = 0.02 # finite-sample tolerance — MAPIE guarantees ≥ target -# asymptotically, but per-Mondrian cells can dip a bit. - - -@pytest.fixture(scope="module") -def df() -> pd.DataFrame: - if not ARTIFACT.is_file(): - pytest.skip(f"{ARTIFACT} not available locally — run `dvc pull` to fetch it.") - return pd.read_parquet(ARTIFACT) - - -def test_artifact_has_expected_columns(df: pd.DataFrame) -> None: - required = { - "y_true", - "y_pred", - "pd_low_90", - "pd_high_90", - "pd_low_95", - "pd_high_95", - "width_90", - "width_95", - "grade", - } - assert required <= set(df.columns), f"Missing columns: {required - set(df.columns)}" - - -def test_predictions_in_unit_interval(df: pd.DataFrame) -> None: - for col in ("y_pred", "pd_low_90", "pd_high_90", "pd_low_95", "pd_high_95"): - values = df[col].to_numpy() - assert np.all((values >= 0.0) & (values <= 1.0)), f"{col} has values outside [0, 1]" - - -def test_intervals_are_monotone(df: pd.DataFrame) -> None: - """pd_low_90 ≤ y_pred ≤ pd_high_90 and same for 95%.""" - assert (df["pd_low_90"] <= df["y_pred"] + 1e-9).all() - assert (df["y_pred"] <= df["pd_high_90"] + 1e-9).all() - assert (df["pd_low_95"] <= df["y_pred"] + 1e-9).all() - assert (df["y_pred"] <= df["pd_high_95"] + 1e-9).all() - - -def test_95_intervals_contain_90_intervals(df: pd.DataFrame) -> None: - """The 95% interval must be at least as wide as the 90% interval.""" - assert (df["pd_low_95"] <= df["pd_low_90"] + 1e-9).all() - assert (df["pd_high_95"] >= df["pd_high_90"] - 1e-9).all() - - -def test_widths_are_non_negative(df: pd.DataFrame) -> None: - assert (df["width_90"] >= -1e-9).all() - assert (df["width_95"] >= -1e-9).all() - - -def test_width_matches_interval(df: pd.DataFrame) -> None: - """``width_*`` should equal ``pd_high_* - pd_low_*`` up to float rounding.""" - delta_90 = df["pd_high_90"].to_numpy() - df["pd_low_90"].to_numpy() - delta_95 = df["pd_high_95"].to_numpy() - df["pd_low_95"].to_numpy() - np.testing.assert_allclose(df["width_90"].to_numpy(), delta_90, atol=1e-9) - np.testing.assert_allclose(df["width_95"].to_numpy(), delta_95, atol=1e-9) - - -def test_global_coverage_meets_target(df: pd.DataFrame) -> None: - """The conformal coverage guarantee: ``P(y ∈ [low, high]) ≥ target``.""" - inside_90 = ((df["y_true"] >= df["pd_low_90"]) & (df["y_true"] <= df["pd_high_90"])).mean() - inside_95 = ((df["y_true"] >= df["pd_low_95"]) & (df["y_true"] <= df["pd_high_95"])).mean() - assert inside_90 >= TARGET_90 - COVERAGE_SLACK, ( - f"90% coverage observed = {inside_90:.4f}, expected ≥ {TARGET_90 - COVERAGE_SLACK:.4f}" - ) - assert inside_95 >= TARGET_95 - COVERAGE_SLACK, ( - f"95% coverage observed = {inside_95:.4f}, expected ≥ {TARGET_95 - COVERAGE_SLACK:.4f}" - ) - - -def test_per_grade_coverage_within_tolerance(df: pd.DataFrame) -> None: - """Mondrian conditional coverage: each grade should also achieve target ± slack.""" - df = df.copy() - df["in_90"] = (df["y_true"] >= df["pd_low_90"]) & (df["y_true"] <= df["pd_high_90"]) - per_grade = df.groupby("grade", observed=True)["in_90"].mean() - failures = per_grade[per_grade < TARGET_90 - COVERAGE_SLACK] - assert failures.empty, f"Mondrian coverage failed for grades: {failures.to_dict()}" - - -def test_artifact_row_count_matches_champion(df: pd.DataFrame) -> None: - """The frozen run produced 276 869 rows; any change to this number breaks - the paper's contribution count.""" - assert len(df) == 276869 - - -@pytest.mark.slow -@pytest.mark.integration -def test_mapie_drift_harness_skipped_by_default() -> None: - """Placeholder for the full MAPIE 1.x drift harness. - - The harness re-runs ``scripts/generate_conformal_intervals.py`` on a - branch, compares hash-by-hash against the frozen parquet, and asserts - ``max abs diff ≤ 1e-6`` per loan. Today it is skipped by default; trigger - it manually with: - - uv run pytest -m "slow and integration" tests/test_models/test_conformal_artifact_properties.py - - See ``docs/refactor/MAPIE_MIGRATION_PLAN.md`` for the full procedure. - """ - pytest.skip("Manual drift harness — run `dvc repro crpto.conformal.intervals` instead.") diff --git a/tests/test_models/test_conformal_artifacts.py b/tests/test_models/test_conformal_artifacts.py deleted file mode 100644 index dfc53d3..0000000 --- a/tests/test_models/test_conformal_artifacts.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Tests for canonical conformal artifact loading contract.""" - -from __future__ import annotations - -import pandas as pd -import pytest - -from src.models import conformal_artifacts as ca - - -def test_load_conformal_intervals_uses_canonical_only(tmp_path, monkeypatch) -> None: - canonical = tmp_path / "conformal_intervals_mondrian.parquet" - legacy = tmp_path / "conformal_intervals.parquet" - pd.DataFrame({"x": [1, 2]}).to_parquet(canonical, index=False) - pd.DataFrame({"x": [3, 4]}).to_parquet(legacy, index=False) - - monkeypatch.setattr(ca, "CANONICAL_INTERVALS_PATH", canonical) - path, is_legacy = ca.resolve_intervals_path() - assert path == canonical - assert is_legacy is False - - -def test_resolve_intervals_path_raises_without_canonical(tmp_path, monkeypatch) -> None: - canonical = tmp_path / "missing.parquet" - legacy = tmp_path / "conformal_intervals.parquet" - pd.DataFrame({"x": [3, 4]}).to_parquet(legacy, index=False) - - monkeypatch.setattr(ca, "CANONICAL_INTERVALS_PATH", canonical) - with pytest.raises(FileNotFoundError): - ca.resolve_intervals_path() diff --git a/tests/test_models/test_conformal_edge_cases.py b/tests/test_models/test_conformal_edge_cases.py deleted file mode 100644 index 8c211da..0000000 --- a/tests/test_models/test_conformal_edge_cases.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Edge-case tests for conformal prediction and new features. - -Covers: -- All-positive / all-negative calibration sets -- Degenerate distributions (near-constant features) -- Very small calibration sets -- Brier decomposition -- PSI feature filter -- Monotonic constraints resolver -- sklearn Pipeline wrapper -- Survival conformal intervals -""" - -import numpy as np -import pandas as pd -import pytest - -from src.evaluation.metrics import brier_score_decomposition -from src.models.conformal import _conformal_quantile, validate_coverage - -# ── Conformal edge cases ────────────────────────────────────────────── - - -def test_conformal_all_positive_calibration(): - """Calibration set with all y=1 should not crash.""" - scores = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) - q = _conformal_quantile(scores, alpha=0.1) - assert 0 <= q <= 1.0 - - -def test_conformal_all_negative_calibration(): - """Calibration set with all y=0 should not crash.""" - scores = np.array([0.01, 0.02, 0.03, 0.04, 0.05]) - q = _conformal_quantile(scores, alpha=0.1) - assert 0 <= q <= 1.0 - - -def test_conformal_very_small_calibration(): - """Calibration set with 2 observations.""" - scores = np.array([0.1, 0.9]) - q = _conformal_quantile(scores, alpha=0.1) - assert np.isfinite(q) - - -def test_conformal_constant_scores(): - """All conformity scores identical.""" - scores = np.full(100, 0.5) - q = _conformal_quantile(scores, alpha=0.1) - assert q == pytest.approx(0.5) - - -def test_validate_coverage_all_covered(): - """All observations within intervals → 100% coverage.""" - y_true = np.array([0.3, 0.5, 0.7]) - intervals = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]) - result = validate_coverage(y_true, intervals, alpha=0.1) - assert result["empirical_coverage"] == pytest.approx(1.0) - - -def test_validate_coverage_none_covered(): - """All observations outside intervals → 0% coverage.""" - y_true = np.array([0.0, 0.0, 0.0]) - intervals = np.array([[0.5, 1.0], [0.5, 1.0], [0.5, 1.0]]) - result = validate_coverage(y_true, intervals, alpha=0.1) - assert result["empirical_coverage"] == pytest.approx(0.0) - - -# ── Brier decomposition ────────────────────────────────────────────── - - -def test_brier_decomposition_perfect_predictions(): - """Perfect predictions: reliability=0, resolution=uncertainty.""" - y_true = np.array([0, 0, 0, 1, 1, 1, 0, 0, 1, 1]) - y_prob = y_true.astype(float) - result = brier_score_decomposition(y_true, y_prob, n_bins=5) - assert result["reliability"] == pytest.approx(0.0, abs=0.01) - assert result["brier_decomposed"] == pytest.approx(0.0, abs=0.01) - - -def test_brier_decomposition_constant_predictions(): - """Constant predictions at base rate: reliability=0, resolution=0.""" - y_true = np.array([0, 0, 0, 1, 1]) - base_rate = y_true.mean() - y_prob = np.full_like(y_true, base_rate, dtype=float) - result = brier_score_decomposition(y_true, y_prob, n_bins=3) - assert result["reliability"] == pytest.approx(0.0, abs=0.02) - assert result["resolution"] == pytest.approx(0.0, abs=0.02) - assert result["uncertainty"] == pytest.approx(base_rate * (1 - base_rate), abs=0.01) - - -def test_brier_decomposition_components_sum(): - """Verify: Brier = Reliability - Resolution + Uncertainty.""" - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, 500) - y_prob = np.clip(y_true + rng.normal(0, 0.3, 500), 0, 1) - result = brier_score_decomposition(y_true, y_prob, n_bins=10) - expected_brier = result["reliability"] - result["resolution"] + result["uncertainty"] - assert result["brier_decomposed"] == pytest.approx(expected_brier, abs=0.01) - - -def test_brier_decomposition_shares_nonnegative(): - """Miscalibration and discrimination shares should be non-negative.""" - rng = np.random.RandomState(99) - y_true = rng.randint(0, 2, 200) - y_prob = np.clip(rng.random(200), 0.01, 0.99) - result = brier_score_decomposition(y_true, y_prob) - assert result["miscalibration_share"] >= 0 - assert result["discrimination_share"] >= 0 - - -# ── PSI feature filter ──────────────────────────────────────────────── - - -def test_filter_high_psi_stable(): - """Features with same distribution should be stable.""" - from src.evaluation.backtesting import filter_high_psi_features - - rng = np.random.RandomState(42) - train = pd.DataFrame({"a": rng.normal(0, 1, 1000), "b": rng.normal(5, 2, 1000)}) - test = pd.DataFrame({"a": rng.normal(0, 1, 500), "b": rng.normal(5, 2, 500)}) - result = filter_high_psi_features(train, test, ["a", "b"]) - assert len(result["stable_features"]) == 2 - assert len(result["drifted_features"]) == 0 - - -def test_filter_high_psi_detects_drift(): - """Feature with shifted distribution should be flagged.""" - from src.evaluation.backtesting import filter_high_psi_features - - rng = np.random.RandomState(42) - train = pd.DataFrame({"a": rng.normal(0, 1, 1000), "b": rng.normal(0, 1, 1000)}) - # Shift 'b' by 5 standard deviations → extreme drift - test = pd.DataFrame({"a": rng.normal(0, 1, 500), "b": rng.normal(5, 1, 500)}) - result = filter_high_psi_features(train, test, ["a", "b"], psi_threshold=0.25) - assert "b" in result["drifted_features"] - assert "a" in result["stable_features"] - - -# ── Monotonic constraints ──────────────────────────────────────────── - - -def test_resolve_monotonic_constraints_basic(): - """Direct config dict produces correct constraint string.""" - from src.models.pd_model import resolve_monotonic_constraints - - features = ["int_rate", "annual_inc", "dti", "grade_woe"] - constraints = {"int_rate": 1, "annual_inc": -1, "dti": 1} - result = resolve_monotonic_constraints(features, constraints_config=constraints) - assert result == "1,-1,1,0" - - -def test_resolve_monotonic_constraints_empty(): - """No constraints configured returns None.""" - from src.models.pd_model import resolve_monotonic_constraints - - features = ["a", "b", "c"] - result = resolve_monotonic_constraints(features, constraints_config={}) - assert result is None - - -def test_resolve_monotonic_constraints_all_zero(): - """All features unconstrained returns None.""" - from src.models.pd_model import resolve_monotonic_constraints - - features = ["a", "b"] - result = resolve_monotonic_constraints(features, constraints_config={"x": 1}) - assert result is None - - -# ── sklearn Pipeline wrapper ───────────────────────────────────────── - - -def test_pd_pipeline_feature_selector(): - """Feature selector extracts and orders columns correctly.""" - from src.models.pd_pipeline import _make_feature_selector - - selector = _make_feature_selector(["b", "a", "c"]) - df = pd.DataFrame({"a": [1, 2], "b": [3, 4], "c": [5, 6], "extra": [7, 8]}) - result = selector.transform(df) - assert list(result.columns) == ["b", "a", "c"] - assert result["b"].tolist() == [3, 4] - - -def test_pd_pipeline_feature_selector_missing_col(): - """Missing column filled with default value.""" - from src.models.pd_pipeline import _make_feature_selector - - selector = _make_feature_selector(["a", "missing"], fill_value=-1.0) - df = pd.DataFrame({"a": [1, 2]}) - result = selector.transform(df) - assert list(result.columns) == ["a", "missing"] - assert result["missing"].tolist() == [-1.0, -1.0] - - -def test_catboost_sklearn_adapter_fit_predict(): - """CatBoostSklearnAdapter can fit and predict.""" - from src.models.pd_pipeline import CatBoostSklearnAdapter - - rng = np.random.RandomState(42) - X = pd.DataFrame({"a": rng.random(100), "b": rng.random(100)}) - y = rng.randint(0, 2, 100) - - adapter = CatBoostSklearnAdapter( - catboost_params={"iterations": 10, "verbose": 0, "allow_writing_files": False} - ) - adapter.fit(X, y) - proba = adapter.predict_proba(X) - assert proba.shape == (100, 2) - assert np.all(proba >= 0) and np.all(proba <= 1) diff --git a/tests/test_models/test_conformal_mapie_drift.py b/tests/test_models/test_conformal_mapie_drift.py deleted file mode 100644 index 7648e42..0000000 --- a/tests/test_models/test_conformal_mapie_drift.py +++ /dev/null @@ -1,408 +0,0 @@ -"""Champion conformal drift harness (Track B gate). - -Recomputes the frozen champion conformal intervals from scratch under the -CURRENT dependency stack (MAPIE 1.x runtime, current numpy/pandas/catboost/ -sklearn) using the exact frozen recipe recorded in -``models/conformal_results_mondrian.pkl``, and compares against the frozen -``data/processed/conformal_intervals_mondrian.parquet``. - -This is the acceptance gate of ``docs/refactor/MAPIE_MIGRATION_PLAN.md``: - -* per-loan max abs diff on y_pred / interval endpoints <= 1e-6 -* per-Mondrian-cell coverage delta <= 5e-4 - -It does NOT rerun any protected DVC stage and writes nothing; it is a pure -in-memory recomputation. Marked ``slow`` because it scores ~514k rows with -the champion CatBoost model twice (calibration + test). - -GATE STATUS (2026-06-10): GREEN — RESOLVED with zero drift. The April -search candidate (``models/search_pd/pd-hpo-local-2026-04-03-1325``, the -exact ``model_path`` recorded in the frozen results pkl) was restored and -the canonical PD identity was unified to it (see the april-lineage -unification entry in ``EXTRACTION_MANIFEST.json`` and -``docs/refactor/drift_report_mapie_2026-06.md``). The harness now -reproduces every frozen column exactly (max abs diff 0.0) and re-learns -the identical floor multipliers. A RED result on this gate from now on -means something moved the frozen binaries or the conformal code path — -treat it as a stop-the-line signal, not a known condition. - -The harness stays opt-in because it scores ~514k rows (slow), not because -it is expected to fail:: - - CRPTO_RUN_CHAMPION_DRIFT=1 uv run pytest \ - tests/test_models/test_conformal_mapie_drift.py -q -s -""" - -from __future__ import annotations - -import os -import pickle -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -import pytest - -ROOT = Path(__file__).resolve().parents[2] - -if os.getenv("CRPTO_RUN_CHAMPION_DRIFT", "").lower() not in {"1", "true", "yes"}: - pytest.skip( - "Champion drift harness is opt-in (set CRPTO_RUN_CHAMPION_DRIFT=1): " - "it scores ~514k rows. Gate is GREEN since the 2026-06-10 " - "april-lineage unification; RED now means real drift.", - allow_module_level=True, - ) - -FROZEN_INTERVALS_PATH = ROOT / "data" / "processed" / "conformal_intervals_mondrian.parquet" -FROZEN_RESULTS_PATH = ROOT / "models" / "conformal_results_mondrian.pkl" -REQUIRED_ARTIFACTS = ( - FROZEN_INTERVALS_PATH, - FROZEN_RESULTS_PATH, - ROOT / "models" / "pd_canonical.cbm", - ROOT / "models" / "pd_canonical_calibrator.pkl", - ROOT / "data" / "processed" / "calibration_fe.parquet", - ROOT / "data" / "processed" / "test_fe.parquet", -) - -MAX_ABS_ENDPOINT_DIFF = 1e-6 -MAX_CELL_COVERAGE_DELTA = 5e-4 - -pytestmark = pytest.mark.slow - - -def _require_artifacts() -> None: - missing = [str(p.relative_to(ROOT)) for p in REQUIRED_ARTIFACTS if not p.is_file()] - if missing: - pytest.skip(f"Frozen artifacts not available locally (run `dvc pull`): {missing}") - - -@pytest.fixture(scope="module") -def frozen() -> dict[str, Any]: - _require_artifacts() - with FROZEN_RESULTS_PATH.open("rb") as fh: - results = pickle.load(fh) - intervals = pd.read_parquet(FROZEN_INTERVALS_PATH) - return {"results": results, "intervals": intervals} - - -@pytest.fixture(scope="module") -def recomputed(frozen: dict[str, Any]) -> dict[str, Any]: - """Recompute the frozen interval table from the recorded recipe.""" - from scripts.generate_conformal_intervals import ( - GROUP_COL, - TARGET_COL, - _build_feature_matrix, - _load_calibrator, - _load_model, - _resolve_features, - ) - from src.models.conformal import ( - apply_probability_calibrator, - build_mondrian_partition_labels, - create_pd_intervals_mondrian, - ) - from src.models.conformal_tuning import ( - apply_group_multipliers, - split_calibration_for_tuning, - ) - from src.utils.io_utils import read_with_fallback - - results = frozen["results"] - best_cfg = results["tuning_90_best"] - - # Preconditions: the harness reproduces exactly the frozen post-processing - # chain. If a future re-promotion changes these switches the harness must - # be revisited rather than silently mis-reproducing. - assert best_cfg["partition"] == "score_decile_mondrian" - assert best_cfg["partition_probability_source"] == "raw" - assert not results.get("temporal_segment_multipliers"), ( - "Frozen run applied temporal multipliers; harness does not model them." - ) - assert not results.get("shrinkback_enabled"), ( - "Frozen run used shrinkback; harness does not model it." - ) - assert not results.get("global_rebalance", {}).get("applied"), ( - "Frozen run applied global rebalance; harness does not model it." - ) - - # Prefer the literal frozen recipe: the results pkl records the exact - # model binary that produced the frozen intervals (the April search - # candidate). Since the 2026-06-10 april-lineage unification the - # canonical pd_canonical.cbm is a byte-copy of that candidate, so the - # fallback path is equivalent — it exists for partial checkouts where - # the search bundle was not restored locally. - recorded_model_path = ROOT / str(results.get("model_path", "")) - if recorded_model_path.is_file(): - from catboost import CatBoostClassifier - - model = CatBoostClassifier() - model.load_model(str(recorded_model_path)) - recorded_calibrator_path = recorded_model_path.with_name("pd_candidate_calibrator.pkl") - calibrator = _load_calibrator( - str(recorded_calibrator_path) if recorded_calibrator_path.is_file() else None - ) - print(f"\nUsing frozen-recipe model: {recorded_model_path}") - else: - model, _ = _load_model() - calibrator = _load_calibrator(results.get("calibrator_override_path") or None) - print( - "\nFrozen-recipe bundle not in local checkout; using canonical " - "binaries (byte-equal to the candidate since the april-lineage " - "unification — the gate must still be GREEN)." - ) - - cal_df = read_with_fallback( - "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" - ).reset_index(drop=True) - test_df = read_with_fallback( - "data/processed/test_fe.parquet", "data/processed/test.parquet" - ).reset_index(drop=True) - - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df[TARGET_COL].astype(float) - X_test = _build_feature_matrix(test_df, features, categorical) - group_cal_base = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str) - group_test_base = test_df[GROUP_COL].fillna("UNKNOWN").astype(str) - - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - y_prob_test_raw = model.predict_proba(X_test)[:, 1] - - split_meta = results["calibration_split"] - idx_fit, idx_tune = split_calibration_for_tuning( - y_cal=y_cal, - group_cal=group_cal_base, - issue_dates=cal_df.get("issue_d"), - holdout_ratio=float(split_meta["holdout_ratio"]), - random_state=int(split_meta["random_state"]), - ) - assert len(idx_fit) == int(split_meta["fit_n"]), ( - f"Calibration fit split drifted: {len(idx_fit)} vs frozen {split_meta['fit_n']}" - ) - assert len(idx_tune) == int(split_meta["holdout_n"]), ( - f"Calibration holdout split drifted: {len(idx_tune)} vs frozen {split_meta['holdout_n']}" - ) - - X_cal_fit = X_cal.iloc[idx_fit].reset_index(drop=True) - y_cal_fit = y_cal.iloc[idx_fit].reset_index(drop=True) - group_cal_fit_base = group_cal_base.iloc[idx_fit].reset_index(drop=True) - prob_fit_raw = y_prob_cal_raw[idx_fit] - - group_cal_fit, group_test, partition_meta = build_mondrian_partition_labels( - y_prob_cal=prob_fit_raw, - y_prob_eval=y_prob_test_raw, - partition=best_cfg["partition"], - base_groups_cal=group_cal_fit_base, - base_groups_eval=group_test_base, - n_score_bins=int(best_cfg["n_score_bins"]), - min_group_size=int(best_cfg["min_group_size"]), - fallback_mode=str(best_cfg["fallback_mode"]), - ) - frozen_edges = np.asarray(results["partition_meta"]["score_band_edges"], dtype=float) - recomputed_edges = np.asarray(partition_meta["score_band_edges"], dtype=float) - assert len(frozen_edges) == len(recomputed_edges), "Score band count drifted." - edge_drift = float(np.max(np.abs(frozen_edges - recomputed_edges))) - - y_pred_90, y_int_90, _ = create_pd_intervals_mondrian( - classifier=model, - X_cal=X_cal_fit, - y_cal=y_cal_fit, - X_test=X_test, - group_cal=group_cal_fit, - group_test=group_test, - alpha=float(best_cfg["alpha_used_90"]), - min_group_size=int(best_cfg["min_group_size"]), - calibrator=calibrator, - scaled_scores=bool(best_cfg["scaled_scores"]), - score_scale_family=str(best_cfg["score_scale_family"]), - ) - y_pred_95, y_int_95, _ = create_pd_intervals_mondrian( - classifier=model, - X_cal=X_cal_fit, - y_cal=y_cal_fit, - X_test=X_test, - group_cal=group_cal_fit, - group_test=group_test, - alpha=float(results["alpha_used_95"]), - min_group_size=int(best_cfg["min_group_size"]), - calibrator=calibrator, - scaled_scores=bool(best_cfg["scaled_scores"]), - score_scale_family=str(best_cfg["score_scale_family"]), - ) - - multipliers = {str(k): float(v) for k, v in results["group_coverage_multipliers"].items()} - if multipliers: - y_int_90 = apply_group_multipliers(y_pred_90, y_int_90, group_test, multipliers) - y_int_95 = apply_group_multipliers(y_pred_95, y_int_95, group_test, multipliers) - - y_prob_test_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_test_raw) - if calibrator is not None - else np.asarray(y_prob_test_raw, dtype=float) - ) - - return { - "y_pred": np.asarray(y_pred_90, dtype=float), - "y_pred_95": np.asarray(y_pred_95, dtype=float), - "y_pred_calibrated_direct": np.asarray(y_prob_test_calibrated, dtype=float), - "intervals_90": np.asarray(y_int_90, dtype=float), - "intervals_95": np.asarray(y_int_95, dtype=float), - "groups": pd.Series(group_test).astype(str).reset_index(drop=True), - "y_true": test_df[TARGET_COL].astype(float).to_numpy(), - "edge_drift": edge_drift, - "multipliers": multipliers, - "idx_tune": idx_tune, - "y_cal": y_cal, - "cal_df": cal_df, - "X_cal": X_cal, - "group_cal_fit_base": group_cal_fit_base, - "prob_fit_raw": prob_fit_raw, - "y_prob_cal_raw": y_prob_cal_raw, - "model": model, - "calibrator": calibrator, - "best_cfg": best_cfg, - "X_cal_fit": X_cal_fit, - "y_cal_fit": y_cal_fit, - } - - -def test_recomputed_intervals_match_frozen_parquet( - frozen: dict[str, Any], recomputed: dict[str, Any] -) -> None: - table = frozen["intervals"] - assert len(table) == len(recomputed["y_pred"]), ( - f"Row count drifted: frozen {len(table)} vs recomputed {len(recomputed['y_pred'])}" - ) - - diffs = { - "y_pred": np.max(np.abs(table["y_pred"].to_numpy(dtype=float) - recomputed["y_pred"])), - "pd_low_90": np.max( - np.abs(table["pd_low_90"].to_numpy(dtype=float) - recomputed["intervals_90"][:, 0]) - ), - "pd_high_90": np.max( - np.abs(table["pd_high_90"].to_numpy(dtype=float) - recomputed["intervals_90"][:, 1]) - ), - "pd_low_95": np.max( - np.abs(table["pd_low_95"].to_numpy(dtype=float) - recomputed["intervals_95"][:, 0]) - ), - "pd_high_95": np.max( - np.abs(table["pd_high_95"].to_numpy(dtype=float) - recomputed["intervals_95"][:, 1]) - ), - "score_band_edges": recomputed["edge_drift"], - } - print("\nDrift report (max abs diff per column):") - for name, value in diffs.items(): - print(f" {name:18s} {value:.3e}") - - violations = {k: v for k, v in diffs.items() if v > MAX_ABS_ENDPOINT_DIFF} - assert not violations, ( - f"Champion conformal drift above {MAX_ABS_ENDPOINT_DIFF:g}: {violations}. " - "Per docs/refactor/MAPIE_MIGRATION_PLAN.md this is a model change, not a " - "refactor — STOP Track B and open a revalidation decision." - ) - - -def test_recomputed_partition_labels_match_frozen( - frozen: dict[str, Any], recomputed: dict[str, Any] -) -> None: - table = frozen["intervals"] - frozen_labels = table["grade"].astype(str).reset_index(drop=True) - mismatches = int((frozen_labels != recomputed["groups"]).sum()) - assert mismatches == 0, f"{mismatches} Mondrian partition labels drifted." - - -def test_per_cell_coverage_within_tolerance( - frozen: dict[str, Any], recomputed: dict[str, Any] -) -> None: - from src.models.conformal import conditional_coverage_by_group - - table = frozen["intervals"] - y_true = table["y_true"].to_numpy(dtype=float) - frozen_cov = conditional_coverage_by_group( - y_true, - table[["pd_low_90", "pd_high_90"]].to_numpy(dtype=float), - table["grade"], - ).set_index("group")["coverage"] - new_cov = conditional_coverage_by_group( - recomputed["y_true"], - recomputed["intervals_90"], - recomputed["groups"], - ).set_index("group")["coverage"] - - aligned = pd.concat([frozen_cov, new_cov], axis=1, keys=["frozen", "new"]).dropna() - deltas = (aligned["frozen"] - aligned["new"]).abs() - print("\nPer-cell 90% coverage deltas:") - for group, delta in deltas.items(): - print(f" {group:12s} {delta:.3e}") - worst = float(deltas.max()) - assert worst <= MAX_CELL_COVERAGE_DELTA, ( - f"Per-cell coverage drift {worst:.3e} exceeds {MAX_CELL_COVERAGE_DELTA:g}." - ) - - -def test_recomputed_floor_multipliers_match_frozen( - frozen: dict[str, Any], recomputed: dict[str, Any] -) -> None: - """Re-learn the group coverage floor multipliers on the holdout split. - - The frozen run learned ``group_coverage_multipliers`` on the calibration - holdout via a fixed grid; reproducing the exact dict shows the tuning - path is stable, not just the base intervals. - """ - from src.models.conformal import build_mondrian_partition_labels, create_pd_intervals_mondrian - from src.models.conformal_tuning import enforce_group_coverage_floor - - results = frozen["results"] - best_cfg = recomputed["best_cfg"] - idx_tune = recomputed["idx_tune"] - cal_df = recomputed["cal_df"] - X_cal = recomputed["X_cal"] - y_cal = recomputed["y_cal"] - prob_tune_raw = recomputed["y_prob_cal_raw"][idx_tune] - group_tune_base = ( - cal_df["grade"].fillna("UNKNOWN").astype(str).iloc[idx_tune].reset_index(drop=True) - ) - - group_cal_fit_holdout, group_tune, _ = build_mondrian_partition_labels( - y_prob_cal=recomputed["prob_fit_raw"], - y_prob_eval=prob_tune_raw, - partition=best_cfg["partition"], - base_groups_cal=recomputed["group_cal_fit_base"], - base_groups_eval=group_tune_base, - n_score_bins=int(best_cfg["n_score_bins"]), - min_group_size=int(best_cfg["min_group_size"]), - fallback_mode=str(best_cfg["fallback_mode"]), - ) - X_tune = X_cal.iloc[idx_tune].reset_index(drop=True) - y_tune = y_cal.iloc[idx_tune].reset_index(drop=True) - y_pred_tune, y_int_tune, _ = create_pd_intervals_mondrian( - classifier=recomputed["model"], - X_cal=recomputed["X_cal_fit"], - y_cal=recomputed["y_cal_fit"], - X_test=X_tune, - group_cal=group_cal_fit_holdout, - group_test=group_tune, - alpha=float(best_cfg["alpha_used_90"]), - min_group_size=int(best_cfg["min_group_size"]), - calibrator=recomputed["calibrator"], - scaled_scores=bool(best_cfg["scaled_scores"]), - score_scale_family=str(best_cfg["score_scale_family"]), - ) - _, learned, _ = enforce_group_coverage_floor( - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_tune, - groups=group_tune, - target_coverage=float(results["group_coverage_floor_target_90"]), - multiplier_grid=(1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - ) - learned_clean = {str(k): float(v) for k, v in learned.items() if float(v) > 1.0} - frozen_mult = { - str(k): float(v) for k, v in results["group_coverage_multipliers"].items() if float(v) > 1.0 - } - print(f"\nFloor multipliers — frozen: {frozen_mult} / relearned: {learned_clean}") - assert learned_clean == frozen_mult, ( - "Group coverage floor multipliers drifted between the frozen run and " - "the recomputation on the current stack." - ) diff --git a/tests/test_models/test_conformal_tuning.py b/tests/test_models/test_conformal_tuning.py deleted file mode 100644 index 8ae730e..0000000 --- a/tests/test_models/test_conformal_tuning.py +++ /dev/null @@ -1,465 +0,0 @@ -"""Tests for src/models/conformal_tuning.py. - -Covers calibration splitting, Pareto front, config selection, -group multipliers, and coverage floor enforcement. -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from src.models.conformal_tuning import ( - apply_group_multipliers, - build_group_temporal_segments, - choose_best_tuning_row, - empirical_interval_coverage, - enforce_group_coverage_floor, - enforce_segment_coverage_floor, - mark_pareto_front, - min_group_interval_coverage, - shrink_group_multipliers, - split_calibration_for_tuning, - to_python_scalar, -) - -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- - - -@pytest.fixture -def cal_data(): - """Small calibration dataset with two classes and two groups.""" - rng = np.random.default_rng(42) - n = 200 - y = pd.Series(rng.integers(0, 2, size=n)) - groups = pd.Series(np.where(np.arange(n) < n // 2, "A", "B")) - dates = pd.Series(pd.date_range("2018-01-01", periods=n, freq="W")) - return y, groups, dates - - -@pytest.fixture -def tuning_results(): - """DataFrame resembling multi-config tuning output.""" - return pd.DataFrame( - { - "empirical_coverage": [0.88, 0.91, 0.93, 0.95, 0.90], - "min_group_coverage": [0.85, 0.89, 0.90, 0.93, 0.87], - "avg_interval_width": [0.10, 0.12, 0.15, 0.20, 0.11], - "coverage_gap": [0.02, 0.01, 0.00, -0.05, 0.01], - } - ) - - -# --------------------------------------------------------------------------- -# split_calibration_for_tuning -# --------------------------------------------------------------------------- - - -class TestSplitCalibration: - def test_partitions_are_disjoint_and_cover_all(self, cal_data): - y, groups, dates = cal_data - idx_fit, idx_tune = split_calibration_for_tuning(y, groups, dates) - all_idx = np.sort(np.concatenate([idx_fit, idx_tune])) - np.testing.assert_array_equal(all_idx, np.arange(len(y))) - - def test_holdout_ratio_respected(self, cal_data): - y, groups, dates = cal_data - ratio = 0.20 - _, idx_tune = split_calibration_for_tuning(y, groups, dates, holdout_ratio=ratio) - actual_ratio = len(idx_tune) / len(y) - assert abs(actual_ratio - ratio) < 0.05 - - def test_temporal_split_when_dates_available(self, cal_data): - y, groups, dates = cal_data - idx_fit, idx_tune = split_calibration_for_tuning(y, groups, dates) - # Tuning set should contain the latest dates - fit_max = dates.iloc[idx_fit].max() - tune_min = dates.iloc[idx_tune].min() - assert tune_min >= fit_max or len(idx_tune) > 0 - - def test_fallback_without_dates(self, cal_data): - y, groups, _ = cal_data - idx_fit, idx_tune = split_calibration_for_tuning(y, groups, issue_dates=None) - assert len(idx_fit) + len(idx_tune) == len(y) - - def test_single_row_returns_empty_tune(self): - y = pd.Series([1]) - groups = pd.Series(["A"]) - idx_fit, idx_tune = split_calibration_for_tuning(y, groups) - assert len(idx_fit) == 1 - assert len(idx_tune) == 0 - - def test_holdout_ratio_is_clipped(self, cal_data): - y, groups, _ = cal_data - # Extreme ratios should be clipped to [0.05, 0.50] - _, idx_tune_low = split_calibration_for_tuning(y, groups, holdout_ratio=0.001) - _, idx_tune_high = split_calibration_for_tuning(y, groups, holdout_ratio=0.99) - assert len(idx_tune_low) >= 1 - assert len(idx_tune_high) <= len(y) - 1 - - -# --------------------------------------------------------------------------- -# mark_pareto_front -# --------------------------------------------------------------------------- - - -class TestParetoFront: - def test_single_row_is_pareto(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.90], - "min_group_coverage": [0.85], - "avg_interval_width": [0.10], - } - ) - result = mark_pareto_front(df) - assert result.iloc[0] is True or result.iloc[0] == True # noqa: E712 - - def test_dominated_point_excluded(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.90, 0.95], - "min_group_coverage": [0.85, 0.90], - "avg_interval_width": [0.15, 0.10], - } - ) - result = mark_pareto_front(df) - # Row 0 is dominated by row 1 (worse on all 3 objectives) - assert result.iloc[0] == False # noqa: E712 - assert result.iloc[1] == True # noqa: E712 - - def test_incomparable_points_both_pareto(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.95, 0.90], - "min_group_coverage": [0.85, 0.90], - "avg_interval_width": [0.10, 0.10], - } - ) - result = mark_pareto_front(df) - # Neither dominates the other - assert result.all() - - -# --------------------------------------------------------------------------- -# choose_best_tuning_row -# --------------------------------------------------------------------------- - - -class TestChooseBestTuningRow: - def test_selects_meeting_target(self, tuning_results): - row, tier = choose_best_tuning_row( - tuning_results, target_coverage=0.90, min_group_coverage_target=0.85 - ) - assert row["empirical_coverage"] >= 0.90 - - def test_prefers_narrow_width(self, tuning_results): - row, tier = choose_best_tuning_row( - tuning_results, target_coverage=0.90, min_group_coverage_target=0.85 - ) - # Among qualifying rows, should prefer smaller width - assert row["avg_interval_width"] <= 0.20 - - def test_fallback_when_nothing_meets_target(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.60, 0.65], - "min_group_coverage": [0.50, 0.55], - "avg_interval_width": [0.10, 0.12], - "coverage_gap": [0.30, 0.25], - } - ) - row, tier = choose_best_tuning_row(df, target_coverage=0.99, min_group_coverage_target=0.95) - assert tier == "fallback_penalty" - - def test_width_budget_prefers_narrow(self, tuning_results): - # With tight budget, width tiers should be tried first; - # if nothing qualifies for width, it falls through to non-width tiers - row_strict, tier_strict = choose_best_tuning_row( - tuning_results, - target_coverage=0.90, - min_group_coverage_target=0.85, - max_width_budget=0.11, - ) - row_loose, tier_loose = choose_best_tuning_row( - tuning_results, - target_coverage=0.90, - min_group_coverage_target=0.85, - max_width_budget=None, - ) - # Both should still return valid rows - assert row_strict["empirical_coverage"] >= 0.90 or tier_strict == "fallback_penalty" - assert row_loose["empirical_coverage"] >= 0.90 or tier_loose == "fallback_penalty" - - def test_prefers_lower_winkler_within_same_feasible_tier(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.915, 0.915], - "min_group_coverage": [0.89, 0.89], - "avg_interval_width": [0.15, 0.16], - "winkler_90": [1.10, 1.35], - "max_monthly_gap": [0.03, 0.03], - "stability_over_time": [0.02, 0.02], - "coverage_gap": [0.015, 0.015], - } - ) - row, tier = choose_best_tuning_row( - df, - target_coverage=0.90, - min_group_coverage_target=0.88, - max_width_budget=0.20, - ) - assert tier == "strong_global+strong_group+width" - assert row["winkler_90"] == pytest.approx(1.10) - - def test_prefers_closer_to_target_before_extra_conservatism(self): - df = pd.DataFrame( - { - "empirical_coverage": [0.905, 0.930], - "min_group_coverage": [0.89, 0.89], - "avg_interval_width": [0.78, 0.80], - "winkler_90": [1.12, 1.05], - "max_monthly_gap": [0.02, 0.02], - "stability_over_time": [0.01, 0.01], - "coverage_gap": [0.005, 0.03], - } - ) - row, _tier = choose_best_tuning_row( - df, - target_coverage=0.90, - min_group_coverage_target=0.88, - max_width_budget=0.80, - coverage_guardband=0.0, - ) - assert row["empirical_coverage"] == pytest.approx(0.905) - - -# --------------------------------------------------------------------------- -# apply_group_multipliers -# --------------------------------------------------------------------------- - - -class TestApplyGroupMultipliers: - def test_identity_when_factor_one(self): - y_pred = np.array([0.5, 0.5]) - intervals = np.array([[0.3, 0.7], [0.4, 0.6]]) - groups = np.array(["A", "A"]) - result = apply_group_multipliers(y_pred, intervals, groups, {"A": 1.0}) - np.testing.assert_array_almost_equal(result, intervals) - - def test_widens_intervals_for_group(self): - y_pred = np.array([0.5, 0.5]) - intervals = np.array([[0.4, 0.6], [0.4, 0.6]]) - groups = np.array(["A", "B"]) - result = apply_group_multipliers(y_pred, intervals, groups, {"A": 1.5}) - # Group A should be wider, group B unchanged - width_a = result[0, 1] - result[0, 0] - width_b = result[1, 1] - result[1, 0] - assert width_a > width_b - - def test_clips_to_zero_one(self): - y_pred = np.array([0.05]) - intervals = np.array([[0.0, 0.10]]) - groups = np.array(["A"]) - result = apply_group_multipliers(y_pred, intervals, groups, {"A": 5.0}) - assert result[0, 0] >= 0.0 - assert result[0, 1] <= 1.0 - - -# --------------------------------------------------------------------------- -# enforce_group_coverage_floor -# --------------------------------------------------------------------------- - - -class TestEnforceGroupCoverageFloor: - def test_already_covered_groups_unchanged(self): - rng = np.random.default_rng(42) - n = 100 - y_pred = rng.uniform(0.1, 0.9, n) - y_true = y_pred + rng.normal(0, 0.01, n) # Tight around predictions - intervals = np.column_stack([y_pred - 0.3, y_pred + 0.3]) - groups = np.array(["A"] * n) - - new_intervals, factors, report = enforce_group_coverage_floor( - y_true, y_pred, intervals, groups, target_coverage=0.80 - ) - # Coverage already high → no adjustment needed - assert len(factors) == 0 or all(f == 1.0 for f in factors.values()) - - def test_returns_report_dataframe(self): - n = 50 - y_pred = np.full(n, 0.5) - y_true = np.full(n, 0.5) - intervals = np.column_stack([np.full(n, 0.4), np.full(n, 0.6)]) - groups = np.array(["A"] * 25 + ["B"] * 25) - - _, _, report = enforce_group_coverage_floor( - y_true, y_pred, intervals, groups, target_coverage=0.90 - ) - assert isinstance(report, pd.DataFrame) - assert "group" in report.columns - assert "coverage_before" in report.columns - assert "multiplier" in report.columns - assert len(report) == 2 # A and B - - -# --------------------------------------------------------------------------- -# temporal segment helpers -# --------------------------------------------------------------------------- - - -class TestTemporalSegmentCoverageFloor: - def test_build_group_temporal_segments_formats_keys(self): - groups = pd.Series(["A", "B", "A"]) - dates = pd.Series(pd.to_datetime(["2020-01-15", "2020-07-20", None])) - seg = build_group_temporal_segments(groups, dates, freq="Q") - assert seg.iloc[0].startswith("A|vintage=2020Q1") - assert seg.iloc[1].startswith("B|vintage=2020Q3") - assert seg.iloc[2].startswith("A|vintage=UNKNOWN") - - def test_enforce_segment_coverage_floor_respects_min_support(self): - n = 60 - y_pred = np.full(n, 0.5) - y_true = np.full(n, 0.95) - intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) - segments = np.array(["A|vintage=2020Q1"] * 50 + ["B|vintage=2020Q1"] * 10) - - _new_intervals, factors, report = enforce_segment_coverage_floor( - y_true=y_true, - y_pred=y_pred, - y_intervals=intervals, - segments=segments, - target_coverage=0.90, - min_segment_size=20, - multiplier_grid=(1.0, 1.2), - ) - - assert "A|vintage=2020Q1" in report["segment"].values - assert "B|vintage=2020Q1" in report["segment"].values - assert "A|vintage=2020Q1" in factors - assert "B|vintage=2020Q1" not in factors - - -class TestShrinkGroupMultipliers: - def test_reduces_width_while_preserving_constraints(self): - n_a = 80 - n_b = 80 - y_pred = np.concatenate([np.full(n_a, 0.20), np.full(n_b, 0.80)]) - y_true = np.concatenate([np.full(n_a, 0.14), np.full(n_b, 0.82)]) - base_intervals = np.column_stack( - [ - np.concatenate([np.full(n_a, 0.15), np.full(n_b, 0.75)]), - np.concatenate([np.full(n_a, 0.25), np.full(n_b, 0.85)]), - ] - ) - groups = np.array(["A"] * n_a + ["B"] * n_b) - issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n_a + n_b, freq="D")) - - widened, shrunk_group_factors, shrunk_temporal_factors, report = shrink_group_multipliers( - y_true=y_true, - y_pred=y_pred, - base_intervals=base_intervals, - groups=groups, - issue_dates=issue_dates, - group_factors={"A": 1.20, "B": 1.20}, - temporal_segments=None, - temporal_factors=None, - target_coverage=0.90, - min_group_coverage_target=0.88, - max_monthly_gap_target=0.20, - alpha=0.10, - group_multiplier_grid=(1.0, 1.05, 1.10, 1.15, 1.20), - ) - - baseline_width = float(np.mean((base_intervals[:, 1] - base_intervals[:, 0]) * 1.20)) - shrunk_width = float(np.mean(widened[:, 1] - widened[:, 0])) - - assert isinstance(report, pd.DataFrame) - assert "accepted" in report.columns - assert shrunk_width < baseline_width - assert empirical_interval_coverage(y_true, widened) >= 0.90 - assert min_group_interval_coverage(y_true, widened, groups) >= 0.88 - assert all(f <= 1.20 for f in shrunk_group_factors.values()) - assert shrunk_temporal_factors == {} - - def test_temporal_factors_shrink_when_base_intervals_are_feasible(self): - n = 48 - y_pred = np.full(n, 0.50) - y_true = np.full(n, 0.50) - base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) - groups = np.array(["A"] * n) - temporal_segments = np.array(["early"] * (n // 2) + ["late"] * (n // 2)) - issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) - - widened, group_factors, temporal_factors, report = shrink_group_multipliers( - y_true=y_true, - y_pred=y_pred, - base_intervals=base_intervals, - groups=groups, - issue_dates=issue_dates, - group_factors=None, - temporal_segments=temporal_segments, - temporal_factors={"early": 1.20, "late": 1.20}, - target_coverage=0.90, - min_group_coverage_target=0.88, - temporal_multiplier_grid=(1.0, 1.20), - ) - - accepted_temporal = report[ - (report["stage"] == "accepted") & (report["factor_scope"] == "temporal") - ] - - assert np.allclose(widened, base_intervals) - assert group_factors == {} - assert temporal_factors == {} - assert set(accepted_temporal["factor_key"]) == {"early", "late"} - - def test_returns_initial_infeasible_report_without_shrinking(self): - n = 20 - y_pred = np.full(n, 0.50) - y_true = np.full(n, 0.95) - base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) - groups = np.array(["A"] * (n // 2) + ["B"] * (n // 2)) - issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) - - _widened, group_factors, temporal_factors, report = shrink_group_multipliers( - y_true=y_true, - y_pred=y_pred, - base_intervals=base_intervals, - groups=groups, - issue_dates=issue_dates, - group_factors={"A": 1.20}, - temporal_segments=None, - temporal_factors=None, - target_coverage=0.90, - min_group_coverage_target=0.88, - group_multiplier_grid=(1.0, 1.20), - ) - - assert report["stage"].tolist() == ["initial", "initial_infeasible"] - assert report["accepted"].tolist() == [True, False] - assert group_factors == {"A": 1.20} - assert temporal_factors == {} - - -# --------------------------------------------------------------------------- -# to_python_scalar -# --------------------------------------------------------------------------- - - -class TestToPythonScalar: - def test_numpy_float(self): - result = to_python_scalar(np.float64(3.14)) - assert isinstance(result, float) - assert result == pytest.approx(3.14) - - def test_numpy_int(self): - result = to_python_scalar(np.int32(42)) - assert isinstance(result, int) - - def test_python_native_passthrough(self): - assert to_python_scalar("hello") == "hello" - assert to_python_scalar(42) == 42 diff --git a/tests/test_models/test_pd_contract.py b/tests/test_models/test_pd_contract.py deleted file mode 100644 index 07fdb74..0000000 --- a/tests/test_models/test_pd_contract.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Tests for src/models/pd_contract.py. - -Covers path resolution, contract serialization, feature validation, -and model feature inference. -""" - -from __future__ import annotations - -from pathlib import Path -from unittest.mock import MagicMock - -import pandas as pd -import pytest - -from src.models.pd_contract import ( - build_contract_payload, - infer_model_feature_contract, - load_contract, - resolve_calibrator_path, - resolve_model_path, - save_contract, - validate_features_in_splits, -) - -# --------------------------------------------------------------------------- -# resolve_model_path / resolve_calibrator_path -# --------------------------------------------------------------------------- - - -class TestPathResolution: - def test_resolve_model_path_prefers_upstream_search_pd_candidate(self, tmp_path, monkeypatch): - upstream_model = tmp_path / "models" / "search_pd" / "run-123" / "pd_candidate_model.cbm" - upstream_model.parent.mkdir(parents=True) - upstream_model.touch() - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("UPSTREAM_CANONICAL_RUN_TAG", "run-123") - assert resolve_model_path().resolve() == upstream_model.resolve() - - def test_resolve_model_path_finds_canonical(self, tmp_path, monkeypatch): - model_file = tmp_path / "models" / "pd_canonical.cbm" - model_file.parent.mkdir(parents=True) - model_file.touch() - monkeypatch.setattr("src.models.pd_contract.CANONICAL_MODEL_PATH", model_file) - assert resolve_model_path().resolve() == model_file.resolve() - - def test_resolve_model_path_fallback(self, tmp_path, monkeypatch): - # Canonical doesn't exist, but fallback does - monkeypatch.setattr("src.models.pd_contract.CANONICAL_MODEL_PATH", tmp_path / "nope.cbm") - fallback = tmp_path / "models" / "pd_catboost.cbm" - fallback.parent.mkdir(parents=True) - fallback.touch() - # Patch the candidates list inside the function - import src.models.pd_contract as mod - - def patched(): - candidates = [tmp_path / "nope.cbm", fallback] - path = next((p for p in candidates if p.exists()), None) - if path is None: - raise FileNotFoundError("No PD model artifact found in models/.") - return path - - monkeypatch.setattr(mod, "resolve_model_path", patched) - assert mod.resolve_model_path() == fallback - - def test_resolve_model_path_raises_when_missing(self, tmp_path, monkeypatch): - import src.models.pd_contract as mod - - def _no_model(): - candidates = [tmp_path / "a.cbm", tmp_path / "b.cbm"] - path = next((p for p in candidates if p.exists()), None) - if path is None: - raise FileNotFoundError("No PD model artifact found in models/.") - return path - - monkeypatch.setattr(mod, "resolve_model_path", _no_model) - with pytest.raises(FileNotFoundError): - mod.resolve_model_path() - - def test_resolve_calibrator_returns_none_when_missing(self, tmp_path, monkeypatch): - import src.models.pd_contract as mod - - def _no_cal(): - candidates = [tmp_path / "a.pkl", tmp_path / "b.pkl"] - return next((p for p in candidates if p.exists()), None) - - monkeypatch.setattr(mod, "resolve_calibrator_path", _no_cal) - result = mod.resolve_calibrator_path() - assert result is None - - def test_resolve_calibrator_prefers_upstream_search_pd_candidate(self, tmp_path, monkeypatch): - upstream_cal = tmp_path / "models" / "search_pd" / "run-123" / "pd_candidate_calibrator.pkl" - upstream_cal.parent.mkdir(parents=True) - upstream_cal.touch() - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("UPSTREAM_CANONICAL_RUN_TAG", "run-123") - assert resolve_calibrator_path().resolve() == upstream_cal.resolve() - - -# --------------------------------------------------------------------------- -# load_contract / save_contract -# --------------------------------------------------------------------------- - - -class TestContractIO: - def test_load_nonexistent_returns_none(self, tmp_path): - result = load_contract(tmp_path / "no_contract.json") - assert result is None - - def test_roundtrip(self, tmp_path): - path = tmp_path / "contract.json" - payload = {"model_path": "models/test.cbm", "feature_names": ["a", "b"]} - save_contract(payload, path) - loaded = load_contract(path) - assert loaded["model_path"] == "models/test.cbm" - assert loaded["feature_names"] == ["a", "b"] - - -# --------------------------------------------------------------------------- -# build_contract_payload -# --------------------------------------------------------------------------- - - -class TestBuildPayload: - def test_required_fields_present(self): - payload = build_contract_payload( - model_path=Path("models/test.cbm"), - calibrator_path=Path("models/cal.pkl"), - feature_names=["f1", "f2", "f3"], - categorical_features=["f1"], - ) - assert payload["model_path"] == "models/test.cbm" - assert payload["calibrator_path"] == "models/cal.pkl" - assert payload["n_features"] == 3 - assert "created_at_utc" in payload - - def test_posix_paths(self): - payload = build_contract_payload( - model_path=Path("models/nested/path.cbm"), - calibrator_path=None, - feature_names=["x"], - categorical_features=[], - ) - # as_posix() always uses forward slashes - assert "/" in payload["model_path"] - assert payload["model_path"] == "models/nested/path.cbm" - assert payload["calibrator_path"] is None - - def test_split_shapes_included(self): - payload = build_contract_payload( - model_path=Path("m.cbm"), - calibrator_path=None, - feature_names=["a"], - categorical_features=[], - split_shapes={"train": (1000, 10), "test": (200, 10)}, - ) - assert payload["split_shapes"]["train"] == [1000, 10] - - -# --------------------------------------------------------------------------- -# infer_model_feature_contract -# --------------------------------------------------------------------------- - - -class TestInferContract: - def test_extracts_features_and_categoricals(self): - mock_model = MagicMock() - mock_model.feature_names_ = ["feat_a", "feat_b", "feat_c"] - mock_model.get_cat_feature_indices.return_value = [1] - - features, categoricals = infer_model_feature_contract(mock_model) - assert features == ["feat_a", "feat_b", "feat_c"] - assert categoricals == ["feat_b"] - - def test_no_categoricals(self): - mock_model = MagicMock() - mock_model.feature_names_ = ["x", "y"] - mock_model.get_cat_feature_indices.return_value = [] - - features, categoricals = infer_model_feature_contract(mock_model) - assert len(features) == 2 - assert len(categoricals) == 0 - - -# --------------------------------------------------------------------------- -# validate_features_in_splits -# --------------------------------------------------------------------------- - - -class TestValidateFeatures: - def test_no_missing_features(self): - df = pd.DataFrame({"a": [1], "b": [2], "c": [3]}) - shapes, missing = validate_features_in_splits(["a", "b"], {"train": df}) - assert missing["train"] == [] - assert shapes["train"] == (1, 3) - - def test_reports_missing_features(self): - df = pd.DataFrame({"a": [1]}) - shapes, missing = validate_features_in_splits(["a", "b", "c"], {"test": df}) - assert "b" in missing["test"] - assert "c" in missing["test"] diff --git a/tests/test_models/test_pd_model.py b/tests/test_models/test_pd_model.py deleted file mode 100644 index e051842..0000000 --- a/tests/test_models/test_pd_model.py +++ /dev/null @@ -1,359 +0,0 @@ -"""Unit tests for PD model training and calibration.""" - -import joblib -import numpy as np -import optuna -import pandas as pd -import pytest -import yaml -from sklearn.datasets import make_classification - -from src.models.calibration import ( - calibrate_isotonic, - evaluate_calibration, - expected_calibration_error, -) -from src.models.optuna_tuning import ( - SEARCH_SPACE_VERSION, - _build_optuna_sampler_pruner, - resolve_optuna_study_name, -) -from src.models.pd_model import ( - get_available_features, - resolve_feature_sets, - temporal_train_val_split, - train_baseline, - train_catboost_default, - train_catboost_tuned_optuna, -) - - -@pytest.fixture -def binary_dataset(): - """Create a synthetic binary classification dataset.""" - X, y = make_classification( - n_samples=500, - n_features=5, - n_informative=3, - n_redundant=1, - random_state=42, - flip_y=0.15, - ) - cols = ["feat_0", "feat_1", "feat_2", "feat_3", "feat_4"] - X_train = pd.DataFrame(X[:300], columns=cols) - X_test = pd.DataFrame(X[300:], columns=cols) - y_train = pd.Series(y[:300]) - y_test = pd.Series(y[300:]) - return X_train, y_train, X_test, y_test - - -@pytest.fixture -def catboost_dataset(): - """Synthetic dataset with one categorical column for CatBoost tests.""" - rng = np.random.RandomState(7) - n = 700 - x1 = rng.normal(0, 1, n) - x2 = rng.normal(0, 1, n) - bucket = np.where(x1 + x2 > 0.5, "high", np.where(x1 + x2 < -0.5, "low", "mid")) - signal = 1.2 * x1 - 0.7 * x2 + (bucket == "high") * 0.8 + (bucket == "low") * (-0.5) - y = (signal + rng.normal(0, 0.8, n) > 0).astype(int) - - df = pd.DataFrame( - { - "issue_d": pd.date_range("2014-01-01", periods=n, freq="D"), - "x1": x1, - "x2": x2, - "bucket": bucket.astype(str), - "target": y, - } - ) - train_df = df.iloc[:560].copy().reset_index(drop=True) - test_df = df.iloc[560:].copy().reset_index(drop=True) - return train_df, test_df - - -# ── get_available_features ── - - -def test_get_available_features_filters_correctly(): - df = pd.DataFrame({"loan_amnt": [1], "annual_inc": [2], "fake_col": [3]}) - result = get_available_features(df) - assert "loan_amnt" in result - assert "annual_inc" in result - assert "fake_col" not in result - - -def test_build_optuna_sampler_pruner_tpe_median() -> None: - sampler, pruner = _build_optuna_sampler_pruner( - optuna, - sampler="tpe", - pruner="median", - n_startup_trials=3, - multivariate_tpe=False, - group_tpe=False, - constant_liar=False, - pruner_n_startup_trials=2, - pruner_n_warmup_steps=0, - ) - - assert isinstance(sampler, optuna.samplers.TPESampler) - assert isinstance(pruner, optuna.pruners.MedianPruner) - - -def test_build_optuna_sampler_pruner_random_no_pruning() -> None: - sampler, pruner = _build_optuna_sampler_pruner( - optuna, - sampler="random", - pruner="none", - n_startup_trials=40, - multivariate_tpe=False, - group_tpe=True, - constant_liar=False, - pruner_n_startup_trials=20, - pruner_n_warmup_steps=50, - ) - - assert isinstance(sampler, optuna.samplers.RandomSampler) - assert isinstance(pruner, optuna.pruners.NopPruner) - - -def test_get_available_features_empty_df(): - df = pd.DataFrame({"unrelated": [1]}) - result = get_available_features(df) - assert result == [] - - -def test_resolve_feature_sets_prefers_yaml_companion(tmp_path): - df = pd.DataFrame( - { - "yaml_feature": [1.0, 2.0], - "pickle_feature": [3.0, 4.0], - "grade": ["A", "B"], - } - ) - pkl = tmp_path / "feature_config.pkl" - yml = tmp_path / "feature_config.yml" - joblib.dump( - { - "CATBOOST_FEATURES": ["pickle_feature"], - "CATEGORICAL_FEATURES": [], - "LOGREG_FEATURES": ["pickle_feature"], - }, - pkl, - ) - yml.write_text( - yaml.safe_dump( - { - "CATBOOST_FEATURES": ["yaml_feature", "grade"], - "CATEGORICAL_FEATURES": ["grade"], - "LOGREG_FEATURES": ["yaml_feature"], - } - ), - encoding="utf-8", - ) - - result = resolve_feature_sets( - df, - feature_source="feature_config", - feature_config_path=pkl, - ) - - assert result["feature_source"] == "feature_config" - assert result["catboost_features"] == ["yaml_feature", "grade"] - assert result["categorical_features"] == ["grade"] - assert result["logreg_features"] == ["yaml_feature"] - - -# ── train_baseline ── - - -def test_baseline_returns_model_and_metrics(binary_dataset): - X_train, y_train, X_test, y_test = binary_dataset - model, metrics = train_baseline(X_train, y_train, X_test, y_test) - assert hasattr(model, "predict_proba") - assert "auc_roc" in metrics - assert metrics["model_type"] == "logistic_regression" - - -def test_baseline_auc_above_random(binary_dataset): - X_train, y_train, X_test, y_test = binary_dataset - _, metrics = train_baseline(X_train, y_train, X_test, y_test) - assert metrics["auc_roc"] > 0.5, "AUC should be better than random" - - -def test_baseline_probabilities_bounded(binary_dataset): - X_train, y_train, X_test, y_test = binary_dataset - model, _ = train_baseline(X_train, y_train, X_test, y_test) - probs = model.predict_proba(X_test)[:, 1] - assert np.all(probs >= 0), "Probabilities must be >= 0" - assert np.all(probs <= 1), "Probabilities must be <= 1" - assert not np.any(np.isnan(probs)), "No NaN probabilities allowed" - - -def test_temporal_train_val_split_keeps_order(catboost_dataset): - train_df, _ = catboost_dataset - fit_df, val_df = temporal_train_val_split(train_df, val_fraction=0.2, date_col="issue_d") - assert not fit_df.empty - assert not val_df.empty - assert fit_df["issue_d"].max() <= val_df["issue_d"].min() - - -def test_catboost_tuned_and_default_predictions_differ(catboost_dataset): - train_df, test_df = catboost_dataset - fit_df, val_df = temporal_train_val_split(train_df, val_fraction=0.2, date_col="issue_d") - - X_fit = fit_df[["x1", "x2", "bucket"]].copy() - y_fit = fit_df["target"].astype(int) - X_val = val_df[["x1", "x2", "bucket"]].copy() - y_val = val_df["target"].astype(int) - X_test = test_df[["x1", "x2", "bucket"]].copy() - y_test = test_df["target"].astype(int) - - cb_default, _ = train_catboost_default( - X_fit, - y_fit, - X_val, - y_val, - X_test=X_test, - y_test=y_test, - cat_features=["bucket"], - params={"iterations": 120, "early_stopping_rounds": 25, "verbose": 0}, - ) - cb_tuned, tuned_metrics = train_catboost_tuned_optuna( - X_fit, - y_fit, - X_val, - y_val, - X_test=X_test, - y_test=y_test, - cat_features=["bucket"], - base_params={"iterations": 120, "early_stopping_rounds": 25}, - n_trials=5, - sampler="tpe", - pruner="median", - timeout_minutes=0, - multivariate_tpe=False, - group_tpe=False, - warn_independent_sampling=False, - use_pruning_callback=False, - ) - - y_default = cb_default.predict_proba(X_test)[:, 1] - y_tuned = cb_tuned.predict_proba(X_test)[:, 1] - - assert tuned_metrics["hpo_trials_executed"] >= 1 - assert tuned_metrics["validation_auc"] >= 0.5 - assert not np.allclose(y_default, y_tuned), ( - "Tuned and default CatBoost should produce different predictions when HPO is active." - ) - - -def test_resolve_optuna_study_name_appends_search_space_suffix_once(): - resolved = resolve_optuna_study_name("pd_catboost_optuna_temporal") - assert resolved.endswith(f"__{SEARCH_SPACE_VERSION}") - assert resolve_optuna_study_name(resolved) == resolved - - -def test_local_refine_best_params_are_materialized_for_catboost(): - df = pd.DataFrame( - { - "issue_d": pd.date_range("2019-01-01", periods=120, freq="D"), - "x1": np.linspace(0.0, 1.0, 120), - "x2": np.linspace(1.0, 0.0, 120), - "bucket": np.where(np.arange(120) % 2 == 0, "A", "B"), - "target": (np.arange(120) % 3 == 0).astype(int), - } - ) - train_df = df.iloc[:80].copy() - val_df = df.iloc[80:100].copy() - test_df = df.iloc[100:].copy() - - X_fit = train_df[["x1", "x2", "bucket"]].copy() - y_fit = train_df["target"].astype(int) - X_val = val_df[["x1", "x2", "bucket"]].copy() - y_val = val_df["target"].astype(int) - X_test = test_df[["x1", "x2", "bucket"]].copy() - y_test = test_df["target"].astype(int) - - _, tuned_metrics = train_catboost_tuned_optuna( - X_fit, - y_fit, - X_val, - y_val, - X_test=X_test, - y_test=y_test, - cat_features=["bucket"], - base_params={"iterations": 60, "early_stopping_rounds": 15}, - n_trials=2, - sampler="tpe", - pruner="median", - timeout_minutes=0, - multivariate_tpe=False, - group_tpe=False, - warn_independent_sampling=False, - use_pruning_callback=False, - search_space_mode="local_refine", - local_refine_space={ - "enqueue_base_trial": False, - "iterations": {"choices": [60]}, - "learning_rate": {"choices": [0.05]}, - "depth": {"choices": [4]}, - "l2_leaf_reg": {"choices": [3.0]}, - "min_data_in_leaf": {"choices": [20]}, - "random_strength": {"choices": [1e-6]}, - "border_count": {"choices": [64]}, - "subsample": {"choices": [0.8]}, - "leaf_estimation_iterations": {"choices": [2]}, - "feature_weights": {"x1": [1.2]}, - "first_feature_use_penalties": {"x2": [0.5]}, - "penalties_coefficient": [1.25], - }, - ) - - resolved = tuned_metrics["best_params_resolved"] - assert "feature_weight__x1" not in resolved - assert "first_use_penalty__x2" not in resolved - assert resolved["feature_weights"]["x1"] == pytest.approx(1.2) - assert resolved["first_feature_use_penalties"]["x2"] == pytest.approx(0.5) - - -# ── Calibration ── - - -def test_expected_calibration_error_perfect(): - """Perfect calibration should have ECE ~ 0.""" - y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) - y_prob = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]) - ece = expected_calibration_error(y_true, y_prob, n_bins=5) - assert ece < 0.01 - - -def test_expected_calibration_error_bounded(): - """ECE should be between 0 and 1.""" - rng = np.random.RandomState(42) - y_true = rng.randint(0, 2, size=200) - y_prob = rng.random(200) - ece = expected_calibration_error(y_true, y_prob) - assert 0 <= ece <= 1 - - -def test_calibrate_isotonic_output_bounded(): - """Isotonic calibrator should produce probabilities in [0, 1].""" - rng = np.random.RandomState(42) - y_cal = rng.randint(0, 2, size=200).astype(float) - proba_cal = rng.random(200) - iso = calibrate_isotonic(y_cal, proba_cal) - calibrated = iso.predict(np.linspace(0, 1, 50)) - assert np.all(calibrated >= 0) - assert np.all(calibrated <= 1) - - -def test_evaluate_calibration_returns_dict(binary_dataset): - X_train, y_train, X_test, y_test = binary_dataset - model, _ = train_baseline(X_train, y_train, X_test, y_test) - probs = model.predict_proba(X_test)[:, 1] - result = evaluate_calibration(y_test.values, probs, name="test") - assert "ece" in result - assert "brier_score" in result - assert result["ece"] >= 0 - assert 0 <= result["brier_score"] <= 1 diff --git a/tests/test_models/test_venn_abers_wrapper.py b/tests/test_models/test_venn_abers_wrapper.py deleted file mode 100644 index 738e96c..0000000 --- a/tests/test_models/test_venn_abers_wrapper.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Smoke tests for :class:`src.models.venn_abers.VennAbersScoreCalibrator`. - -These do not re-train the champion calibrator; they only verify the wrapper -contract (1-D input, ordered bounds, point estimate within [0, 1]) on a small -synthetic dataset. -""" - -from __future__ import annotations - -import numpy as np -import pytest - -from src.models.venn_abers import VennAbersScoreCalibrator - - -@pytest.fixture -def synthetic_scores() -> tuple[np.ndarray, np.ndarray]: - rng = np.random.default_rng(42) - n = 200 - # Linear logit, well-separated classes - y = rng.integers(0, 2, size=n) - base = np.where(y == 1, 0.7, 0.2) - noise = rng.normal(0.0, 0.05, size=n) - scores = np.clip(base + noise, 0.0, 1.0) - return scores, y - - -def test_fit_returns_self(synthetic_scores: tuple[np.ndarray, np.ndarray]) -> None: - scores, y = synthetic_scores - cal = VennAbersScoreCalibrator() - out = cal.fit(scores, y) - assert out is cal - - -def test_predict_before_fit_raises() -> None: - cal = VennAbersScoreCalibrator() - with pytest.raises(RuntimeError, match="not fitted"): - cal.predict(np.array([0.1, 0.5, 0.9])) - - -def test_predict_returns_unit_interval( - synthetic_scores: tuple[np.ndarray, np.ndarray], -) -> None: - scores, y = synthetic_scores - cal = VennAbersScoreCalibrator().fit(scores, y) - out = cal.predict(scores) - assert out.shape == scores.shape - assert np.all((out >= 0.0) & (out <= 1.0)) - - -def test_predict_proba_columns_sum_to_one( - synthetic_scores: tuple[np.ndarray, np.ndarray], -) -> None: - scores, y = synthetic_scores - cal = VennAbersScoreCalibrator().fit(scores, y) - proba = cal.predict_proba(scores) - assert proba.shape == (scores.shape[0], 2) - assert np.allclose(proba.sum(axis=1), 1.0, atol=1e-6) - - -def test_predict_intervals_ordered_low_le_high( - synthetic_scores: tuple[np.ndarray, np.ndarray], -) -> None: - scores, y = synthetic_scores - cal = VennAbersScoreCalibrator().fit(scores, y) - low, high = cal.predict_intervals(scores) - assert low.shape == high.shape == scores.shape - assert np.all(low <= high + 1e-12) - assert np.all((low >= 0.0) & (high <= 1.0)) - - -def test_one_dimensional_input_accepted() -> None: - """The wrapper exists primarily to accept 1-D score vectors directly.""" - rng = np.random.default_rng(7) - scores_1d = rng.uniform(0.0, 1.0, size=64) - y = (scores_1d > 0.5).astype(int) - cal = VennAbersScoreCalibrator().fit(scores_1d, y) - out = cal.predict(scores_1d) - assert out.ndim == 1 - assert out.shape == scores_1d.shape diff --git a/tests/test_operational_guardrails.py b/tests/test_operational_guardrails.py new file mode 100644 index 0000000..aa62a74 --- /dev/null +++ b/tests/test_operational_guardrails.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + + +def _text(path: str) -> str: + return Path(path).read_text(encoding="utf-8") + + +def test_one_shot_alias_is_read_only_and_current_only() -> None: + justfile = _text("justfile") + + assert "all: submission-check" in justfile + assert "submission-check: ijds-active-check drift-gate" in justfile + assert "run_crpto_pipeline.py" not in justfile + assert "dvc repro" not in justfile + + +def test_active_drift_gate_is_read_only_and_claim_bound() -> None: + justfile = _text("justfile") + + assert "drift-gate: publication-integrity" in justfile + assert "tests/test_models/test_binary_conformal_guardrail.py" in justfile + assert "tests/test_ijds_active_claim_sync.py" in justfile + assert "CRPTO_RUN_CHAMPION_DRIFT" not in justfile + + +def test_compatibility_surfaces_are_not_active_recipes() -> None: + justfile = _text("justfile").lower() + + for retired_recipe in ("book", "dbt", "notebook", "scripts/search", "dvc repro"): + assert retired_recipe not in justfile + + +def test_publication_contract_names_every_executable_protocol() -> None: + config = yaml.safe_load(_text("configs/crpto_publication_targets.yaml")) + surface = config["active_scientific_contract"]["active_code_surface"] + declared = { + *surface["paper_pipeline"], + *surface["protocol_entrypoints"], + *surface["support_tools"], + } + actual_experiments = { + path.as_posix() + for path in Path("scripts/experiments").glob("*.py") + if path.name != "__init__.py" + } + + assert actual_experiments == set(surface["protocol_entrypoints"]) + assert all(Path(path).is_file() for path in declared) + + +def test_extra_scripts_are_only_sealed_path_bound_compatibility() -> None: + config = yaml.safe_load(_text("configs/crpto_publication_targets.yaml")) + surface = config["active_scientific_contract"]["active_code_surface"] + active = { + *surface["paper_pipeline"], + *surface["protocol_entrypoints"], + *surface["support_tools"], + } + dvc = yaml.safe_load(_text("dvc.yaml")) + path_bound = { + item + for stage in dvc["stages"].values() + for item in stage.get("deps", []) + if isinstance(item, str) and item.startswith("scripts/") and item.endswith(".py") + } + manifest = json.loads(_text("EXTRACTION_MANIFEST.json")) + path_bound.update( + path + for path in manifest["critical_hashes"] + if path.startswith("scripts/") and path.endswith(".py") + ) + actual = { + path.as_posix() for path in Path("scripts").rglob("*.py") if path.name != "__init__.py" + } + + assert actual == active | path_bound + + +def test_manual_full_workflow_runs_the_collected_suite() -> None: + workflow = _text(".github/workflows/tests-full.yml") + assert "run: uv run pytest -q" in workflow + assert "run: just drift-gate" in workflow + + +def test_type_gates_cover_product_and_test_code() -> None: + justfile = _text("justfile") + pyproject = _text("pyproject.toml") + + assert "uv run mypy src scripts tests" in justfile + assert 'files = ["src", "scripts", "tests"]' in pyproject + + +def test_paper_owns_its_bibliography_and_citation_style() -> None: + body = _text("paper/CRPTO_ijds.qmd") + supplement = _text("paper/supplement_ijds.qmd") + template = _text("paper/submission/informs-pandoc-template.tex") + + assert "bibliography: references.bib" in body + assert "csl: apa.csl" in body + assert "bibliography: references.bib" in supplement + assert "csl: apa.csl" in supplement + assert r"\bibliography{../references}" in template diff --git a/tests/test_optimization/test_certificate_semantics.py b/tests/test_optimization/test_certificate_semantics.py deleted file mode 100644 index 3b17f8e..0000000 --- a/tests/test_optimization/test_certificate_semantics.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import numpy as np -import pandas as pd -import pytest -import yaml - -from src.optimization.certificate_semantics import ( - IJDS_DECLARED_ALPHA_GRID, - IJDS_DECLARED_ALPHA_GRID_CSV, - add_policy_aware_bound_columns, - compute_funded_certificate_metrics, -) - -ROOT = Path(__file__).resolve().parents[2] - - -def test_ijds_alpha_grid_matches_profile_and_claim_registry() -> None: - profile_path = ROOT / "configs" / "profiles" / "search_portfolio_pool93_stage1_claim_26_06.yaml" - profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) - claims = (ROOT / "docs" / "research" / "active_claims_2026-07-04.md").read_text( - encoding="utf-8" - ) - claim_grid = ", ".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) - - assert profile["grids"]["alpha_grid"] == IJDS_DECLARED_ALPHA_GRID_CSV - assert f"`A = {{{claim_grid}}}`" in claims - - -def test_linear_blend_certificate_decomposition() -> None: - weights = np.array([0.25, 0.75]) - point = np.array([0.10, 0.20]) - high = np.array([0.50, 0.60]) - gamma = 0.50 - effective = point + gamma * (high - point) - - metrics = compute_funded_certificate_metrics( - weights, - outcomes=np.array([0.0, 1.0]), - pd_point=point, - pd_high=high, - pd_effective=effective, - alpha=0.01, - risk_tolerance=float(weights @ effective), - ) - - assert metrics.gamma_cp == pytest.approx(0.40) - assert metrics.gamma_internalized == pytest.approx(gamma * metrics.gamma_cp) - assert metrics.gamma_residual == pytest.approx((1.0 - gamma) * metrics.gamma_cp) - assert metrics.endpoint_budget == pytest.approx(float(weights @ high)) - assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) - assert metrics.markov_loss_threshold == pytest.approx(metrics.endpoint_budget + 0.10) - assert metrics.markov_loss_cap == pytest.approx(metrics.markov_loss_threshold) - - -def test_tail_policy_uses_actual_residual_not_linear_blend_shortcut() -> None: - weights = np.array([0.50, 0.50]) - point = np.array([0.10, 0.10]) - high = np.array([0.50, 0.90]) - gamma = 0.50 - effective = np.array([0.10, 0.50]) - - metrics = compute_funded_certificate_metrics( - weights, - outcomes=np.array([0.0, 1.0]), - pd_point=point, - pd_high=high, - pd_effective=effective, - alpha=0.01, - risk_tolerance=float(weights @ effective), - ) - - linear_shortcut = (1.0 - gamma) * metrics.gamma_cp - assert metrics.gamma_cp == pytest.approx(0.60) - assert metrics.gamma_internalized == pytest.approx(0.20) - assert metrics.gamma_residual == pytest.approx(0.40) - assert linear_shortcut == pytest.approx(0.30) - assert metrics.gamma_residual > linear_shortcut - assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) - - -def test_policy_cap_slack_is_included_in_endpoint_upper() -> None: - metrics = compute_funded_certificate_metrics( - weights=np.array([1.0]), - outcomes=np.array([1.0]), - pd_point=np.array([0.10]), - pd_high=np.array([0.60]), - pd_effective=np.array([0.40]), - alpha=0.04, - risk_tolerance=0.35, - pd_cap_slack=0.05, - ) - - assert metrics.endpoint_budget == pytest.approx(0.60) - assert metrics.gamma_residual == pytest.approx(0.20) - assert metrics.endpoint_budget_upper == pytest.approx(0.60) - assert metrics.markov_loss_cap == pytest.approx(0.80) - assert metrics.effective_constraint_excess == pytest.approx(0.0) - - -@pytest.mark.parametrize( - ("weights", "match"), - [ - (np.array([0.25, 0.25]), "sum to one"), - (np.array([1.1, -0.1]), "nonnegative"), - ], -) -def test_certificate_rejects_invalid_weights(weights: np.ndarray, match: str) -> None: - with pytest.raises(ValueError, match=match): - compute_funded_certificate_metrics( - weights=weights, - outcomes=np.array([0.0, 1.0]), - pd_point=np.array([0.10, 0.20]), - pd_high=np.array([0.30, 0.40]), - pd_effective=np.array([0.20, 0.30]), - alpha=0.01, - risk_tolerance=0.30, - ) - - -def test_bound_frame_rehydrates_tail_policy_without_linear_shortcut() -> None: - frame = pd.DataFrame( - { - "alpha": [0.01], - "risk_tolerance": [0.20], - "gamma": [0.50], - "gamma_cp": [0.60], - "weighted_pd_point": [0.10], - "weighted_pd_constraint_used": [0.20], - "weighted_pd_high": [0.60], - "pd_cap_slack": [0.0], - } - ) - - result = add_policy_aware_bound_columns(frame).iloc[0] - - assert result["gamma_internalized"] == pytest.approx(0.10) - assert result["gamma_residual"] == pytest.approx(0.40) - assert result["endpoint_budget"] == pytest.approx(0.60) - assert result["endpoint_budget_upper"] == pytest.approx(0.60) - assert result["markov_loss_threshold"] == pytest.approx(0.70) - assert result["markov_loss_cap"] == pytest.approx(0.70) - assert pytest.approx(0.50) == 0.20 + (1.0 - 0.50) * 0.60 - - -def test_bound_frame_separates_exact_threshold_from_slack_upper() -> None: - frame = pd.DataFrame( - { - "alpha": [0.04], - "tau": [0.50], - "weighted_pd_constraint_used": [0.40], - "weighted_pd_high": [0.60], - } - ) - - result = add_policy_aware_bound_columns(frame).iloc[0] - - assert result["effective_constraint_slack"] == pytest.approx(0.10) - assert result["endpoint_budget"] == pytest.approx(0.60) - assert result["endpoint_budget_upper"] == pytest.approx(0.70) - assert result["markov_loss_threshold"] == pytest.approx(0.80) - assert result["markov_loss_cap"] == pytest.approx(0.90) diff --git a/tests/test_optimization/test_input_alignment.py b/tests/test_optimization/test_input_alignment.py deleted file mode 100644 index 5616148..0000000 --- a/tests/test_optimization/test_input_alignment.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from src.optimization.input_alignment import align_candidate_intervals - - -def test_id_alignment_preserves_left_order_and_interval_payload() -> None: - candidates = pd.DataFrame( - { - "id": ["b", "a", "c"], - "grade": ["B", "A", "C"], - "y_pred": [9.0, 8.0, 7.0], - } - ) - intervals = pd.DataFrame( - { - "id": ["a", "b", "c"], - "grade": ["score_q01", "score_q02", "score_q03"], - "y_pred": [0.1, 0.2, 0.3], - } - ) - - aligned = align_candidate_intervals( - candidates, - intervals, - max_candidates=None, - random_state=42, - ) - - assert aligned.mode == "id" - assert aligned.available_rows == 3 - assert aligned.selected_rows == 3 - assert aligned.candidates["id"].tolist() == ["b", "a", "c"] - assert aligned.candidates["grade"].tolist() == ["B", "A", "C"] - assert aligned.intervals["id"].tolist() == ["b", "a", "c"] - assert aligned.intervals["grade"].tolist() == ["score_q02", "score_q01", "score_q03"] - np.testing.assert_allclose(aligned.intervals["y_pred"], [0.2, 0.1, 0.3]) - - -def test_id_alignment_sampling_is_sorted_and_reproducible() -> None: - candidates = pd.DataFrame({"id": np.arange(20), "row": np.arange(20)}) - intervals = pd.DataFrame( - { - "id": np.arange(19, -1, -1), - "y_pred": np.arange(19, -1, -1) / 100.0, - } - ) - expected_positions = np.sort(np.random.default_rng(17).choice(20, size=6, replace=False)) - - first = align_candidate_intervals( - candidates, - intervals, - max_candidates=6, - random_state=17, - ) - second = align_candidate_intervals( - candidates, - intervals, - max_candidates=6, - random_state=17, - ) - - assert first.candidates["row"].tolist() == expected_positions.tolist() - pd.testing.assert_frame_equal(first.candidates, second.candidates) - pd.testing.assert_frame_equal(first.intervals, second.intervals) - - -def test_row_number_alignment_reorders_interval_rows_without_losing_source_columns() -> None: - candidates = pd.DataFrame({"loan": ["a", "b", "c"], "grade": ["A", "B", "C"]}) - intervals = pd.DataFrame( - { - "_row_number": [2, 0, 1], - "grade": ["q3", "q1", "q2"], - "y_pred": [0.3, 0.1, 0.2], - } - ) - - aligned = align_candidate_intervals( - candidates, - intervals, - max_candidates=0, - random_state=42, - ) - - assert aligned.mode == "row_number" - assert aligned.candidates["loan"].tolist() == ["a", "b", "c"] - assert aligned.intervals["_row_number"].tolist() == [0, 1, 2] - assert aligned.intervals["grade"].tolist() == ["q1", "q2", "q3"] - - -def test_positional_fallback_samples_from_full_alignable_universe() -> None: - candidates = pd.DataFrame({"row": np.arange(10)}) - intervals = pd.DataFrame({"y_pred": np.arange(10) / 100.0}) - expected_positions = np.sort(np.random.default_rng(7).choice(10, size=4, replace=False)) - - aligned = align_candidate_intervals( - candidates, - intervals, - max_candidates=4, - random_state=7, - ) - - assert aligned.mode == "position" - assert aligned.available_rows == 10 - assert aligned.candidates["row"].tolist() == expected_positions.tolist() - np.testing.assert_allclose( - aligned.intervals["y_pred"], - expected_positions / 100.0, - ) - assert max(expected_positions) >= 4 - - -@pytest.mark.parametrize( - ("candidate_ids", "interval_ids", "message"), - [ - (["a", "a"], ["a", "b"], "candidate id alignment key is not unique"), - (["a", "b"], ["a", "a"], "interval id alignment key is not unique"), - (["a", None], ["a", "b"], "candidate id alignment key contains missing"), - (["a", "b"], ["a", None], "interval id alignment key contains missing"), - ], -) -def test_id_alignment_rejects_ambiguous_keys( - candidate_ids: list[str | None], - interval_ids: list[str | None], - message: str, -) -> None: - candidates = pd.DataFrame({"id": candidate_ids}) - intervals = pd.DataFrame({"id": interval_ids, "y_pred": [0.1, 0.2]}) - - with pytest.raises(ValueError, match=message): - align_candidate_intervals( - candidates, - intervals, - max_candidates=None, - random_state=42, - ) - - -def test_id_alignment_rejects_disjoint_universes() -> None: - candidates = pd.DataFrame({"id": ["a", "b"]}) - intervals = pd.DataFrame({"id": ["c", "d"], "y_pred": [0.1, 0.2]}) - - with pytest.raises(ValueError, match="produced zero rows"): - align_candidate_intervals( - candidates, - intervals, - max_candidates=None, - random_state=42, - ) diff --git a/tests/test_optimization/test_policy_evaluation.py b/tests/test_optimization/test_policy_evaluation.py index 45a5866..2a1c7fe 100644 --- a/tests/test_optimization/test_policy_evaluation.py +++ b/tests/test_optimization/test_policy_evaluation.py @@ -107,3 +107,29 @@ def fake_optimize(**kwargs): assert np.allclose(captured["pd_constraint_override"], result.effective_pd) assert captured["uncertainty_aversion"] == 0.05 assert result.objective_risk_mode == "point_pd_plus_aversion" + + +def test_explicit_constraint_score_bypasses_policy_score_construction(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([0.5, 0.75]), + "objective_value": 1.0, + "n_funded": 2, + "total_allocated": 200.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + override = np.array([0.42, 0.37]) + + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + pd_constraint_override=override, + ) + + np.testing.assert_array_equal(result.effective_pd, override) + np.testing.assert_array_equal(captured["pd_constraint_override"], override) diff --git a/tests/test_optimization/test_portfolio_model.py b/tests/test_optimization/test_portfolio_model.py index 53ed6ce..055407c 100644 --- a/tests/test_optimization/test_portfolio_model.py +++ b/tests/test_optimization/test_portfolio_model.py @@ -163,6 +163,30 @@ def test_highspy_matches_sparse_highs_objective_on_toy_lp() -> None: assert native["total_allocated"] == pytest.approx(sparse["total_allocated"], rel=1e-6) +def test_explicit_objective_rate_override_matches_independent_reconciliation() -> None: + loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() + coherent_rates = (1.0 - pd_point) * int_rates - pd_point * lgd + result = optimize_portfolio_allocation( + loans=loans, + pd_point=pd_point, + pd_low=pd_low, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + objective_rate_override=coherent_rates, + total_budget=3500, + max_concentration=0.60, + max_portfolio_pd=0.11, + robust=True, + pd_constraint_override=pd_high, + solver_backend="highspy", + ) + + allocation = np.asarray(result["allocation_vector"], dtype=float) + exposure = allocation * loans["loan_amnt"].to_numpy(dtype=float) + assert result["objective_value"] == pytest.approx(float(exposure @ coherent_rates)) + + def test_highspy_falls_back_to_sparse_highs_when_native_solver_fails( monkeypatch, ) -> None: diff --git a/tests/test_optimization/test_robust_opt.py b/tests/test_optimization/test_robust_opt.py deleted file mode 100644 index fe9f947..0000000 --- a/tests/test_optimization/test_robust_opt.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Property-based tests for ``src.optimization.robust_opt``. - -The module is a thin set of math helpers that translate conformal prediction -intervals into uncertainty sets and scenario-based loss estimates. The -invariants are simple enough that Hypothesis can exhaustively check them -without re-running the frozen champion. -""" - -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest -from hypothesis import HealthCheck, given, settings, strategies as st -from hypothesis.extra.numpy import arrays - -from src.optimization.robust_opt import ( - build_box_uncertainty_set, - scenario_analysis, - worst_case_expected_loss, -) - -SUPPRESS = (HealthCheck.function_scoped_fixture,) - - -@st.composite -def pd_low_high(draw: st.DrawFn) -> tuple[np.ndarray, np.ndarray]: - """Draw (pd_low, pd_high) with pd_low <= pd_high element-wise.""" - n = draw(st.integers(min_value=1, max_value=64)) - low = draw(arrays(np.float64, shape=n, elements=st.floats(0.0, 0.95))) - extra = draw(arrays(np.float64, shape=n, elements=st.floats(0.0, 0.05))) - high = np.clip(low + extra, 0.0, 1.0) - return low, high - - -@st.composite -def pd_triple(draw: st.DrawFn) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Draw (pd_low, pd_point, pd_high) ordered element-wise.""" - n = draw(st.integers(min_value=1, max_value=32)) - low = draw(arrays(np.float64, shape=n, elements=st.floats(0.0, 0.6))) - width = draw(arrays(np.float64, shape=n, elements=st.floats(0.0, 0.3))) - high = np.clip(low + width, 0.0, 1.0) - # pd_point lies somewhere between low and high - alpha = draw(arrays(np.float64, shape=n, elements=st.floats(0.0, 1.0))) - point = low + alpha * (high - low) - return low, point, high - - -# --------------------------------------------------------------------------- -# build_box_uncertainty_set -# --------------------------------------------------------------------------- - - -@given(pair=pd_low_high()) -@settings(max_examples=80, suppress_health_check=SUPPRESS, deadline=None) -def test_box_uncertainty_keys_and_shapes(pair: tuple[np.ndarray, np.ndarray]) -> None: - low, high = pair - out = build_box_uncertainty_set(low, high) - assert set(out.keys()) == {"pd_low", "pd_high", "pd_center", "pd_radius"} - for arr in out.values(): - assert arr.shape == low.shape - - -@given(pair=pd_low_high()) -@settings(max_examples=80, suppress_health_check=SUPPRESS, deadline=None) -def test_box_center_between_low_and_high(pair: tuple[np.ndarray, np.ndarray]) -> None: - low, high = pair - out = build_box_uncertainty_set(low, high) - assert np.all(out["pd_center"] >= low - 1e-12) - assert np.all(out["pd_center"] <= high + 1e-12) - - -@given(pair=pd_low_high()) -@settings(max_examples=80, suppress_health_check=SUPPRESS, deadline=None) -def test_box_radius_non_negative(pair: tuple[np.ndarray, np.ndarray]) -> None: - low, high = pair - out = build_box_uncertainty_set(low, high) - assert np.all(out["pd_radius"] >= -1e-12) - - -def test_box_with_lgd_bounds_includes_lgd_keys() -> None: - pd_low = np.array([0.05, 0.10]) - pd_high = np.array([0.12, 0.20]) - lgd_low = np.array([0.30, 0.35]) - lgd_high = np.array([0.50, 0.55]) - out = build_box_uncertainty_set(pd_low, pd_high, lgd_low=lgd_low, lgd_high=lgd_high) - assert {"lgd_low", "lgd_high", "lgd_center", "lgd_radius"} <= out.keys() - assert np.allclose(out["lgd_center"], (lgd_low + lgd_high) / 2) - assert np.allclose(out["lgd_radius"], (lgd_high - lgd_low) / 2) - - -# --------------------------------------------------------------------------- -# worst_case_expected_loss -# --------------------------------------------------------------------------- - - -@given(pair=pd_low_high(), gamma=st.floats(0.0, 1.0)) -@settings(max_examples=60, suppress_health_check=SUPPRESS, deadline=None) -def test_worst_case_loss_non_negative(pair: tuple[np.ndarray, np.ndarray], gamma: float) -> None: - _, pd_high = pair - n = pd_high.shape[0] - allocation = np.full(n, gamma) - loan_amounts = np.ones(n) * 1000.0 - loss = worst_case_expected_loss(allocation, loan_amounts, pd_high) - assert loss >= -1e-9 - - -@given(pair=pd_low_high()) -@settings(max_examples=40, suppress_health_check=SUPPRESS, deadline=None) -def test_worst_case_loss_scales_with_allocation( - pair: tuple[np.ndarray, np.ndarray], -) -> None: - _, pd_high = pair - n = pd_high.shape[0] - loan_amounts = np.ones(n) * 1000.0 - half = worst_case_expected_loss(np.full(n, 0.5), loan_amounts, pd_high) - full = worst_case_expected_loss(np.full(n, 1.0), loan_amounts, pd_high) - zero = worst_case_expected_loss(np.zeros(n), loan_amounts, pd_high) - assert zero == pytest.approx(0.0, abs=1e-9) - assert half <= full + 1e-9 - if np.any(pd_high > 0): - assert full > zero - - -def test_worst_case_loss_uses_lgd_high_when_provided() -> None: - allocation = np.array([1.0, 1.0]) - loan_amounts = np.array([1000.0, 1000.0]) - pd_high = np.array([0.10, 0.20]) - lgd_high = np.array([0.50, 0.60]) - loss_with = worst_case_expected_loss(allocation, loan_amounts, pd_high, lgd_high=lgd_high) - loss_default = worst_case_expected_loss(allocation, loan_amounts, pd_high) - expected_with = 1000.0 * 0.10 * 0.50 + 1000.0 * 0.20 * 0.60 - expected_default = 1000.0 * 0.10 * 0.45 + 1000.0 * 0.20 * 0.45 - assert loss_with == pytest.approx(expected_with) - assert loss_default == pytest.approx(expected_default) - - -# --------------------------------------------------------------------------- -# scenario_analysis -# --------------------------------------------------------------------------- - - -@given(triple=pd_triple()) -@settings(max_examples=60, suppress_health_check=SUPPRESS, deadline=None) -def test_scenario_ordering_best_le_expected_le_worst( - triple: tuple[np.ndarray, np.ndarray, np.ndarray], -) -> None: - low, point, high = triple - n = low.shape[0] - allocation = np.full(n, 0.5) - loan_amounts = np.ones(n) * 1000.0 - lgd = np.full(n, 0.45) - df = scenario_analysis(allocation, loan_amounts, low, point, high, lgd) - row = df.iloc[0] - assert row["best_case"] <= row["expected"] + 1e-9 - assert row["expected"] <= row["worst_case"] + 1e-9 - assert row["range"] >= -1e-9 - - -def test_scenario_analysis_columns() -> None: - n = 4 - allocation = np.full(n, 1.0) - loan_amounts = np.full(n, 500.0) - lgd = np.full(n, 0.45) - df = scenario_analysis( - allocation, - loan_amounts, - pd_low=np.full(n, 0.05), - pd_point=np.full(n, 0.10), - pd_high=np.full(n, 0.20), - lgd=lgd, - ) - assert isinstance(df, pd.DataFrame) - assert list(df.columns) == ["best_case", "expected", "worst_case", "range"] - assert len(df) == 1 diff --git a/tests/test_optimization/test_tail_satisficing_objective.py b/tests/test_optimization/test_tail_satisficing_objective.py deleted file mode 100644 index ba71408..0000000 --- a/tests/test_optimization/test_tail_satisficing_objective.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import pytest - -from src.optimization.tail_satisficing_objective import ( - SatisficingThreshold, - entropic_oce, - evaluate_satisficing_margins, - funded_loss_rate, - score_tail_satisficing_objective, - weighted_cvar, - weighted_mean, -) - - -def test_weighted_cvar_uses_upper_loss_tail() -> None: - losses = [0.01, 0.02, 0.50] - weights = [0.45, 0.45, 0.10] - - assert weighted_cvar(losses, weights, tail=0.90) == pytest.approx(0.50) - - -def test_entropic_oce_is_at_least_mean_for_positive_theta() -> None: - losses = [-0.08, 0.02, 0.25] - weights = [0.4, 0.4, 0.2] - - assert entropic_oce(losses, weights, theta=5.0) >= weighted_mean(losses, weights) - - -def test_funded_loss_rate_reprices_defaults_and_performing_loans() -> None: - losses = funded_loss_rate([1, 0, 0], [0.10, 0.12, 0.08], lgd=0.45) - - assert losses.tolist() == pytest.approx([0.45, -0.12, -0.08]) - - -def test_satisficing_margins_support_min_max_and_equals() -> None: - margins = evaluate_satisficing_margins( - { - "expected_return": 170_000.0, - "weighted_miscoverage": 0.036, - "exact_pass": True, - }, - ( - SatisficingThreshold("expected_return", "min", 150_000.0), - SatisficingThreshold("weighted_miscoverage", "max", 0.04), - SatisficingThreshold("exact_pass", "equals", True), - ), - ) - - assert [margin.passed for margin in margins] == [True, True, True] - assert min(margin.margin for margin in margins) >= -1e-12 - - -def test_tail_satisficing_score_penalizes_risk_and_shortfall() -> None: - thresholds = ( - SatisficingThreshold("expected_return", "min", 100.0), - SatisficingThreshold("cvar_loss_rate", "max", 0.20), - ) - - safer = score_tail_satisficing_objective( - expected_return=120.0, - loss_rates=[0.01, 0.02, 0.10], - weights=[0.4, 0.4, 0.2], - thresholds=thresholds, - cvar_penalty=1.0, - oce_penalty=1.0, - satisficing_shortfall_penalty=10.0, - risk_scale=100.0, - ) - riskier = score_tail_satisficing_objective( - expected_return=120.0, - loss_rates=[0.01, 0.02, 0.50], - weights=[0.4, 0.4, 0.2], - thresholds=thresholds, - cvar_penalty=1.0, - oce_penalty=1.0, - satisficing_shortfall_penalty=10.0, - risk_scale=100.0, - ) - - assert safer.satisficing_pass - assert not riskier.satisficing_pass - assert safer.objective_value > riskier.objective_value diff --git a/tests/test_pool93_body_claim_sync.py b/tests/test_pool93_body_claim_sync.py deleted file mode 100644 index 50054de..0000000 --- a/tests/test_pool93_body_claim_sync.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Historical integrity checks for the manifest-protected pool93 bundle.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest - -REPO = Path(__file__).resolve().parents[1] -TABLES = REPO / "reports" / "crpto" / "tables" -MANIFEST = REPO / "EXTRACTION_MANIFEST.json" -PROMOTION = REPO / "models" / "final_project_promotion.json" -TERMINAL_GOVERNANCE = ( - REPO - / "models/experiments/champion_reopen" - / "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" - / "portfolio/pool93_ijds_claim_governance.json" -) -CONSOLIDATED_GOVERNANCE = ( - REPO - / "models/experiments/champion_reopen" - / "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" - / "portfolio/pool93_ijds_consolidated_governance.json" -) -POOL93_TABLE_STEMS = ( - "crpto_tableA35_pool93_ijds_frontier", - "crpto_tableA36_pool93_body_funded_grade_audit", - "crpto_tableA37_pool93_body_tail_risk", - "crpto_tableA38_pool93_body_cluster_bound_audit", - "crpto_tableA39_pool93_body_bootstrap_metrics", - "crpto_tableA40_pool93_point_baseline", -) - - -def _load(path: Path) -> dict[str, Any]: - if not path.is_file(): - pytest.skip(f"Historical artifact unavailable: {path}") - return json.loads(path.read_text(encoding="utf-8")) - - -def test_historical_pool93_bundle_remains_internally_coherent() -> None: - consolidated = _load(CONSOLIDATED_GOVERNANCE) - terminal = _load(TERMINAL_GOVERNANCE) - promotion = _load(PROMOTION) - - assert consolidated["counts"]["deduped_semantic_policies"] == 50_010 - assert consolidated["counts"]["eligible_all_alpha_return_floor_policies"] == 27_508 - assert terminal["runtime_status"]["total_checks"] == 296_544 - assert promotion["run_tag"] == "ijds-rebaseline-2026-06-07" - assert terminal["claim_summary"]["declared_return_floor"] == pytest.approx( - round(promotion["final_champion"]["realized_total_return"], 2) - ) - - -def test_historical_pool93_tables_remain_hash_protected() -> None: - manifest = _load(MANIFEST) - hashed = set(manifest["critical_hashes"]) - expected = { - f"reports/crpto/tables/{stem}.{suffix}" - for stem in POOL93_TABLE_STEMS - for suffix in ("csv", "tex") - } - assert expected.issubset(hashed) - assert all((TABLES / path.split("/")[-1]).is_file() for path in expected) diff --git a/tests/test_publication_targets.py b/tests/test_publication_targets.py index 2cb08e8..149124e 100644 --- a/tests/test_publication_targets.py +++ b/tests/test_publication_targets.py @@ -1,113 +1,154 @@ from __future__ import annotations +import json from pathlib import Path import yaml +from src.ijds_audit.publication_sources import load_source_registry -def test_publication_target_config_points_to_existing_sources() -> None: - cfg = yaml.safe_load(Path("configs/crpto_publication_targets.yaml").read_text(encoding="utf-8")) +ROOT = Path(__file__).resolve().parents[1] +REGISTRY_PATH = ROOT / "configs/ijds_active_evidence_sources.yaml" + +def _config() -> dict: + return yaml.safe_load( + Path("configs/crpto_publication_targets.yaml").read_text(encoding="utf-8") + ) + + +def _evidence() -> dict: + return json.loads( + Path("reports/crpto/ijds_binary_geometry_frontier_v4_evidence.json").read_text( + encoding="utf-8" + ) + ) + + +def _registry() -> dict: + return load_source_registry(REGISTRY_PATH, repo_root=ROOT) + + +def test_publication_target_points_to_active_sources() -> None: + cfg = _config() primary = cfg["primary_target"] + active = cfg["active_scientific_contract"] + registry = _registry() + + assert cfg["version"] == str(registry["schema_version"]).rsplit(".", maxsplit=1)[0] + assert cfg["decision_status"] == "prefreeze_active" assert primary["id"] == "informs_ijds" assert cfg["current_decision"]["write_first_for"] == "informs_ijds" assert cfg["current_decision"]["keep_second_ready_for"] == "ejor" - - assert Path(primary["manuscript_source"]).exists() - assert Path(primary["supplement_source"]).exists() - - strategy = Path("docs/research/crpto_publication_strategy_2026-05-12.md") - assert strategy.exists() + for key in ("manuscript_source", "supplement_source", "official_tex_source"): + assert Path(primary[key]).is_file() + for key in ("claim_registry", "source_registry", "evidence_manifest"): + assert Path(active[key]).is_file() def test_publication_target_urls_are_official_https() -> None: - cfg = yaml.safe_load(Path("configs/crpto_publication_targets.yaml").read_text(encoding="utf-8")) - - targets = [cfg["primary_target"], *cfg["secondary_targets"]] - for target in targets: - for url in target["official_urls"].values(): - assert url.startswith("https://"), (target["id"], url) + urls = _config()["primary_target"]["official_urls"] + assert urls + assert all(url.startswith("https://") for url in urls.values()) def test_ijds_sources_are_anonymous_by_default() -> None: - paths = [Path("paper/CRPTO_ijds.qmd"), Path("paper/supplement_ijds.qmd")] - for path in paths: + for path in (Path("paper/CRPTO_ijds.qmd"), Path("paper/supplement_ijds.qmd")): text = path.read_text(encoding="utf-8") assert 'author: "Anonymous"' in text assert "Carlos Alfredo Vergara Rojas" not in text - assert "TODO(manuscript)" not in text - - -def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> None: - cfg = yaml.safe_load(Path("configs/crpto_publication_targets.yaml").read_text(encoding="utf-8")) - boundary = cfg["current_decision"]["p2_p3_boundary"] - pack = cfg["journal_strengthening_pack"] - included = pack["include_in_current_submission"] - backlog = pack["backlog_not_blocking"] - - assert "no longer a blanket exclusion" in boundary - assert "outside the submitted claim" in boundary - assert "not acceptance criteria" in boundary - assert set(included) == { - "regret_auditability_frontier", - "tail_risk_oce_cvar_diagnostic", - "exact_alpha_calibration_selected_policy", - "matched_point_pd_baseline", - "robust_satisficing_margins", - "dependence_aware_bound", - "tail_satisficing_challenger_audit", - "tail_constrained_reoptimization", - "distribution_online_diagnostics", - "multidataset_external_replication", + assert "cavr94@gmail.com" not in text + tex = Path("paper/submission/CRPTO_ijds_submission.tex").read_text(encoding="utf-8") + assert r"\documentclass[ijds,dblanonrev]{informs4}" in tex + assert r"\ACKNOWLEDGMENT" not in tex + assert "Carlos Alfredo Vergara Rojas" not in tex + for path in ( + Path("paper/submission/TITLE_PAGE_DRAFT.md"), + Path("paper/submission/COVER_LETTER_AND_DISCLOSURE.md"), + ): + text = path.read_text(encoding="utf-8") + assert "Carlos Alfredo Vergara Rojas" not in text + assert "cavr94@gmail.com" not in text + + +def test_active_contract_has_one_numeric_source_and_current_lineages() -> None: + active = _config()["active_scientific_contract"] + evidence = _evidence() + registry = _registry() + + assert "headline" not in active + assert active["lineage_and_dvc_authority"] == active["source_registry"] + assert not { + "outcome_free_run_tag", + "run_tag", + "two_ruler_outcome_free_run_tag", + "two_ruler_run_tag", + "credit_control_outcome_free_run_tag", + "credit_control_run_tag", + "dvc_pointers", + }.intersection(active) + lineages = registry["lineages"] + assert lineages["binary_geometry"]["outcome_free"]["run_tag"].endswith("2026-07-12-v1") + assert lineages["binary_geometry"]["evaluation"]["run_tag"].endswith("2026-07-15-v5") + assert lineages["two_ruler"]["outcome_free"]["run_tag"].endswith("2026-07-13-v1c") + assert lineages["two_ruler"]["evaluation"]["run_tag"].endswith("2026-07-15-v5") + assert lineages["credit_controls"]["outcome_free"]["run_tag"].endswith("2026-07-13-v1b") + assert lineages["credit_controls"]["evaluation"]["run_tag"].endswith("2026-07-15-v5") + assert evidence["lineages"] == lineages + assert active["previously_inspected_retrospective_archive"] is True + assert active["archive_is_verified_point_in_time_snapshot"] is False + assert active["confirmatory"] is False + assert active["prospective"] is False + assert active["causal"] is False + assert active["policy_winner_allowed"] is False + + assert evidence["design"]["primary_oot_candidates"] == 376890 + assert evidence["design"]["primary_oot_resolved"] == 364814 + assert evidence["design"]["primary_oot_unresolved"] == 12076 + assert evidence["design"]["archive_is_verified_point_in_time_snapshot"] is False + assert evidence["coverage"]["catboost_bound_max"] < 0.90 + assert evidence["coverage"]["logistic_bound_max"] < 0.90 + assert evidence["credit_risk_controls"]["all_five_all_eight_upper_below_nominal"] is True + assert evidence["decision_challenger"]["counts"]["evaluated_portfolios"] == 6240 + assert evidence["decision_challenger"]["interpretation"]["policy_winner"] is None + assert evidence["portfolio"]["broad_stress_cells"] == 216 + assert evidence["portfolio"]["broad_stress_all_envelopes_cross_zero"] is True + + +def test_active_capsule_paths_exist() -> None: + active = _config()["active_scientific_contract"] + registry = _registry() + evidence = _evidence() + support_artifacts = { + "reports/crpto/ijds_policy_support_tie_evidence.json", + "reports/crpto/tables/crpto_ijds_comparator_support_domain.csv", + "reports/crpto/tables/crpto_ijds_gamma_endpoint_audit.csv", + "reports/crpto/tables/crpto_ijds_policy_family_domain.csv", } - assert included["regret_auditability_frontier"]["status"] == "include_body" - assert included["tail_risk_oce_cvar_diagnostic"]["status"] == "include_supplement" - assert included["exact_alpha_calibration_selected_policy"]["status"] == ( - "include_body_and_supplement" - ) - assert included["matched_point_pd_baseline"]["status"] == ("include_body_and_supplement") - assert included["robust_satisficing_margins"]["status"] == ("include_supplement_or_short_body") - assert included["dependence_aware_bound"]["status"] == "include_theory_appendix_or_caveat" - assert included["tail_satisficing_challenger_audit"]["status"] == "include_supplement" - assert included["tail_constrained_reoptimization"]["status"] == "include_supplement" - assert included["distribution_online_diagnostics"]["status"] == "include_supplement" - assert included["multidataset_external_replication"]["status"] == ( - "include_supplement_or_short_body" - ) - active_artifacts = included["exact_alpha_calibration_selected_policy"]["artifacts"] - assert "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv" in active_artifacts - assert "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv" in ( - active_artifacts - ) - assert "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv" in ( - active_artifacts - ) - for artifact in active_artifacts: - assert Path(artifact).exists(), artifact - multidataset_artifacts = included["multidataset_external_replication"]["artifacts"] - assert "reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv" in ( - multidataset_artifacts - ) - assert "reports/crpto/tables/crpto_tableA33_freddie_segment_sensitivity.csv" in ( - multidataset_artifacts - ) - assert "reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png" in ( - multidataset_artifacts - ) - for artifact in multidataset_artifacts: - assert Path(artifact).exists(), artifact - assert backlog["prospective_multidataset_validation"]["status"] == ( - "future_protocol_not_blocker" - ) - - body = Path("paper/CRPTO_ijds.qmd").read_text(encoding="utf-8") - supplement = Path("paper/supplement_ijds.qmd").read_text(encoding="utf-8") - paper_readme = Path("paper/README.md").read_text(encoding="utf-8") - - for text in (body, supplement, paper_readme): - assert "midpoint" in text.lower() - assert "calibration" in text.lower() - assert "point-PD" in text - - for diagnostic in ("OCE/CVaR", "SPO+", "Prosper"): - assert diagnostic in supplement + expected_artifacts = { + active["evidence_manifest"], + *(descriptor["path"] for descriptor in evidence["paper_artifacts"].values()), + *support_artifacts, + } + assert set(active["required_artifacts"]) == expected_artifacts + for artifact in active["required_artifacts"]: + assert Path(artifact).is_file(), artifact + for pointer in registry["dvc_pointers"]: + assert Path(pointer).is_file(), pointer + + code_surface = active["active_code_surface"] + assert code_surface["historical_execution_in_active_capsule"] is False + for path in code_surface["source_roots"]: + assert Path(path).is_dir(), path + for group in ("paper_pipeline", "protocol_entrypoints", "support_tools"): + for path in code_surface[group]: + assert Path(path).is_file(), path + + +def test_active_capsule_does_not_advertise_retired_result_families() -> None: + config = _config() + assert "historical_boundary" not in config + serialized = json.dumps(config).lower() + assert "pool93" not in serialized + assert "compact-v7" not in serialized + assert "a1--a40" not in serialized diff --git a/tests/test_quarto_book_guardrails.py b/tests/test_quarto_book_guardrails.py deleted file mode 100644 index 10194f1..0000000 --- a/tests/test_quarto_book_guardrails.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import yaml - -LEGACY_BRAND = ["paper " + "estrella", "paper_" + "estrella", "paper-" + "estrella"] -LEGACY_PUBLIC = ["estrella" + "_fig", "paper1" + "_table", "paper1" + "_e2e"] - - -def public_files() -> list[Path]: - roots = [Path("book"), Path("docs"), Path("reports"), Path("configs"), Path("tests")] - files: list[Path] = [Path("README.md")] - ignored_suffixes = { - ".png", - ".jpg", - ".pdf", - ".parquet", - ".pkl", - ".cbm", - ".pyc", - } - for root in roots: - if root.exists(): - files.extend( - p - for p in root.rglob("*") - if p.is_file() - and "__pycache__" not in p.parts - and p.suffix.lower() not in ignored_suffixes - ) - return files - - -def test_quarto_book_lists_existing_chapters() -> None: - cfg = yaml.safe_load(Path("book/_quarto.yml").read_text(encoding="utf-8")) - chapters = list(_flatten_chapters(cfg["book"]["chapters"])) - assert cfg["book"]["title"] == "CRPTO" - for chapter in chapters: - assert (Path("book") / chapter).exists(), chapter - - -def _flatten_chapters(entries: list[str | dict]) -> list[str]: - chapters: list[str] = [] - for entry in entries: - if isinstance(entry, str): - chapters.append(entry) - elif isinstance(entry, dict): - chapters.extend(_flatten_chapters(entry.get("chapters", []))) - return chapters - - -def test_public_branding_is_crpto_only() -> None: - offenders: list[str] = [] - legacy = [token.lower() for token in [*LEGACY_BRAND, *LEGACY_PUBLIC]] - for path in public_files(): - text = path.read_text(encoding="utf-8", errors="ignore").lower() - if any(token in text for token in legacy): - offenders.append(str(path)) - assert not offenders diff --git a/tests/test_scripts/test_benchmark_conformal_variants.py b/tests/test_scripts/test_benchmark_conformal_variants.py deleted file mode 100644 index b3f55e2..0000000 --- a/tests/test_scripts/test_benchmark_conformal_variants.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Tests for scripts/benchmark_conformal_variants.py.""" - -from __future__ import annotations - -from scripts.benchmark_conformal_variants import _build_output_paths, _normalize_search_space - - -def test_build_output_paths_uses_namespaced_shadow_locations() -> None: - paths = _build_output_paths("abc/def") - - assert paths["data_dir"].as_posix().endswith("data/processed/conformal_gap/abc_def") - assert paths["models_dir"].as_posix().endswith("models/conformal_gap/abc_def") - assert ( - paths["selection_status"] - .as_posix() - .endswith("models/conformal_gap/abc_def/conformal_variant_selection_status.json") - ) - - -def test_normalize_search_space_dedupes_and_applies_defaults() -> None: - space = _normalize_search_space( - calibration_size_fractions=(0.25, 1.5, 0.5), - partition_candidates=(" grade ", "grade", ""), - partition_probability_sources=("RAW", "raw"), - n_score_bins_candidates=(0, 10), - fallback_modes=("", "Grade_Then_Global", "grade_then_global"), - score_scale_families=("",), - min_group_sizes=None, - min_group_size_default=500, - ) - - assert space.partition_candidates == ("grade",) - assert space.partition_probability_sources == ("raw",) - assert space.n_score_bins_candidates == (10,) - assert space.fallback_modes == ("grade_then_global",) - assert space.score_scale_families == ("none",) - assert space.min_group_sizes == (500,) - assert space.calibration_size_fractions == (0.25, 0.5) diff --git a/tests/test_scripts/test_benchmark_pd_set_prediction.py b/tests/test_scripts/test_benchmark_pd_set_prediction.py deleted file mode 100644 index 2188404..0000000 --- a/tests/test_scripts/test_benchmark_pd_set_prediction.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import numpy as np - -from scripts.benchmark_pd_set_prediction import ( - _build_output_paths, - _promotion_gate, - _set_benchmark_settings, -) - - -def test_build_output_paths_uses_namespaced_shadow_locations() -> None: - paths = _build_output_paths("set/audit") - - assert ( - paths["cases"] - .as_posix() - .endswith("data/processed/conformal_gap/set_audit/pd_set_prediction_cases.parquet") - ) - assert ( - paths["status"] - .as_posix() - .endswith("models/conformal_gap/set_audit/pd_set_prediction_status.json") - ) - - -def test_set_benchmark_settings_normalizes_inputs_and_fallback() -> None: - settings = _set_benchmark_settings( - alpha=0.1, - method="lac", - methods=("lac", "aps", "lac"), - partitions=("global", "global", "grade"), - partition_probability_source="RAW", - n_score_bins=10, - min_group_size=500, - fallback_mode="score_only", - calibration_size_fractions=(0.25, 1.5, 0.50), - prob_cal_lookup={"raw": np.array([0.1]), "calibrated": np.array([0.2])}, - ) - - assert settings.methods == ("lac", "aps") - assert settings.partitions == ("global", "grade") - assert settings.partition_probability_source == "raw" - assert settings.effective_fallback_mode == "global_only" - assert settings.calibration_size_fractions == (0.25, 0.50) - - -def test_promotion_gate_requires_coverage_grade_a_and_breadth() -> None: - gate = _promotion_gate( - {"set_coverage": 0.90}, - [ - {"slice_value": "A", "singleton_rate": 0.81}, - {"slice_value": "B", "singleton_rate": 0.41}, - {"slice_value": "C", "singleton_rate": 0.42}, - {"slice_value": "D", "singleton_rate": 0.43}, - ], - ) - - assert gate["pass"] is True - assert gate["grade_a_singleton_rate"] == 0.81 - assert gate["grades_with_singleton_above_40pct"] == 4 diff --git a/tests/test_scripts/test_build_bound_tightening_audit.py b/tests/test_scripts/test_build_bound_tightening_audit.py deleted file mode 100644 index 428eb16..0000000 --- a/tests/test_scripts/test_build_bound_tightening_audit.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.build_bound_tightening_audit import ( - FUNDED_LOANS_PATH, - _cluster_assumption_rows, - _threshold_rows, -) - - -def test_bound_tightening_audit_preserves_main_markov_claim() -> None: - import pandas as pd - - funded = pd.read_csv(FUNDED_LOANS_PATH) - table, stats = _threshold_rows(funded) - alpha01 = table[table["alpha"].eq(0.01)] - - markov = alpha01.loc[alpha01["bound"].eq("markov")].iloc[0] - assert markov["threshold_t"] == pytest.approx(0.1) - assert not bool(markov["tighter_than_markov"]) - assert stats["empirical_v"] == pytest.approx(0.028875, abs=1e-12) - assert stats["n_eff"] == pytest.approx(126.066, rel=1e-4) - - -def test_conditional_bounds_show_tightness_but_require_extra_assumptions() -> None: - import pandas as pd - - funded = pd.read_csv(FUNDED_LOANS_PATH) - table, _ = _threshold_rows(funded) - alpha01 = table[table["alpha"].eq(0.01)] - - hoeffding = alpha01.loc[alpha01["bound"].eq("hoeffding")].iloc[0] - bernstein = alpha01[ - alpha01["bound"].eq("bernstein") & alpha01["variance_mode"].eq("strong_individual_validity") - ].iloc[0] - freedman = alpha01[ - alpha01["bound"].eq("freedman_martingale") - & alpha01["variance_mode"].eq("strong_individual_validity") - ].iloc[0] - cantelli = alpha01[ - alpha01["bound"].eq("cantelli_one_sided") - & alpha01["variance_mode"].eq("weak_weighted_validity") - ].iloc[0] - - assert hoeffding["threshold_t"] > 0.1 - assert bernstein["threshold_t"] == pytest.approx(0.069832, rel=1e-4) - assert freedman["threshold_t"] == pytest.approx(bernstein["threshold_t"]) - assert cantelli["threshold_t"] == pytest.approx(0.066125, rel=1e-4) - assert bool(bernstein["tighter_than_markov"]) - assert "chernoff_mgf" not in set(alpha01["bound"]) - assert "chebyshev_two_sided" not in set(alpha01["bound"]) - assert "azuma_hoeffding_martingale" not in set(alpha01["bound"]) - assert "union_markov_45_policy_region" not in set(alpha01["bound"]) - - -def test_assumption_audit_marks_independence_as_unverified() -> None: - import pandas as pd - - funded = pd.read_csv(FUNDED_LOANS_PATH) - _, stats = _threshold_rows(funded) - audit = _cluster_assumption_rows(funded, stats) - - loan_independence = audit.loc[audit["assumption"].eq("loan_independence")].iloc[0] - union_markov = audit.loc[audit["assumption"].eq("post_selection_uniformity")].iloc[0] - martingale = audit.loc[audit["assumption"].eq("sequential_martingale_protocol")].iloc[0] - chernoff = audit.loc[audit["assumption"].eq("chernoff_mgf")].iloc[0] - chebyshev = audit.loc[audit["assumption"].eq("chebyshev_two_sided")].iloc[0] - cluster_rows = audit[audit["assumption"].str.startswith("cluster_independence_")] - - assert loan_independence["status"] == "not_verified" - assert union_markov["status"] == "not_supported_by_markov" - assert martingale["status"] == "not_available" - assert chernoff["status"] == "drop_from_table" - assert chebyshev["status"] == "drop_from_table" - assert set(cluster_rows["status"]) == {"conditional_loose"} diff --git a/tests/test_scripts/test_build_crpto_journal_package.py b/tests/test_scripts/test_build_crpto_journal_package.py deleted file mode 100644 index 88a919d..0000000 --- a/tests/test_scripts/test_build_crpto_journal_package.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -import time -from collections.abc import Iterator -from contextlib import contextmanager -from pathlib import Path - -import pytest - - -def _restore_payload(path: Path, payload: bytes | None) -> None: - for attempt in range(5): - try: - if payload is None: - path.unlink(missing_ok=True) - return - - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.restore.{attempt}.tmp") - tmp.write_bytes(payload) - tmp.replace(path) - return - except OSError: - tmp.unlink(missing_ok=True) if "tmp" in locals() else None - if attempt == 4: - raise - time.sleep(0.2 * (attempt + 1)) - - -@contextmanager -def _preserve_files(paths: list[Path]) -> Iterator[None]: - snapshots = {path: path.read_bytes() if path.exists() else None for path in paths} - try: - yield - finally: - for path, payload in snapshots.items(): - _restore_payload(path, payload) - - -def test_build_crpto_journal_package_script_runs() -> None: - pytest.importorskip("matplotlib") - generated = [ - Path("models/crpto_journal_package_status.json"), - Path("docs/research/crpto_journal_package_2026-05-04.md"), - Path("book/assets/figures/publication/crpto_fig1_journal_pipeline.png"), - Path("book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf"), - Path("book/assets/figures/publication/crpto_fig1_journal_pipeline.svg"), - Path("reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png"), - Path("reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.pdf"), - Path("book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.png"), - Path("book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf"), - Path("reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.png"), - Path("reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf"), - Path("book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.png"), - Path("book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf"), - Path("reports/crpto/figures/crpto_fig14_robust_region_heatmap.png"), - Path("reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf"), - Path("book/assets/figures/publication/crpto_fig14_robust_region_heatmap.png"), - Path("book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf"), - Path("reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png"), - Path("reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf"), - Path("book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.png"), - Path("book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf"), - Path("reports/crpto/figures/crpto_fig20_bound_claim_layers.png"), - Path("reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf"), - Path("book/assets/figures/publication/crpto_fig20_bound_claim_layers.png"), - Path("book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf"), - Path("reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv"), - Path("reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.tex"), - ] - with _preserve_files(generated): - subprocess.run([sys.executable, "scripts/build_crpto_journal_package.py"], check=True) - assert Path("models/crpto_journal_package_status.json").exists() - assert Path("reports/crpto/tables/crpto_tableA18_robust_region_policy_family.csv").exists() - assert Path("reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv").exists() diff --git a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py deleted file mode 100644 index 9443fa4..0000000 --- a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from scripts.build_ijds_calibration_selected_evidence import ( - build_baseline_table, - build_bootstrap_table, - build_grade_table, -) - - -def _allocations() -> pd.DataFrame: - return pd.DataFrame( - { - "role": ["calibration_selected", "calibration_selected", "point_pd_matched_tau"], - "issue_d": ["2020-01-01", "2020-02-01", "2020-01-01"], - "grade": ["A", "B", "A"], - "funded_exposure": [100.0, 100.0, 200.0], - "funded_weight": [0.5, 0.5, 1.0], - "outcome": [0.0, 1.0, 0.0], - "miscoverage": [0.0, 1.0, 0.0], - "pd_point": [0.1, 0.2, 0.2], - "pd_effective": [0.2, 0.3, 0.2], - "pd_high": [0.4, 0.6, 0.8], - "int_rate": [0.1, 0.2, 0.1], - "realized_return_contribution": [10.0, -45.0, 20.0], - } - ) - - -def _evaluation() -> pd.DataFrame: - return pd.DataFrame( - [ - { - "period": "full_oot", - "role": "calibration_selected", - "n_panel": 10, - "n_funded": 2, - "total_allocated": 200.0, - "expected_objective": 5.0, - "realized_return": -35.0, - "weighted_outcome": 0.5, - "weighted_miscoverage": 0.5, - "weighted_pd_point": 0.15, - "weighted_pd_effective": 0.25, - "gamma_cp": 0.35, - "gamma_internalized": 0.10, - "gamma_residual": 0.25, - "endpoint_budget": 0.50, - "markov_loss_threshold": 0.80, - }, - { - "period": "full_oot", - "role": "point_pd_matched_tau", - "n_panel": 10, - "n_funded": 1, - "total_allocated": 200.0, - "expected_objective": 10.0, - "realized_return": 20.0, - "weighted_outcome": 0.0, - "weighted_miscoverage": 0.0, - "weighted_pd_point": 0.20, - "weighted_pd_effective": 0.20, - "gamma_cp": 0.60, - "gamma_internalized": 0.0, - "gamma_residual": 0.60, - "endpoint_budget": 0.80, - "markov_loss_threshold": 1.10, - }, - ] - ) - - -def test_grade_table_preserves_selected_exposure() -> None: - table = build_grade_table(_allocations()) - - assert table["exposure"].sum() == 200.0 - assert table["exposure_share"].sum() == 1.0 - - -def test_bootstrap_is_deterministic_and_uses_official_observed_values() -> None: - first = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) - second = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) - - pd.testing.assert_frame_equal(first, second) - assert set(first["bootstrap_unit"]) == {"origination_month", "funded_loan"} - observed = first.loc[first["bootstrap_unit"].eq("origination_month")].set_index("metric")[ - "observed" - ] - assert observed["realized_return"] == -35.0 - assert observed["Gamma_CP"] == 0.35 - - -def test_bootstrap_rejects_allocation_evaluation_drift() -> None: - evaluation = _evaluation() - evaluation.loc[evaluation["role"].eq("calibration_selected"), "realized_return"] = 1.0 - - with pytest.raises(ValueError, match="realized_return"): - build_bootstrap_table(_allocations(), evaluation, n_draws=10, seed=7) - - -def test_baseline_table_uses_selected_policy_as_zero_delta() -> None: - table = build_baseline_table(_evaluation()) - selected = table.loc[table["policy"].eq("Calibration-selected 50/50 CRPTO")].iloc[0] - - assert np.isclose(selected["return_delta_vs_selected"], 0.0) - assert np.isclose(selected["default_delta_vs_selected"], 0.0) diff --git a/tests/test_scripts/test_build_papers_tesis_deep_audit.py b/tests/test_scripts/test_build_papers_tesis_deep_audit.py deleted file mode 100644 index ace49b5..0000000 --- a/tests/test_scripts/test_build_papers_tesis_deep_audit.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from scripts.build_papers_tesis_deep_audit import write_audit - - -def _row( - *, - relative_path: str, - decision: str, - action_required: str = "none_now", - bib_status: str = "existing", -) -> dict[str, object]: - return { - "folder": relative_path.split("/", 1)[0], - "relative_path": relative_path, - "title": f"Title {relative_path}", - "status": "published", - "primary_domain": "conformal risk control", - "bib_key": "paperkey", - "bib_status": bib_status, - "core_concepts": "coverage; decision risk", - "key_claims": "claim summary", - "conclusions": "use with finite-grid boundary", - "figures_tables_useful": "inspiration only", - "limitations": "not CRPTO evidence", - "decision": decision, - "action_required": action_required, - "crpto_value": "supports CRPTO framing", - "extended_lab_value": "future-work value", - "evidence_gate": "do not reopen champion", - "artifact_sink": "docs/research/example.md", - "stop_rule": "keep as literature unless claim changes", - "implementation_or_experiment": "none", - } - - -def test_write_audit_builds_editorial_sections(tmp_path: Path) -> None: - rows = [ - _row(relative_path="paper/promote.pdf", decision="promote_crpto_body"), - _row(relative_path="paper/append.pdf", decision="append_crpto_related_work"), - _row( - relative_path="supplement/experiment.pdf", - decision="append_tail_risk", - action_required="experiment_completed_appendix_diagnostic", - ), - _row( - relative_path="tesis/future.pdf", - decision="park_future_work", - bib_status="needs_bib_if_cited", - ), - ] - curated_visual_rows = [ - { - "relative_path": "paper/promote.pdf", - "caption_type": "figure", - "caption_index": 1, - "editorial_sink": "own schematic", - "why_useful": "layout inspiration", - "claim_boundary": "do not reproduce", - } - ] - audit_path = tmp_path / "audit.md" - - write_audit( - audit_path, - rows, - tmp_path / "matrix.csv", - tmp_path / "captions.csv", - tmp_path / "visuals.csv", - curated_visual_rows, - ) - - text = audit_path.read_text(encoding="utf-8") - - assert "# Papers_tesis Deep Audit" in text - assert "## Lectura integrada para Paper CRPTO" in text - assert "paper/promote.pdf" in text - assert "paper/append.pdf" in text - assert "supplement/experiment.pdf" in text - assert "tesis/future.pdf" in text - assert "| action_required | n |" in text - assert "experiment_completed_appendix_diagnostic" in text - assert "needs_bib_if_cited" in text - assert "no cambia el champion CRPTO" in text diff --git a/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py b/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py deleted file mode 100644 index 1e870c7..0000000 --- a/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pandas as pd -import pytest - -from scripts.search import build_pool93_ijds_consolidated_frontier as frontier - - -def test_load_leaderboards_rehydrates_policy_aware_tail_bound( - tmp_path: Path, - monkeypatch, -) -> None: - leaderboard_path = tmp_path / "leaderboard.parquet" - bound_path = tmp_path / "bound.parquet" - pd.DataFrame( - { - "local_candidate_id": [7], - "semantic_policy_key": ["tail-policy"], - "alpha01_endpoint_budget_upper": [0.50], - "alpha01_markov_loss_cap": [0.60], - } - ).to_parquet(leaderboard_path, index=False) - pd.DataFrame( - { - "local_candidate_id": [7], - "semantic_policy_key": ["tail-policy"], - "alpha": [0.01], - "risk_tolerance": [0.20], - "gamma_cp": [0.60], - "weighted_pd_point": [0.10], - "weighted_pd_constraint_used": [0.20], - "weighted_pd_high": [0.60], - "pd_cap_slack": [0.0], - } - ).to_parquet(bound_path, index=False) - monkeypatch.setattr(frontier, "_leaderboard_path", lambda _tag: leaderboard_path) - monkeypatch.setattr(frontier, "_bound_eval_path", lambda _tag: bound_path) - - row = frontier._load_leaderboards(["unit-tag"]).iloc[0] - - assert row["alpha01_gamma_internalized"] == pytest.approx(0.10) - assert row["alpha01_gamma_residual"] == pytest.approx(0.40) - assert row["alpha01_endpoint_budget"] == pytest.approx(0.60) - assert row["alpha01_endpoint_budget_upper"] == pytest.approx(0.60) - assert row["alpha01_markov_loss_threshold"] == pytest.approx(0.70) - assert row["alpha01_markov_loss_cap"] == pytest.approx(0.70) - - -def test_body_selection_uses_exact_markov_threshold() -> None: - eligible = pd.DataFrame( - { - "semantic_policy_key": ["tail", "linear"], - "alpha01_realized_total_return": [220_000.0, 190_000.0], - "alpha01_markov_loss_threshold": [0.70, 0.34], - } - ) - - selected = frontier._body_candidate(eligible, markov_threshold=0.35) - - assert selected["semantic_policy_key"] == "linear" - - -def test_threshold_frontier_does_not_mislabel_tail_policy_as_under_half() -> None: - eligible = pd.DataFrame( - { - "semantic_policy_key": ["tail", "linear"], - "alpha01_realized_total_return": [220_000.0, 190_000.0], - "alpha01_markov_loss_threshold": [0.70, 0.45], - } - ) - - selected = frontier._best_under_threshold(eligible, 0.50) - - assert selected is not None - assert selected["semantic_policy_key"] == "linear" diff --git a/tests/test_scripts/test_build_pool93_point_baseline_audit.py b/tests/test_scripts/test_build_pool93_point_baseline_audit.py deleted file mode 100644 index 715c911..0000000 --- a/tests/test_scripts/test_build_pool93_point_baseline_audit.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import pandas as pd -import pytest - -from scripts.search.build_pool93_point_baseline_audit import ( - _comparison_table, - _format_comparison_tex, -) - - -def test_comparison_table_reports_return_cost_and_certificate_metrics() -> None: - point = { - "realized_return": 200.0, - "expected_return_net_point": 210.0, - "certificate": { - "n_funded": 10, - "weighted_outcome": 0.12, - "weighted_miscoverage": 0.11, - "gamma_cp": 0.50, - "endpoint_budget": 0.60, - "markov_loss_threshold": 0.70, - }, - } - selected = { - "realized_return": 180.0, - "expected_return_net_point": 170.0, - "certificate": { - "n_funded": 14, - "weighted_outcome": 0.04, - "weighted_miscoverage": 0.03, - "gamma_cp": 0.16, - "endpoint_budget": 0.25, - "markov_loss_threshold": 0.35, - }, - } - - table = _comparison_table(point, selected).set_index("policy") - - assert isinstance(table, pd.DataFrame) - assert table.loc["Point-PD two-stage LP", "return_cost_vs_point_pct"] == pytest.approx(0.0) - assert table.loc["Selected CRPTO", "return_cost_vs_point_pct"] == pytest.approx(10.0) - assert table.loc["Selected CRPTO", "Markov_threshold_alpha01"] == pytest.approx(0.35) - assert table.loc["Selected CRPTO", "weighted_default_rate"] == pytest.approx(0.04) - - tex = _format_comparison_tex(table.reset_index()) - assert "Policy & Realized return & Weighted default" in tex - assert "Selected CRPTO & \\$180.00 & 0.040000 & 0.160000 & 0.250000 & 0.350000" in tex - assert "expected\\_return\\_net\\_point" not in tex diff --git a/tests/test_scripts/test_build_pool93_tail_risk_audit.py b/tests/test_scripts/test_build_pool93_tail_risk_audit.py deleted file mode 100644 index 38b51c7..0000000 --- a/tests/test_scripts/test_build_pool93_tail_risk_audit.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import pytest - -from scripts.search.build_pool93_tail_risk_audit import ( - DEFAULT_ALLOCATION_PATH, - DEFAULT_BODY_AUDIT_PATH, - _load_allocation, - _read_json, - build_bootstrap_table, - build_cluster_bound_table, - build_tail_risk_table, -) - - -def _pool93_inputs(): - if not DEFAULT_ALLOCATION_PATH.is_file(): - pytest.skip("pool93 body allocation artifact is not present locally") - return _load_allocation(DEFAULT_ALLOCATION_PATH), _read_json(DEFAULT_BODY_AUDIT_PATH) - - -def test_pool93_tail_repricing_matches_promoted_body_return() -> None: - funded, body_audit = _pool93_inputs() - table = build_tail_risk_table(funded, body_audit=body_audit, lgds=(0.35, 0.45, 0.60)) - baseline = table.loc[table["lgd"].eq(0.45)].iloc[0] - - assert baseline["funded_rows"] == 314 - assert baseline["total_allocated"] == pytest.approx(1_000_000.0) - assert baseline["funded_set_repriced_return"] == pytest.approx(184_832.475845, rel=1e-9) - assert baseline["weighted_default_rate"] == pytest.approx(0.03535, abs=1e-12) - assert baseline["realized_cvar95_loss_rate"] > baseline["decision_time_cvar95_loss_rate"] - assert baseline["alpha01_weighted_miscoverage_V"] == pytest.approx(0.03535, abs=1e-12) - - -def test_pool93_cluster_bounds_remain_looser_than_markov() -> None: - funded, body_audit = _pool93_inputs() - table = build_cluster_bound_table(funded, body_audit=body_audit, alpha=0.01, delta=0.10) - - assert set(table["cluster_type"]) == {"period", "grade_bucket", "period_grade", "score_vintage"} - assert not bool(table["cluster_bound_tighter_than_markov"].any()) - assert table["markov_threshold"].nunique() == 1 - assert float(table["markov_threshold"].iloc[0]) == pytest.approx(0.1) - period_grade = table.loc[table["cluster_type"].eq("period_grade")].iloc[0] - assert period_grade["cluster_hoeffding_threshold"] == pytest.approx(0.281247, rel=1e-6) - assert period_grade["sum_cluster_exposure_sq"] > period_grade["sum_w2_tightening_threshold"] - - -def test_pool93_bootstrap_table_is_fixed_allocation_diagnostic() -> None: - funded, body_audit = _pool93_inputs() - table = build_bootstrap_table( - funded, - body_audit=body_audit, - n_draws=5000, - seed=20260702, - lgd=0.45, - ) - metrics = {row["metric"]: row for row in table.to_dict(orient="records")} - return_row = metrics["funded_set_repriced_return_lgd45"] - v_row = metrics["weighted_miscoverage_V"] - gamma_row = metrics["alpha01_gamma_cp"] - - assert return_row["observed"] == pytest.approx(184_832.475845, rel=1e-9) - assert return_row["boot_p025"] == pytest.approx(167_963.197413, rel=1e-9) - assert return_row["boot_p975"] == pytest.approx(198_650.467343, rel=1e-9) - assert v_row["observed"] == pytest.approx(0.03535, abs=1e-12) - assert v_row["boot_p975"] < 0.1 - assert gamma_row["boot_p975"] < 0.2 - assert table["note"].nunique() == 1 - assert "solver input uncertainty is not resampled" in str(table["note"].iloc[0]) diff --git a/tests/test_scripts/test_compile_ijds_submission.py b/tests/test_scripts/test_compile_ijds_submission.py index 13c5480..58a6ec8 100644 --- a/tests/test_scripts/test_compile_ijds_submission.py +++ b/tests/test_scripts/test_compile_ijds_submission.py @@ -1,6 +1,32 @@ from __future__ import annotations -from scripts.compile_ijds_submission import LatexScan, _windows_latexmk_script +import hashlib +import json +from pathlib import Path + +import pytest + +import scripts.compile_ijds_submission as submission_compiler +from scripts.compile_ijds_submission import ( + JOB_NAME, + OFFICIAL_TEMPLATE_FILES, + STYLE_MANIFEST, + LatexScan, + _missing_template_files, + _template_asset_drift, + _windows_latexmk_script, + main, + scan_submission_logs, +) + + +def _write_clean_submission_outputs(directory: Path, *, pages: str = "27") -> None: + (directory / f"{JOB_NAME}.pdf").write_bytes(b"%PDF-1.7\n") + (directory / f"{JOB_NAME}.blg").write_text("This is BibTeX\n", encoding="utf-8") + (directory / f"{JOB_NAME}.log").write_text( + f"Output written on {JOB_NAME}.pdf ({pages} pages, 12345 bytes).\n", + encoding="utf-8", + ) def test_latex_scan_ok_property_flags_clean_build() -> None: @@ -14,6 +40,87 @@ def test_latex_scan_ok_property_rejects_warnings_or_log_failures() -> None: assert not LatexScan(pages=27, blg_warnings=(), log_failures=("undefined references",)).ok +def test_latex_scan_ok_property_requires_outputs_and_positive_page_count() -> None: + assert not LatexScan(pages=None, blg_warnings=(), log_failures=()).ok + assert not LatexScan(pages=0, blg_warnings=(), log_failures=()).ok + assert not LatexScan( + pages=27, + blg_warnings=(), + log_failures=(), + artifact_failures=("missing PDF",), + ).ok + + +@pytest.mark.parametrize( + ("suffix", "failure"), + [ + ("pdf", "missing PDF"), + ("log", "missing LaTeX log"), + ("blg", "missing BibTeX log"), + ], +) +def test_scan_rejects_each_missing_required_output( + tmp_path: Path, + suffix: str, + failure: str, +) -> None: + _write_clean_submission_outputs(tmp_path) + (tmp_path / f"{JOB_NAME}.{suffix}").unlink() + + scan = scan_submission_logs(tmp_path) + + assert not scan.ok + assert any(item.startswith(failure) for item in scan.artifact_failures) + + +@pytest.mark.parametrize( + ("suffix", "failure"), + [ + ("pdf", "empty PDF"), + ("log", "empty LaTeX log"), + ("blg", "empty BibTeX log"), + ], +) +def test_scan_rejects_each_empty_required_output( + tmp_path: Path, + suffix: str, + failure: str, +) -> None: + _write_clean_submission_outputs(tmp_path) + (tmp_path / f"{JOB_NAME}.{suffix}").write_bytes(b"") + + scan = scan_submission_logs(tmp_path) + + assert not scan.ok + assert any(item.startswith(failure) for item in scan.artifact_failures) + + +def test_scan_rejects_unparseable_page_count(tmp_path: Path) -> None: + _write_clean_submission_outputs(tmp_path, pages="unknown") + + scan = scan_submission_logs(tmp_path) + + assert scan.pages is None + assert not scan.ok + + +def test_scan_rejects_nonpositive_page_count(tmp_path: Path) -> None: + _write_clean_submission_outputs(tmp_path, pages="0") + + scan = scan_submission_logs(tmp_path) + + assert scan.pages == 0 + assert not scan.ok + + +def test_scan_accepts_complete_clean_outputs(tmp_path: Path) -> None: + _write_clean_submission_outputs(tmp_path) + + scan = scan_submission_logs(tmp_path) + + assert scan.ok + + def test_windows_latexmk_script_finds_tinytex_payload(tmp_path) -> None: wrapper = tmp_path / "TinyTeX" / "bin" / "windows" / "latexmk.exe" script = tmp_path / "TinyTeX" / "texmf-dist" / "scripts" / "latexmk" / "latexmk.pl" @@ -23,3 +130,69 @@ def test_windows_latexmk_script_finds_tinytex_payload(tmp_path) -> None: script.touch() assert _windows_latexmk_script(wrapper) == script + + +def test_missing_template_files_reports_only_absent_assets(tmp_path) -> None: + for name in OFFICIAL_TEMPLATE_FILES[:2]: + (tmp_path / name).touch() + + assert _missing_template_files(tmp_path) == OFFICIAL_TEMPLATE_FILES[2:] + + +def test_style_manifest_tracks_the_complete_publisher_asset_set() -> None: + payload = json.loads(STYLE_MANIFEST.read_text(encoding="utf-8")) + + assert set(payload["assets"]) == set(OFFICIAL_TEMPLATE_FILES) + assert payload["informs4_class_version"] == "2024/06/03 v1.02" + + +def test_style_manifest_detects_tampered_publisher_asset(tmp_path) -> None: + assets = {} + for name in OFFICIAL_TEMPLATE_FILES: + content = f"reviewed {name}\n".encode() + (tmp_path / name).write_bytes(content) + assets[name] = { + "bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + } + (tmp_path / STYLE_MANIFEST.name).write_text( + json.dumps({"assets": assets}), + encoding="utf-8", + ) + + assert _template_asset_drift(tmp_path) == () + (tmp_path / OFFICIAL_TEMPLATE_FILES[0]).write_bytes(b"tampered") + + assert _template_asset_drift(tmp_path) == (OFFICIAL_TEMPLATE_FILES[0],) + + +def test_official_template_starts_references_on_a_new_page() -> None: + template = STYLE_MANIFEST.parent / "informs-pandoc-template.tex" + + assert "\\clearpage\n\\bibliographystyle" in template.read_text(encoding="utf-8") + + +def test_scan_only_rejects_redundant_skip_render() -> None: + with pytest.raises(SystemExit) as error: + main(["--scan-only", "--skip-render"]) + + assert error.value.code == 2 + + +def test_scan_only_fails_when_required_outputs_are_missing(tmp_path: Path, monkeypatch) -> None: + _write_clean_submission_outputs(tmp_path) + (tmp_path / f"{JOB_NAME}.pdf").unlink() + monkeypatch.setattr(submission_compiler, "SUBMISSION_DIR", tmp_path) + + assert main(["--scan-only"]) == 1 + + +def test_post_compile_validation_fails_when_page_count_is_unparseable( + tmp_path: Path, + monkeypatch, +) -> None: + _write_clean_submission_outputs(tmp_path, pages="unknown") + monkeypatch.setattr(submission_compiler, "SUBMISSION_DIR", tmp_path) + monkeypatch.setattr(submission_compiler, "compile_submission", lambda **_: 0) + + assert main([]) == 1 diff --git a/tests/test_scripts/test_explicit_protocol_configs.py b/tests/test_scripts/test_explicit_protocol_configs.py new file mode 100644 index 0000000..49157bd --- /dev/null +++ b/tests/test_scripts/test_explicit_protocol_configs.py @@ -0,0 +1,35 @@ +"""CLI contracts for versioned IJDS protocol entrypoints.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence + +import pytest + +from scripts.experiments import ( + run_ijds_binary_geometry_frontier_v4 as binary_geometry, + run_ijds_credit_risk_controls as credit_controls, + run_ijds_normalized_objective_frontier_v2 as two_ruler, + run_ijds_portfolio_structure_sensitivity as structural_sensitivity, + run_ijds_raw_data_audit as raw_data_audit, +) + + +@pytest.mark.parametrize( + ("parser", "argv"), + [ + (binary_geometry.parse_args, ["evaluate"]), + (credit_controls.parse_args, ["evaluate"]), + (two_ruler.parse_args, []), + (structural_sensitivity.parse_args, ["--phase", "freeze"]), + (raw_data_audit.parse_args, []), + ], +) +def test_versioned_protocol_entrypoints_require_explicit_config( + parser: Callable[[Sequence[str] | None], object], + argv: list[str], +) -> None: + with pytest.raises(SystemExit) as error: + parser(argv) + + assert error.value.code == 2 diff --git a/tests/test_scripts/test_export_crpto_tables.py b/tests/test_scripts/test_export_crpto_tables.py deleted file mode 100644 index db3c199..0000000 --- a/tests/test_scripts/test_export_crpto_tables.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -from collections.abc import Iterator -from contextlib import contextmanager -from pathlib import Path - -from scripts import analyze_crpto_evidence - - -@contextmanager -def _preserve_files(paths: list[Path]) -> Iterator[None]: - snapshots = {path: path.read_bytes() if path.exists() else None for path in paths} - try: - yield - finally: - for path, payload in snapshots.items(): - if payload is None: - path.unlink(missing_ok=True) - else: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(payload) - - -def test_export_crpto_tables_script_runs() -> None: - # The script intentionally refreshes the CRPTO evidence status timestamp. - # Preserve the committed artifact so the test remains side-effect free. - generated_paths = [ - *Path("reports/crpto/tables").glob("crpto_table*.*"), - Path("models/crpto_evidence_status.json"), - Path("docs/research/crpto_p1_evidence_2026-05-04.md"), - ] - with _preserve_files(generated_paths): - subprocess.run([sys.executable, "scripts/export_crpto_tables.py"], check=True) - assert Path("reports/crpto/tables/crpto_table0_key_metrics.csv").exists() - - -def test_evidence_repo_path_uses_posix_separators() -> None: - path = analyze_crpto_evidence.ROOT / "reports" / "crpto" / "tables" / "example.csv" - repo_path = analyze_crpto_evidence._repo_path(path) - - assert repo_path == "reports/crpto/tables/example.csv" - assert "\\" not in repo_path diff --git a/tests/test_scripts/test_export_pool93_policy_aware_frontier.py b/tests/test_scripts/test_export_pool93_policy_aware_frontier.py deleted file mode 100644 index c548491..0000000 --- a/tests/test_scripts/test_export_pool93_policy_aware_frontier.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from scripts.search.export_pool93_policy_aware_frontier import ROLE_ORDER, build_table - - -def test_build_table_orders_roles_and_uses_exact_threshold() -> None: - rows = [] - for index, role in enumerate(reversed(ROLE_ORDER), start=1): - rows.append( - { - "role": role, - "run_label": "unit", - "local_candidate_id": index, - "family": "family", - "risk_tolerance": 0.17, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "uncertainty_aversion": 0.1, - "return": 180_000.0 + index, - "return_floor_surplus": 9_000.0, - "Gamma_CP": 0.2, - "Gamma_residual": 0.1, - "V": 0.03, - "endpoint_budget": 0.24, - "endpoint_budget_upper": 0.24, - "Markov_threshold": 0.34, - "Markov_cap": 0.34, - "alpha_pass": "8/8", - "n_funded_mean": 300.0, - } - ) - - table = build_table({"rows": rows}) - - assert len(table) == len(ROLE_ORDER) - assert table.iloc[0]["role"] == "Minimum Markov-threshold endpoint" - assert table.iloc[-1]["role"] == "Max-return economic endpoint" - assert table["Markov_threshold_alpha01"].eq(0.34).all() - assert "Gamma_residual_alpha01" in table.columns diff --git a/tests/test_scripts/test_generate_conformal_intervals_cli.py b/tests/test_scripts/test_generate_conformal_intervals_cli.py deleted file mode 100644 index b58f4dc..0000000 --- a/tests/test_scripts/test_generate_conformal_intervals_cli.py +++ /dev/null @@ -1,436 +0,0 @@ -from __future__ import annotations - -import pytest - -import scripts.generate_conformal_intervals as conformal_script -from scripts.generate_conformal_intervals import ( - _apply_global_rebalance, - _apply_learned_floor_policy, - _build_conformal_artifact_tables, - _build_tuning_split, - _can_use_temporal_segments, - _parse_bool_tuple, - _parse_float_tuple, - _parse_int_tuple, - _parse_str_tuple, - _resolve_tuning_grid, - _select_alpha_95, - _select_best_tuning_config, - _tuning_total_candidates, -) - - -def test_conformal_cli_tuple_parsers_strip_and_cast_values() -> None: - assert _parse_float_tuple("0.1, 0.05") == (0.1, 0.05) - assert _parse_int_tuple("5, 10") == (5, 10) - assert _parse_bool_tuple("true,false,1,0,yes,no,y,n") == ( - True, - False, - True, - False, - True, - False, - True, - False, - ) - assert _parse_str_tuple("raw, calibrated ") == ("raw", "calibrated") - - -@pytest.mark.parametrize( - "parser", - [_parse_float_tuple, _parse_int_tuple, _parse_bool_tuple, _parse_str_tuple], -) -def test_conformal_cli_tuple_parsers_reject_empty_lists(parser) -> None: - with pytest.raises(ValueError, match="Expected at least one"): - parser(" , ") - - -def test_resolve_tuning_grid_normalizes_candidates() -> None: - grid = _resolve_tuning_grid( - partition="grade", - partition_candidates=(" grade ", "grade", "score_bin"), - partition_probability_sources=(" RAW ", "calibrated", "raw"), - n_score_bins_candidates=(0, 5, 5), - fallback_modes=(" Grade_Then_Global ", "global", "global"), - score_scale_families=(" None ", "bernoulli_sqrt", "none"), - scaled_scores_options=(True, False), - ) - - assert grid.partition_candidates == ("grade", "score_bin") - assert grid.partition_probability_sources == ("raw", "calibrated") - assert grid.n_score_bins_candidates == (5, 5) - assert grid.fallback_modes == ("grade_then_global", "global") - assert grid.score_scale_families == ("none", "bernoulli_sqrt") - assert grid.scaled_scores_options == (True, False) - - -def test_resolve_tuning_grid_uses_current_defaults_for_empty_inputs() -> None: - grid = _resolve_tuning_grid( - partition="", - partition_candidates=None, - partition_probability_sources=(), - n_score_bins_candidates=(0,), - fallback_modes=(), - score_scale_families=(), - scaled_scores_options=(), - ) - - assert grid.partition_candidates == ("grade",) - assert grid.partition_probability_sources == ("raw",) - assert grid.n_score_bins_candidates == (10,) - assert grid.fallback_modes == ("grade_then_global",) - assert grid.score_scale_families == ("none",) - assert grid.scaled_scores_options == () - - -def test_tuning_total_candidates_counts_cartesian_grid() -> None: - grid = _resolve_tuning_grid( - partition="grade", - partition_candidates=("grade", "score_bin"), - partition_probability_sources=("raw",), - n_score_bins_candidates=(5, 10), - fallback_modes=("global",), - score_scale_families=("none",), - scaled_scores_options=(True, False), - ) - - assert ( - _tuning_total_candidates( - grid, - alpha_candidates_90=(0.10, 0.09), - min_group_sizes=(200, 500), - ) - == 32 - ) - - -def test_build_tuning_split_materializes_fit_and_holdout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_split_calibration_for_tuning(**kwargs): - return conformal_script.np.array([0, 2]), conformal_script.np.array([1, 3]) - - monkeypatch.setattr( - conformal_script, - "split_calibration_for_tuning", - fake_split_calibration_for_tuning, - ) - cal_df = conformal_script.pd.DataFrame( - { - "issue_d": ["2020-01-01", "2020-02-01", "2020-03-01", "bad-date"], - "feature": [10, 20, 30, 40], - } - ) - test_df = conformal_script.pd.DataFrame({"issue_d": ["2021-01-01", "not-a-date"]}) - x_cal = conformal_script.pd.DataFrame({"feature": [10, 20, 30, 40]}) - y_cal = conformal_script.pd.Series([0.0, 1.0, 0.0, 1.0]) - group_cal = conformal_script.pd.Series(["A", "B", "A", "C"]) - y_prob_raw = conformal_script.np.array([0.1, 0.2, 0.3, 0.4]) - - split = _build_tuning_split( - cal_df=cal_df, - test_df=test_df, - X_cal=x_cal, - y_cal=y_cal, - group_cal_base=group_cal, - y_prob_cal_raw=y_prob_raw, - tuning_holdout_ratio=0.5, - tuning_random_state=7, - ) - - assert split.idx_cal_fit.tolist() == [0, 2] - assert split.idx_cal_tune.tolist() == [1, 3] - assert split.X_cal_fit["feature"].tolist() == [10, 30] - assert split.X_tune["feature"].tolist() == [20, 40] - assert split.y_cal_fit.tolist() == [0.0, 0.0] - assert split.y_tune.tolist() == [1.0, 1.0] - assert split.y_prob_cal_fit.tolist() == [0.1, 0.3] - assert split.y_prob_cal_tune.tolist() == [0.2, 0.4] - assert split.group_cal_fit_base.tolist() == ["A", "A"] - assert split.group_tune_base.tolist() == ["B", "C"] - assert split.issue_tune.isna().tolist() == [False, True] - assert split.issue_test.isna().tolist() == [False, True] - - -def test_apply_global_rebalance_disabled_preserves_current_metrics() -> None: - y_int = conformal_script.np.array([[0.1, 0.4], [0.2, 0.7]]) - metrics = {"empirical_coverage": 0.9} - group_metrics = conformal_script.pd.DataFrame({"group": ["A"], "coverage": [0.9]}) - - result = _apply_global_rebalance( - enabled=False, - min_factor=0.75, - max_factor=1.05, - step=0.01, - y_int_tune_working=y_int, - y_pred_tune=conformal_script.np.array([0.25, 0.45]), - y_tune=conformal_script.pd.Series([0.0, 1.0]), - y_int_90=y_int, - y_pred_90=conformal_script.np.array([0.25, 0.45]), - y_eval_90=conformal_script.pd.Series([0.0, 1.0]), - group_tune=conformal_script.pd.Series(["A", "B"]), - eval_groups_90=conformal_script.pd.Series(["A", "B"]), - alpha_target_90=0.10, - target_coverage_90=0.90, - min_group_coverage_target=0.88, - metrics_90=metrics, - group_metrics_90=group_metrics, - ) - - assert result.y_intervals is y_int - assert result.metrics is metrics - assert result.group_metrics is group_metrics - assert result.factor == 1.0 - assert result.diagnostics == {"enabled": False, "applied": False} - - -def test_can_use_temporal_segments_requires_enabled_dates_and_matching_lengths() -> None: - issue_dates = conformal_script.pd.Series(["2020-01-01", "2020-02-01"]) - groups = conformal_script.pd.Series(["A", "B"]) - - assert _can_use_temporal_segments( - enabled=True, - issue_tune=issue_dates, - eval_issue=issue_dates, - group_tune=groups, - eval_groups=groups, - ) - assert not _can_use_temporal_segments( - enabled=False, - issue_tune=issue_dates, - eval_issue=issue_dates, - group_tune=groups, - eval_groups=groups, - ) - assert not _can_use_temporal_segments( - enabled=True, - issue_tune=conformal_script.pd.Series([None, None]), - eval_issue=issue_dates, - group_tune=groups, - eval_groups=groups, - ) - assert not _can_use_temporal_segments( - enabled=True, - issue_tune=issue_dates, - eval_issue=issue_dates, - group_tune=groups, - eval_groups=conformal_script.pd.Series(["A"]), - ) - - -def test_apply_learned_floor_policy_applies_group_temporal_and_global_factors() -> None: - adjusted = _apply_learned_floor_policy( - y_pred=conformal_script.np.array([0.5, 0.5]), - y_intervals=conformal_script.np.array([[0.4, 0.6], [0.4, 0.6]]), - groups=conformal_script.pd.Series(["A", "B"]), - group_multipliers={"A": 2.0}, - temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), - temporal_segment_multipliers={"B|2020Q1": 3.0}, - global_rebalance_factor=0.5, - ) - - conformal_script.np.testing.assert_allclose( - adjusted, - conformal_script.np.array([[0.4, 0.6], [0.35, 0.65]]), - ) - - -def test_build_conformal_artifact_tables_preserves_holdout_metadata() -> None: - tables = _build_conformal_artifact_tables( - y_eval_90=conformal_script.pd.Series([0.0, 1.0]), - y_pred_90=conformal_script.np.array([0.2, 0.8]), - y_int_90=conformal_script.np.array([[0.1, 0.4], [0.6, 0.9]]), - y_int_95=conformal_script.np.array([[0.0, 0.5], [0.5, 1.0]]), - eval_groups_90=conformal_script.pd.Series(["A", "B"]), - eval_temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), - evaluation_scope_key="holdout", - test_df=conformal_script.pd.DataFrame({"id": ["test-1"], "loan_amnt": [9000.0]}), - cal_df=conformal_script.pd.DataFrame( - { - "id": ["cal-0", "cal-1", "cal-2"], - "loan_amnt": [1000.0, 2000.0, 3000.0], - } - ), - idx_cal_tune=conformal_script.np.array([1, 2]), - group_metrics_90=conformal_script.pd.DataFrame( - { - "group": ["A", "B"], - "coverage": [0.9, 1.0], - "avg_width": [0.3, 0.3], - "median_width": [0.3, 0.3], - } - ), - group_metrics_95=conformal_script.pd.DataFrame( - { - "group": ["A", "B"], - "coverage": [0.95, 1.0], - "avg_width": [0.5, 0.5], - "median_width": [0.5, 0.5], - } - ), - coverage_floor_report=conformal_script.pd.DataFrame( - { - "group": ["A", "B"], - "coverage_before": [0.9, 1.0], - "coverage_after": [0.92, 1.0], - "multiplier": [1.02, 1.0], - "adjusted": [True, False], - } - ), - width_attr_rows=[{"dataset_scope": "holdout", "stage": "base_interval"}], - ) - - assert tables.intervals["_row_number"].tolist() == [0, 1] - assert tables.intervals["id"].tolist() == ["cal-1", "cal-2"] - assert tables.intervals["loan_amnt"].tolist() == [2000.0, 3000.0] - assert tables.intervals["temporal_segment"].tolist() == ["A|2020Q1", "B|2020Q1"] - assert tables.group_metrics["coverage_95"].tolist() == [0.95, 1.0] - assert tables.group_metrics["adjusted"].tolist() == [True, False] - assert tables.width_attribution.to_dict(orient="records") == [ - {"dataset_scope": "holdout", "stage": "base_interval"} - ] - - -def test_build_tuning_split_rejects_empty_holdout(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_split_calibration_for_tuning(**kwargs): - return conformal_script.np.array([0, 1]), conformal_script.np.array([], dtype=int) - - monkeypatch.setattr( - conformal_script, - "split_calibration_for_tuning", - fake_split_calibration_for_tuning, - ) - - with pytest.raises(ValueError, match="holdout split is empty"): - _build_tuning_split( - cal_df=conformal_script.pd.DataFrame({"feature": [1, 2]}), - test_df=conformal_script.pd.DataFrame({"feature": [3]}), - X_cal=conformal_script.pd.DataFrame({"feature": [1, 2]}), - y_cal=conformal_script.pd.Series([0.0, 1.0]), - group_cal_base=conformal_script.pd.Series(["A", "B"]), - y_prob_cal_raw=conformal_script.np.array([0.1, 0.2]), - tuning_holdout_ratio=0.5, - tuning_random_state=7, - ) - - -def test_select_alpha_95_uses_holdout_gap_then_width(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_create_intervals(**kwargs): - alpha = float(kwargs["alpha"]) - return ( - conformal_script.np.array([0.2, 0.3]), - conformal_script.np.array([[0.0, alpha], [0.1, alpha + 0.1]]), - {}, - ) - - def fake_validate_coverage(*args, **kwargs): - alpha = float(kwargs["alpha"]) - return { - "coverage_gap": {0.04: 0.03, 0.05: 0.01, 0.06: 0.01}[alpha], - "avg_interval_width": {0.04: 0.20, 0.05: 0.18, 0.06: 0.16}[alpha], - } - - monkeypatch.setattr( - conformal_script, - "create_pd_intervals_mondrian_from_predictions", - fake_create_intervals, - ) - monkeypatch.setattr(conformal_script, "validate_coverage", fake_validate_coverage) - - selected_alpha = _select_alpha_95( - alpha_95=0.05, - alpha_candidates_95=(0.04, 0.05, 0.06), - interval_fit_pred=conformal_script.np.array([0.1, 0.2]), - interval_tune_pred=conformal_script.np.array([0.2, 0.3]), - y_cal_fit=conformal_script.pd.Series([0.0, 1.0]), - y_tune=conformal_script.pd.Series([0.0, 1.0]), - group_cal_fit_holdout=conformal_script.pd.Series(["A", "B"]), - group_tune=conformal_script.pd.Series(["A", "B"]), - best_cfg={ - "min_group_size": 200, - "scaled_scores": False, - "score_scale_family": "none", - }, - ) - - assert selected_alpha == 0.06 - - -def test_select_best_tuning_config_materializes_promoted_config() -> None: - rows = [ - { - "partition": "grade", - "partition_probability_source": "raw", - "n_score_bins": 10, - "fallback_mode": "grade_then_global", - "alpha_used_90": 0.10, - "scaled_scores": False, - "score_scale_family": "none", - "min_group_size": 200, - "empirical_coverage": 0.902, - "target_coverage": 0.9, - "coverage_gap": 0.002, - "avg_interval_width": 0.40, - "median_interval_width": 0.38, - "min_group_coverage": 0.901, - "max_group_coverage": 0.93, - "std_group_coverage": 0.01, - "winkler_90": 0.30, - "max_monthly_gap": 0.02, - "stability_over_time": 0.98, - }, - { - "partition": "score_bin", - "partition_probability_source": "calibrated", - "n_score_bins": 5, - "fallback_mode": "global", - "alpha_used_90": 0.09, - "scaled_scores": True, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 500, - "empirical_coverage": 0.922, - "target_coverage": 0.9, - "coverage_gap": 0.022, - "avg_interval_width": 0.35, - "median_interval_width": 0.34, - "min_group_coverage": 0.912, - "max_group_coverage": 0.94, - "std_group_coverage": 0.01, - "winkler_90": 0.20, - "max_monthly_gap": 0.01, - "stability_over_time": 0.99, - }, - ] - - selection = _select_best_tuning_config( - rows, - partition_candidates=("grade", "score_bin"), - alpha_target_90=0.10, - min_group_coverage_target=0.90, - group_coverage_floor_target_90=0.92, - coverage_guardband_90=0.015, - min_group_guardband_90=0.0, - max_width_budget_90=0.80, - target_coverage_90=0.90, - ) - - assert {"is_pareto", "global_ok", "group_ok", "width_ok"}.issubset(selection.tuning_df.columns) - assert selection.selection_tier == "strong_global+strong_group+width" - assert selection.best_cfg == { - "partition": "score_bin", - "partition_candidates": ["grade", "score_bin"], - "partition_probability_source": "calibrated", - "n_score_bins": 5, - "fallback_mode": "global", - "alpha_target_90": 0.10, - "alpha_used_90": 0.09, - "scaled_scores": True, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 500, - "min_group_coverage_target": 0.90, - "group_coverage_floor_target_90": 0.92, - "coverage_guardband_90": 0.015, - "min_group_guardband_90": 0.0, - "max_width_budget_90": 0.80, - "selection_tier": "strong_global+strong_group+width", - } diff --git a/tests/test_scripts/test_generate_crpto_figures.py b/tests/test_scripts/test_generate_crpto_figures.py deleted file mode 100644 index 9dc5c5b..0000000 --- a/tests/test_scripts/test_generate_crpto_figures.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -import pandas as pd - -from scripts.generate_crpto_figures import ( - PALETTE, - _alpha_annotation_offset, - _alpha_pareto_column_map, - _alpha_pareto_missing_columns, - _alpha_pareto_subframe, - _alpha_pareto_variant_styles, - _alpha_tick_labels, -) - - -def test_alpha_pareto_column_map_detects_semantic_columns() -> None: - df = pd.DataFrame( - columns=[ - "method_name", - "alpha_level", - "empirical_coverage", - "mean_width", - "n_eligible_loans", - ] - ) - - columns = _alpha_pareto_column_map(df) - - assert columns == { - "variant": "method_name", - "alpha": "alpha_level", - "coverage": "empirical_coverage", - "width": "mean_width", - "eligible": "n_eligible_loans", - } - assert _alpha_pareto_missing_columns(columns) == [] - - -def test_alpha_pareto_variant_styles_label_mondrian_and_global() -> None: - colors, labels = _alpha_pareto_variant_styles(["mondrian", "global"]) - - assert labels == { - "mondrian": "Mondrian CP", - "global": "Global Split-CP", - } - assert colors == { - "mondrian": PALETTE["blue"], - "global": PALETTE["orange"], - } - - -def test_alpha_pareto_subframe_sorts_alpha_and_formats_labels() -> None: - df = pd.DataFrame( - { - "variant": ["global", "mondrian", "global"], - "alpha": [0.2, 0.1, 0.05], - "coverage": [0.93, 0.91, 0.90], - } - ) - - sub = _alpha_pareto_subframe( - df, - variant_col="variant", - alpha_col="alpha", - variant="global", - ) - - assert list(sub["alpha"]) == [0.05, 0.2] - assert _alpha_tick_labels(sub["alpha"]) == ["0.05", "0.2"] - assert _alpha_annotation_offset(0, 2) == (4, 4) - assert _alpha_annotation_offset(1, 2) == (-24, -8) diff --git a/tests/test_scripts/test_generate_governance_status.py b/tests/test_scripts/test_generate_governance_status.py deleted file mode 100644 index 1ea0a13..0000000 --- a/tests/test_scripts/test_generate_governance_status.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np -import pandas as pd - -from scripts.generate_governance_status import ( - GovernanceOutputPaths, - GovernanceThresholds, - _build_explanation_drift_report, - _build_governance_status, - _drift_breach_metrics, -) - - -def _test_thresholds() -> GovernanceThresholds: - return GovernanceThresholds( - psi_threshold=0.25, - ks_pvalue_min=0.01, - cvm_pvalue_min=0.01, - c2st_auc_max=0.60, - max_feature_breach_ratio=0.60, - c2st_max_rows=50_000, - score_psi_max=0.15, - auc_delta_max=0.05, - brier_increase_max=0.02, - calibration_gap_delta_max=0.02, - performance_max_rows=100_000, - min_rank_overlap_top10=0.60, - max_explanation_shap_psi=0.25, - min_reason_code_stability=0.55, - explanation_min_rows_per_slice=80, - psi_bins=10, - random_state=42, - ) - - -def _test_paths() -> GovernanceOutputPaths: - return GovernanceOutputPaths( - drift_path=Path("data/processed/drift_monitoring.parquet"), - status_path=Path("models/governance_status.json"), - explanation_drift_path=Path("data/processed/explanation_drift.parquet"), - fairness_status_path=Path("models/fairness_audit_status.json"), - fairness_frontier_path=Path("data/processed/fairness_threshold_frontier.parquet"), - challenger_report_path=Path("models/challenger_promotion_report.json"), - model_shift_status_path=Path("models/model_shift_status.json"), - ) - - -def test_build_explanation_drift_report_emits_overall_and_grade_rows() -> None: - rows: list[dict[str, object]] = [] - periods = ["2020Q1", "2020Q2", "2020Q3"] - for period in periods: - for grade in ["A", "B"]: - for idx in range(20): - rows.append( - { - "issue_quarter": period, - "grade": grade, - "pd_calibrated": 0.20, - "shap_dti": 0.30 + 0.001 * idx, - "shap_income": 0.10 + 0.001 * idx, - } - ) - shap_raw = pd.DataFrame(rows) - - report = _build_explanation_drift_report( - shap_raw, - primary_threshold=0.50, - min_rank_overlap_top10=0.50, - max_shap_psi=10.0, - min_reason_code_stability=0.50, - min_rows_per_slice=20, - ) - - assert set(report["segment_type"]) == {"overall", "grade"} - assert set(report["segment"]) == {"all", "A", "B"} - assert set(report["comparison_period"]) == {"2020Q3"} - assert report["passed_all"].all() - assert np.isfinite(report["max_shap_psi_top5"]).all() - - details = json.loads(str(report.loc[report["segment"] == "all", "feature_psi_details"].iloc[0])) - assert {row["feature"] for row in details} == {"dti", "income"} - - -def test_governance_status_helpers_preserve_public_contract() -> None: - thresholds = _test_thresholds() - drift_df = pd.DataFrame( - { - "pass_psi": [True, False], - "pass_ks": [True, True], - "pass_cvm": [True, False], - "psi": [0.05, 0.20], - "ks_pvalue": [0.50, 0.40], - "cvm_pvalue": [0.60, 0.30], - "feature": ["a", "b"], - } - ) - c2st = { - "c2st_auc": 0.55, - "materiality": "moderate", - "effective_driver_count": 1, - "top_drivers": ["a"], - "n_rows": 100, - } - performance = { - "score_psi": 0.10, - "auc_delta_train_to_test": 0.02, - "brier_increase_train_to_test": 0.01, - "calibration_gap_delta": 0.01, - } - metrics = _drift_breach_metrics(drift_df, c2st, performance, thresholds) - explanation_drift = pd.DataFrame( - { - "passed_all": [True], - "pass_reason_code_stability": [True], - "rank_overlap_top10": [0.80], - "max_shap_psi_top5": [0.10], - "reason_code_match_rate": [0.90], - } - ) - status = _build_governance_status( - config_path="configs/mrm_policy.yaml", - resolved_run_tag="test-run", - paths=_test_paths(), - thresholds=thresholds, - drift_df=drift_df, - explanation_drift=explanation_drift, - fairness_status={"overall_pass": True, "primary_threshold": 0.42}, - challenger_report={"challenger_promotable": True}, - metrics=metrics, - model_shift={"shift_type": "stable", "governance_posture": "monitor"}, - ) - - assert metrics["psi_breaches"] == 1 - assert metrics["pass_predictive_drift"] is True - assert status["overall_pass"] is True - assert status["checks"]["pass_explainability"] is True - assert status["summary"]["fairness_primary_threshold"] == 0.42 - assert status["summary"]["challenger_promotable"] is True - assert Path(status["artifacts"]["model_shift_status_path"]) == Path( - "models/model_shift_status.json" - ) - - -def test_build_explanation_drift_report_requires_enough_recent_rows() -> None: - shap_raw = pd.DataFrame( - { - "issue_quarter": ["2020Q1", "2020Q2"], - "pd_calibrated": [0.20, 0.20], - "shap_dti": [0.1, 0.2], - } - ) - - report = _build_explanation_drift_report( - shap_raw, - primary_threshold=0.50, - min_rank_overlap_top10=0.50, - max_shap_psi=10.0, - min_reason_code_stability=0.50, - min_rows_per_slice=20, - ) - - assert report.empty diff --git a/tests/test_scripts/test_generate_mrm_report.py b/tests/test_scripts/test_generate_mrm_report.py deleted file mode 100644 index a372f16..0000000 --- a/tests/test_scripts/test_generate_mrm_report.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Tests for scripts/generate_mrm_report.py compliance gate behavior.""" - -from __future__ import annotations - -import json -import pickle - -import numpy as np -import yaml -from sklearn.linear_model import LogisticRegression - -from scripts import generate_mrm_report as mrm_mod - - -def test_generate_mrm_report_overall_pass_with_pipeline_summary(tmp_path) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - report_dir = tmp_path / "reports" / "mrm" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - report_dir.mkdir(parents=True) - - (data_dir / "pipeline_summary.json").write_text( - json.dumps({"pipeline": {"batch_size": 10}, "pd_model": {"final_auc": 0.72}}), - encoding="utf-8", - ) - (model_dir / "conformal_policy_status.json").write_text( - json.dumps({"overall_pass": True}), encoding="utf-8" - ) - (model_dir / "governance_status.json").write_text( - json.dumps({"overall_pass": True}), encoding="utf-8" - ) - (model_dir / "fairness_audit_status.json").write_text( - json.dumps({"overall_pass": True}), encoding="utf-8" - ) - - cfg = { - "model": { - "name": "CorePD", - "version": "1.0", - "owner": "owner", - "champion_artifact": "models/pd_canonical.cbm", - }, - "governance": {"validation_frequency_days": 90, "review_cadence": "quarterly"}, - "retraining_triggers": { - "psi_threshold": 0.25, - "auc_degradation_threshold": 0.03, - "coverage_degradation_threshold": 0.02, - }, - "challenger": {"criteria": [], "promotion_requires": []}, - "artifacts": { - "pipeline_summary": str(data_dir / "pipeline_summary.json"), - "conformal_status": str(model_dir / "conformal_policy_status.json"), - "governance_status": str(model_dir / "governance_status.json"), - "fairness_status": str(model_dir / "fairness_audit_status.json"), - }, - "output": { - "mrm_report_json": str(report_dir / "mrm_validation_report.json"), - "mrm_status_json": str(model_dir / "mrm_report_status.json"), - }, - } - cfg_path = tmp_path / "mrm_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - mrm_mod.main(str(cfg_path)) - - report = json.loads((report_dir / "mrm_validation_report.json").read_text(encoding="utf-8")) - summary = report["compliance_summary"] - assert summary["overall_pass"] is True - assert summary["n_passing"] == 4 - assert summary["subsystems"]["pipeline"] is True - assert "diagnostic_statuses" in report - assert "bootstrap_validation" in report["diagnostic_statuses"] - assert "calibration_mapping" in report["diagnostic_statuses"] - assert "model_shift" in report["diagnostic_statuses"] - assert "skops_governance" in report - - -def test_generate_mrm_report_exports_skops_sidecar(tmp_path, monkeypatch) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - report_dir = tmp_path / "reports" / "mrm" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - report_dir.mkdir(parents=True) - - model = LogisticRegression().fit(np.array([[0.0], [1.0]]), np.array([0, 1])) - with open(model_dir / "pd_logreg_baseline.pkl", "wb") as fh: - pickle.dump({"model": model, "feature_names": ["x"], "fill_values": {"x": 0.0}}, fh) - - (data_dir / "pipeline_summary.json").write_text( - json.dumps({"pipeline": {"batch_size": 10}, "pd_model": {"final_auc": 0.72}}), - encoding="utf-8", - ) - for name in [ - "conformal_policy_status.json", - "governance_status.json", - "fairness_audit_status.json", - ]: - (model_dir / name).write_text(json.dumps({"overall_pass": True}), encoding="utf-8") - - cfg = { - "model": { - "name": "CorePD", - "version": "1.0", - "owner": "owner", - "champion_artifact": "models/pd_canonical.cbm", - }, - "governance": {"validation_frequency_days": 90, "review_cadence": "quarterly"}, - "retraining_triggers": { - "psi_threshold": 0.25, - "auc_degradation_threshold": 0.03, - "coverage_degradation_threshold": 0.02, - }, - "challenger": {"criteria": [], "promotion_requires": []}, - "artifacts": { - "pipeline_summary": str(data_dir / "pipeline_summary.json"), - "conformal_status": str(model_dir / "conformal_policy_status.json"), - "governance_status": str(model_dir / "governance_status.json"), - "fairness_status": str(model_dir / "fairness_audit_status.json"), - }, - "output": { - "mrm_report_json": str(report_dir / "mrm_validation_report.json"), - "mrm_status_json": str(model_dir / "mrm_report_status.json"), - }, - } - cfg_path = tmp_path / "mrm_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - monkeypatch.chdir(tmp_path) - mrm_mod.main(str(cfg_path)) - - report = json.loads((report_dir / "mrm_validation_report.json").read_text(encoding="utf-8")) - skops_sidecar = report["skops_governance"] - assert skops_sidecar["exports"][0]["status"] == "exported" - assert (report_dir / "skops" / "pd_logreg_baseline.skops").exists() - assert (report_dir / "corepd_model_card.json").exists() - assert (report_dir / "corepd_model_card.html").exists() - - -def test_generate_mrm_report_prefers_explicit_or_env_run_tag_over_pipeline_summary( - tmp_path, monkeypatch -) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - report_dir = tmp_path / "reports" / "mrm" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - report_dir.mkdir(parents=True) - - (data_dir / "pipeline_summary.json").write_text( - json.dumps( - { - "run_tag": "stale-pipeline-run", - "pipeline": {"batch_size": 10}, - "pd_model": {"final_auc": 0.72}, - } - ), - encoding="utf-8", - ) - for name in [ - "conformal_policy_status.json", - "governance_status.json", - "fairness_audit_status.json", - ]: - (model_dir / name).write_text(json.dumps({"overall_pass": True}), encoding="utf-8") - - cfg = { - "model": {"name": "CorePD", "version": "1.0", "owner": "owner"}, - "governance": {"validation_frequency_days": 90, "review_cadence": "quarterly"}, - "retraining_triggers": {"psi_threshold": 0.25}, - "challenger": {"criteria": [], "promotion_requires": []}, - "artifacts": { - "pipeline_summary": str(data_dir / "pipeline_summary.json"), - "conformal_status": str(model_dir / "conformal_policy_status.json"), - "governance_status": str(model_dir / "governance_status.json"), - "fairness_status": str(model_dir / "fairness_audit_status.json"), - }, - "output": { - "mrm_report_json": str(report_dir / "mrm_validation_report.json"), - "mrm_status_json": str(model_dir / "mrm_report_status.json"), - }, - } - cfg_path = tmp_path / "mrm_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("PIPELINE_RUN_TAG", "env-run-tag") - mrm_mod.main(str(cfg_path)) - - report = json.loads((report_dir / "mrm_validation_report.json").read_text(encoding="utf-8")) - assert report["run_tag"] == "env-run-tag" diff --git a/tests/test_scripts/test_manage_ijds_dvc_capsule.py b/tests/test_scripts/test_manage_ijds_dvc_capsule.py new file mode 100644 index 0000000..db9b055 --- /dev/null +++ b/tests/test_scripts/test_manage_ijds_dvc_capsule.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest +import yaml + +from scripts.manage_ijds_dvc_capsule import active_dvc_pointers, verify_remote + +RUN_TAGS = ( + "v4-v1", + "v4-v2", + "two-ruler-v1c", + "two-ruler-v2", + "credit-controls-v1b", + "credit-controls-v2b", +) + + +def _targets(tmp_path: Path, *, omit_last: bool = False) -> Path: + pointers: list[str] = [] + for run_tag in RUN_TAGS: + for prefix in ("data/processed", "models"): + relative = f"{prefix}/experiments/ijds_audit/{run_tag}.dvc" + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.safe_dump( + { + "outs": [ + { + "md5": f"{'a' * 32}.dir", + "size": 1, + "nfiles": 1, + "hash": "md5", + "path": run_tag, + } + ] + }, + sort_keys=False, + ), + encoding="utf-8", + ) + pointers.append(relative) + if omit_last: + pointers.pop() + lineage_names = ( + ("binary_geometry", "outcome_free", RUN_TAGS[0]), + ("binary_geometry", "evaluation", RUN_TAGS[1]), + ("two_ruler", "outcome_free", RUN_TAGS[2]), + ("two_ruler", "evaluation", RUN_TAGS[3]), + ("credit_controls", "outcome_free", RUN_TAGS[4]), + ("credit_controls", "evaluation", RUN_TAGS[5]), + ) + lineages: dict[str, dict[str, dict[str, str]]] = {} + for family, phase, run_tag in lineage_names: + lineages.setdefault(family, {})[phase] = { + "run_tag": run_tag, + "protocol_tag": f"protocol/{run_tag}", + "protocol_commit": "a" * 40, + "scientific_uv_lock_sha256": "b" * 64, + } + registry = tmp_path / "registry.yaml" + registry.write_text( + yaml.safe_dump( + { + "schema_version": "test", + "status": "active_ijds_paper_evidence_source_registry", + "lineages": lineages, + "dvc_pointers": pointers, + "sources": {"placeholder": {"path": "unused", "bytes": 0, "sha256": "0" * 64}}, + } + ), + encoding="utf-8", + ) + targets = tmp_path / "targets.yaml" + targets.write_text( + yaml.safe_dump( + { + "active_scientific_contract": { + "source_registry": registry.name, + } + } + ), + encoding="utf-8", + ) + return targets + + +def test_active_dvc_pointers_loads_two_pointers_per_active_run(tmp_path: Path) -> None: + pointers = active_dvc_pointers(root=tmp_path, targets_path=_targets(tmp_path)) + + assert len(pointers) == 12 + assert all(path.is_file() and path.suffix == ".dvc" for path in pointers) + + +def test_active_dvc_pointers_rejects_incomplete_capsule(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="do not match"): + active_dvc_pointers(root=tmp_path, targets_path=_targets(tmp_path, omit_last=True)) + + +def test_verify_remote_accepts_empty_cloud_status( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pointer = tmp_path / "active.dvc" + pointer.write_text("outs: []\n", encoding="utf-8") + observed: list[list[str]] = [] + + def fake_run(command: list[str], **_: object) -> subprocess.CompletedProcess[str]: + observed.append(command) + return subprocess.CompletedProcess(command, 0, stdout="{}\n", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + verify_remote(root=tmp_path, pointers=[pointer]) + + assert observed == [ + [ + "dvc", + "status", + "--cloud", + "--remote", + "dagshub", + "--json", + "active.dvc", + ] + ] + + +def test_verify_remote_rejects_missing_remote_objects( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pointer = tmp_path / "active.dvc" + pointer.write_text("outs: []\n", encoding="utf-8") + + def fake_run(command: list[str], **_: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + command, + 0, + stdout='{"data/run": "new"}\n', + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + with pytest.raises(RuntimeError, match="absent in the configured remote"): + verify_remote(root=tmp_path, pointers=[pointer]) diff --git a/tests/test_scripts/test_optimize_portfolio_tradeoff.py b/tests/test_scripts/test_optimize_portfolio_tradeoff.py deleted file mode 100644 index 688add0..0000000 --- a/tests/test_scripts/test_optimize_portfolio_tradeoff.py +++ /dev/null @@ -1,241 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from scripts.optimize_portfolio import _align_candidates_and_intervals -from scripts.optimize_portfolio_tradeoff import ( - _align_loans_and_intervals, - _build_policy_grid, - _prepare_tradeoff_inputs, - _select_champion_policy, - _solve_single, -) -from src.optimization import policy_evaluation - - -def test_portfolio_alignment_wrappers_share_strict_id_contract() -> None: - candidates = pd.DataFrame( - { - "id": ["b", "a", "c"], - "grade": ["B", "A", "C"], - "y_pred": [9.0, 8.0, 7.0], - } - ) - intervals = pd.DataFrame( - { - "id": ["a", "b", "c"], - "grade": ["score_q01", "score_q02", "score_q03"], - "y_pred": [0.1, 0.2, 0.3], - "pd_low_90": [0.05, 0.15, 0.25], - "pd_high_90": [0.15, 0.25, 0.35], - } - ) - - tradeoff_loans, tradeoff_intervals = _align_loans_and_intervals( - candidates, - intervals, - max_candidates=0, - random_state=42, - ) - optimizer_loans, pd_point, pd_low, pd_high = _align_candidates_and_intervals( - candidates, - intervals, - max_candidates=0, - random_state=42, - ) - - assert tradeoff_loans["id"].tolist() == ["b", "a", "c"] - assert tradeoff_intervals["grade"].tolist() == [ - "score_q02", - "score_q01", - "score_q03", - ] - pd.testing.assert_frame_equal(tradeoff_loans, optimizer_loans) - np.testing.assert_allclose(pd_point, [0.2, 0.1, 0.3]) - np.testing.assert_allclose(pd_low, [0.15, 0.05, 0.25]) - np.testing.assert_allclose(pd_high, [0.25, 0.15, 0.35]) - - -def test_nonrobust_solve_uses_point_pd_contract(monkeypatch: pytest.MonkeyPatch) -> None: - captured: dict[str, object] = {} - - def _fake_optimize_portfolio_allocation(**kwargs: object) -> dict[str, object]: - captured.update(kwargs) - return { - "allocation": {1: 0.5}, - "objective_value": 10.0, - "n_funded": 1, - "pd_cap_slack": 0.0, - "solver_status": "optimal", - } - - monkeypatch.setattr( - policy_evaluation, - "optimize_portfolio_allocation", - _fake_optimize_portfolio_allocation, - ) - loans = pd.DataFrame({"loan_amnt": [1000.0, 2000.0]}) - pd_point = np.array([0.10, 0.20]) - pd_high = np.array([0.40, 0.50]) - - result, allocation = _solve_single( - loans=loans, - pd_point=pd_point, - pd_low=np.array([0.05, 0.10]), - pd_high=pd_high, - lgd=np.array([0.45, 0.45]), - int_rates=np.array([0.10, 0.12]), - default_flag=np.array([0, 0]), - total_budget=1000.0, - max_concentration=1.0, - risk_tolerance=0.25, - robust=False, - uncertainty_aversion=0.0, - min_budget_utilization=0.0, - pd_cap_slack_penalty=0.0, - time_limit=10, - threads=1, - policy_mode="hard_worst_case", - gamma=1.0, - ) - - np.testing.assert_allclose(captured["pd_constraint_override"], pd_point) - np.testing.assert_allclose(allocation, [0.0, 0.5]) - assert result["policy_mode"] == "point_estimate" - assert result["gamma"] == 0.0 - - -def test_build_policy_grid_preserves_tradeoff_frontier_contract() -> None: - grid = _build_policy_grid() - - assert len(grid) == 63 - assert len(grid) == len(set(grid)) - assert grid[0] == ("hard_worst_case", 1.0, 1.0, 1.0) - assert {mode for mode, _, _, _ in grid} == { - "hard_worst_case", - "blended_uncertainty", - "capped_blended_uncertainty", - "tail_blended_uncertainty", - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - } - assert all(0.0 <= gamma <= 1.0 for _, gamma, _, _ in grid) - assert {delta_cap for _, _, delta_cap, _ in grid} == {0.50, 0.75, 0.90, 1.0} - assert {tail_focus for _, _, _, tail_focus in grid} == {0.75, 0.90, 0.95, 1.0} - - -def test_prepare_tradeoff_inputs_resolves_modern_interval_columns() -> None: - loans = pd.DataFrame( - { - "int_rate": ["10.5%", "8.0%", None], - "default_flag": [0, 1, None], - } - ) - intervals = pd.DataFrame( - { - "y_pred": [0.10, 0.20, 0.30], - "pd_low_90": [0.05, 0.10, 0.20], - "pd_high_90": [0.15, 0.30, 0.40], - } - ) - - prepared = _prepare_tradeoff_inputs(loans, intervals) - - np.testing.assert_allclose(prepared.pd_point, [0.10, 0.20, 0.30]) - np.testing.assert_allclose(prepared.pd_low, [0.05, 0.10, 0.20]) - np.testing.assert_allclose(prepared.pd_high, [0.15, 0.30, 0.40]) - np.testing.assert_allclose(prepared.lgd, [0.45, 0.45, 0.45]) - np.testing.assert_allclose(prepared.int_rates, [0.105, 0.08, 0.12]) - np.testing.assert_array_equal(prepared.default_flag, [0, 1, 0]) - - -def test_prepare_tradeoff_inputs_uses_defaults_for_optional_loan_columns() -> None: - loans = pd.DataFrame({"loan_amnt": [1000.0, 2000.0]}) - intervals = pd.DataFrame( - { - "pd_point": [0.11, 0.22], - "pd_low": [0.01, 0.02], - "pd_high": [0.31, 0.42], - } - ) - - prepared = _prepare_tradeoff_inputs(loans, intervals) - - np.testing.assert_allclose(prepared.pd_point, [0.11, 0.22]) - np.testing.assert_allclose(prepared.pd_low, [0.01, 0.02]) - np.testing.assert_allclose(prepared.pd_high, [0.31, 0.42]) - np.testing.assert_allclose(prepared.int_rates, [0.12, 0.12]) - np.testing.assert_array_equal(prepared.default_flag, [0, 0]) - - -def test_select_champion_policy_exposes_dual_selectors() -> None: - frontier = pd.DataFrame( - [ - { - "policy": "nonrobust", - "risk_tolerance": 0.10, - "policy_mode": "point_estimate", - "gamma": 0.0, - "uncertainty_aversion": 0.0, - "realized_total_return": 100.0, - "price_of_robustness": 0.0, - "ab_pass": True, - "n_funded": 10, - "solver_backend": "cuopt", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "pd_cap_slack": 0.0, - "ab_diff_total_return": 0.0, - }, - { - "policy": "robust", - "risk_tolerance": 0.10, - "policy_mode": "blended_uncertainty", - "gamma": 0.0, - "uncertainty_aversion": 0.0, - "realized_total_return": 120.0, - "price_of_robustness": -10.0, - "ab_pass": True, - "n_funded": 11, - "solver_backend": "cuopt", - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "pd_cap_slack": 0.0, - "ab_diff_total_return": 20.0, - }, - { - "policy": "robust", - "risk_tolerance": 0.10, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "uncertainty_aversion": 0.5, - "realized_total_return": 95.0, - "price_of_robustness": 5.0, - "ab_pass": True, - "n_funded": 9, - "solver_backend": "cuopt", - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "pd_cap_slack": 0.0, - "ab_diff_total_return": -5.0, - }, - ] - ) - - frontier_out, selected, robust_selected, balanced_selected, guardrail_selected = ( - _select_champion_policy(frontier) - ) - - assert selected["gamma"] == 0.0 - assert robust_selected is not None - assert robust_selected["gamma"] == 0.5 - assert balanced_selected is not None - assert balanced_selected["gamma"] == 0.5 - assert guardrail_selected is not None - assert guardrail_selected["gamma"] == 0.5 - assert frontier_out["selected_for_champion"].sum() == 1 - assert frontier_out["selected_for_robustness_aware"].sum() == 1 - assert frontier_out["selected_for_balanced_robustness"].sum() == 1 - assert frontier_out["selected_for_guardrail_robustness"].sum() == 1 diff --git a/tests/test_scripts/test_pool93_local_refinement_grid.py b/tests/test_scripts/test_pool93_local_refinement_grid.py deleted file mode 100644 index 55c4087..0000000 --- a/tests/test_scripts/test_pool93_local_refinement_grid.py +++ /dev/null @@ -1,267 +0,0 @@ -from __future__ import annotations - -import hashlib - -import pandas as pd -import pytest - -from scripts.search.run_pool93_ijds_local_refinement import ( - DEFAULT_ALPHA_GRID, - _build_parser, - _claim_summary, - _generate_candidate_grid, - _manifest_payload, - _pending_refinement_tasks, - _resolve_paths, -) -from src.optimization.certificate_semantics import IJDS_DECLARED_ALPHA_GRID - -EXPECTED_PROFILE_FINGERPRINTS = { - "stage1": (1236, "4f4fa9791ad71b3901f0af5aa55ff616426700e3245e6d5fd4cd5e923086f6ad"), - "expanded": (7463, "a0bb03c5ee6491b2f9e1032e50c12c8aae6937a84206a989889acbb4212d371a"), - "claim_expanded": ( - 3659, - "8345545b20e93985462a84e92bf503417911386ab1aaf0194a657aab64d8d329", - ), - "claim_micro": (2949, "1e4083c8b8e200c5689566a06da1e18591a11b219b13ffea5c65d575dfb796bc"), - "claim_micro_ext": ( - 4407, - "3cc8a45b2ca0fde9a2f12cd6229b94cc7bf0119e3eeb7e742d6ce2f7bf601d08", - ), - "claim_bound_closure": ( - 1653, - "8d8ef58d92049809406049ff22b81831a9607c0203a2c401fb026826f2a9acee", - ), - "claim_bound_floor_closure": ( - 2343, - "9cb18594d1ec323cab23ffcdb4c96e481b2fb0798a18c4df73b796737e6b72c5", - ), - "claim_bound_terminal": ( - 37068, - "6d75ef0b7c083f9f60dfc834a50cb5da10873223dffe340665ef130e6c4c88ac", - ), -} - - -def test_pool93_default_alpha_grid_uses_shared_certificate_semantics() -> None: - assert list(IJDS_DECLARED_ALPHA_GRID) == DEFAULT_ALPHA_GRID - - -def _synthetic_anchor_rows() -> pd.DataFrame: - base_fields = { - "tail_focus_quantile": 1.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - } - return pd.DataFrame( - [ - { - **base_fields, - "candidate_rank": 96, - "risk_tolerance": 0.156, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.46, - "uncertainty_aversion": 0.125, - "delta_cap_quantile": 1.0, - }, - { - **base_fields, - "candidate_rank": 219, - "risk_tolerance": 0.171, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "uncertainty_aversion": 0.1, - "delta_cap_quantile": 1.0, - }, - { - **base_fields, - "candidate_rank": 223, - "risk_tolerance": 0.173, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.40, - "uncertainty_aversion": 0.1, - "delta_cap_quantile": 0.95, - }, - ] - ) - - -def _semantic_fingerprint(frame: pd.DataFrame) -> str: - blob = "\n".join(frame["semantic_policy_key"].astype(str)).encode() - return hashlib.sha256(blob).hexdigest() - - -@pytest.mark.parametrize("profile", EXPECTED_PROFILE_FINGERPRINTS) -def test_pool93_local_refinement_grid_is_stable_by_profile(profile: str) -> None: - expected_rows, expected_sha = EXPECTED_PROFILE_FINGERPRINTS[profile] - - candidates = _generate_candidate_grid( - _synthetic_anchor_rows(), - profile=profile, - solver_backend="highspy", - ) - - assert len(candidates) == expected_rows - assert _semantic_fingerprint(candidates) == expected_sha - assert candidates["local_candidate_id"].tolist() == list(range(1, expected_rows + 1)) - assert candidates["semantic_policy_key"].is_unique - - -def test_pool93_local_refinement_rejects_unknown_profile() -> None: - with pytest.raises(ValueError, match="profile must be one of"): - _generate_candidate_grid( - _synthetic_anchor_rows(), - profile="broad_new_search", - solver_backend="highspy", - ) - - -def test_pool93_manifest_paths_and_pending_tasks_are_coherent(tmp_path) -> None: - args = _build_parser().parse_args( - [ - "--run-tag", - "unit/run", - "--output-dir", - str(tmp_path / "out"), - "--model-dir", - str(tmp_path / "model"), - "--source-bound-eval", - str(tmp_path / "source.parquet"), - "--source-selection", - str(tmp_path / "selection.json"), - ] - ) - paths = _resolve_paths(args, run_tag="unit_run") - candidates = pd.DataFrame({"local_candidate_id": [1, 2]}) - - pending = _pending_refinement_tasks( - candidates=candidates, - alpha_grid=[0.01, 0.03], - completed_keys={(1, 0.01)}, - ) - manifest = _manifest_payload( - args=args, - paths=paths, - run_tag="unit_run", - source_bound_eval=tmp_path / "source.parquet", - source_selection=tmp_path / "selection.json", - conformal_intervals_path="data/processed/conformal.parquet", - anchor_ranks=[96, 219, 223], - alpha_grid=[0.01, 0.03], - ) - - assert [(row["local_candidate_id"], alpha) for row, alpha in pending] == [ - (1, 0.03), - (2, 0.01), - (2, 0.03), - ] - assert manifest["candidates_path"] == str(paths.candidates_path) - assert manifest["claim_summary_path"] == str(paths.claim_summary_path) - assert manifest["run_tag"] == "unit_run" - - -def test_claim_summary_exposes_finite_grid_and_balanced_claim() -> None: - leaderboard = pd.DataFrame( - [ - { - "claim_rank": 1, - "local_candidate_id": 1, - "local_family": "endpoint", - "anchor_rank": 96, - "source_reason": "max_return", - "risk_tolerance": 0.18, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.40, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05, - "alpha01_realized_total_return": 190000.0, - "alpha01_gamma_cp": 0.20, - "alpha01_weighted_miscoverage_V": 0.05, - "alpha01_weighted_pd_true": 0.11, - "alpha01_empirical_coverage_funded": 0.92, - "alpha01_exact_pass": True, - "all_alpha_pass": True, - "alpha_exact_pass_count": 2, - "alpha_exact_check_count": 2, - "alpha_mean_gamma_cp": 0.18, - "alpha_mean_weighted_miscoverage_V": 0.04, - "n_funded_mean": 50, - "allocator_backends": "highspy", - }, - { - "claim_rank": 2, - "local_candidate_id": 2, - "local_family": "body", - "anchor_rank": 219, - "source_reason": "balanced", - "risk_tolerance": 0.17, - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.55, - "delta_cap_quantile": 0.975, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.05, - "alpha01_realized_total_return": 185000.0, - "alpha01_gamma_cp": 0.10, - "alpha01_weighted_miscoverage_V": 0.03, - "alpha01_weighted_pd_true": 0.10, - "alpha01_empirical_coverage_funded": 0.94, - "alpha01_exact_pass": True, - "all_alpha_pass": True, - "alpha_exact_pass_count": 2, - "alpha_exact_check_count": 2, - "alpha_mean_gamma_cp": 0.11, - "alpha_mean_weighted_miscoverage_V": 0.03, - "n_funded_mean": 48, - "allocator_backends": "highspy", - }, - { - "claim_rank": 3, - "local_candidate_id": 3, - "local_family": "failed", - "anchor_rank": 223, - "source_reason": "not_all_alpha", - "risk_tolerance": 0.16, - "policy_mode": "blended_uncertainty", - "gamma": 0.60, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.10, - "alpha01_realized_total_return": 200000.0, - "alpha01_gamma_cp": 0.09, - "alpha01_weighted_miscoverage_V": 0.02, - "alpha01_weighted_pd_true": 0.09, - "alpha01_empirical_coverage_funded": 0.95, - "alpha01_exact_pass": True, - "all_alpha_pass": False, - "alpha_exact_pass_count": 1, - "alpha_exact_check_count": 2, - "alpha_mean_gamma_cp": 0.10, - "alpha_mean_weighted_miscoverage_V": 0.02, - "n_funded_mean": 45, - "allocator_backends": "highspy", - }, - ] - ) - bound_eval = pd.DataFrame( - { - "alpha": [0.01, 0.01, 0.03], - "all_bounds_hold": [True, False, True], - "violation": [0.0, 0.02, 0.0], - "gamma_cp": [0.1, 0.2, 0.15], - "weighted_miscoverage_V": [0.03, 0.05, 0.04], - } - ) - - summary = _claim_summary(leaderboard, bound_eval, alpha_grid=[0.03, 0.01, 0.01]) - - assert summary["finite_grid_policy"]["alpha_grid"] == [0.01, 0.03] - assert summary["n_policies"] == 3 - assert summary["n_all_alpha_passers"] == 2 - assert summary["n_all_alpha_passers_above_return_floor"] == 2 - assert summary["max_return_claim"]["local_candidate_id"] == 1 - assert summary["best_gamma_cp_return_floor_claim"]["local_candidate_id"] == 2 - assert summary["best_weighted_miscoverage_return_floor_claim"]["local_candidate_id"] == 2 - assert summary["balanced_return_bound_claim"]["local_candidate_id"] == 2 - assert summary["by_family"]["failed"]["n_all_alpha_passers"] == 0 - assert summary["by_alpha"]["0.01"]["n_checks"] == 2 diff --git a/tests/test_scripts/test_retired_search_entrypoints.py b/tests/test_scripts/test_retired_search_entrypoints.py deleted file mode 100644 index 3a2b2b9..0000000 --- a/tests/test_scripts/test_retired_search_entrypoints.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from scripts.search import run_conformal_search, run_portfolio_search - - -def test_retired_conformal_search_entrypoint_is_actionable(capsys) -> None: - assert run_conformal_search.main([]) == 2 - - captured = capsys.readouterr() - assert "retired" in captured.err - assert "run_conformal_reopen_search.py" in captured.err - - -def test_retired_portfolio_search_entrypoint_is_actionable(capsys) -> None: - assert run_portfolio_search.main([]) == 2 - - captured = capsys.readouterr() - assert "retired" in captured.err - assert "run_pool93_ijds_local_refinement.py" in captured.err diff --git a/tests/test_scripts/test_run_comparison.py b/tests/test_scripts/test_run_comparison.py deleted file mode 100644 index 30c945b..0000000 --- a/tests/test_scripts/test_run_comparison.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -from scripts.run_comparison import _collect_status_metadata, _gate_ab_no_regression - - -def _ab_status(control: float, robust: float, **extra: object) -> dict[str, object]: - return { - "metrics_a": {"total_return": control}, - "metrics_b": {"total_return": robust}, - **extra, - } - - -def _status(run_tag: str) -> dict[str, str]: - return { - "schema_version": "test.1", - "generated_at_utc": "2026-07-08T20:00:00+00:00", - "run_tag": run_tag, - } - - -def test_collect_status_metadata_passes_when_core_statuses_match() -> None: - cur = { - "dvc_metrics_meta": _status("run-a"), - "pipeline_summary": _status("run-a"), - "conformal_status": _status("run-a"), - "fairness_status": _status("run-a"), - "governance_status": _status("run-a"), - "ab_simulation_status": _status("run-a"), - } - - metadata = _collect_status_metadata(cur, expected_run_tag="run-a") - - assert metadata["passed"] is True - assert metadata["all_have_metadata"] is True - assert metadata["run_tags_observed"] == ["run-a"] - assert metadata["mismatched_run_tag_artifacts"] == [] - assert len(metadata["critical_artifacts"]) == 6 - - -def test_collect_status_metadata_allows_causal_insights_only_mismatch() -> None: - cur = { - "dvc_metrics_meta": _status("run-a"), - "pipeline_summary": _status("run-a"), - "conformal_status": _status("run-a"), - "fairness_status": _status("run-a"), - "governance_status": _status("run-a"), - "ab_simulation_status": _status("run-a"), - "causal_effect_status": _status("older-causal-run"), - } - - metadata = _collect_status_metadata(cur, expected_run_tag="run-a") - - assert metadata["passed"] is True - assert metadata["run_tag_matches_expected"] is False - assert metadata["run_tag_matches_expected_operational"] is True - assert metadata["causal_only_mismatch"] is True - assert metadata["non_causal_mismatched_run_tag_artifacts"] == [] - - -def test_gate_ab_no_regression_passes_when_current_self_gate_passes() -> None: - base = {"ab_simulation_status": _ab_status(100.0, 120.0)} - cur = { - "ab_simulation_status": _ab_status( - 101.0, - 119.0, - no_regression={ - "passed": True, - "diff_total_return": 18.0, - "tolerance_total_return": 5.0, - }, - comparison={"p_value": 0.12, "significant": False}, - n_candidates_used=50, - ) - } - - gate = _gate_ab_no_regression(base, cur) - - assert gate.name == "ab_no_regression" - assert gate.passed is True - assert gate.details["checks"]["self_no_regression_ok"] is True - assert gate.details["warnings"]["control_vs_baseline_warning"] is False - assert gate.details["diagnostics"]["gate_mode"] == "no_regression" - - -def test_gate_ab_no_regression_allows_selective_ambiguity_cross_gate() -> None: - base = {"ab_simulation_status": _ab_status(100.0, 120.0)} - cur = { - "ab_simulation_status": _ab_status( - 100.0, - 80.0, - decision_scenario="selective_ambiguity_defer", - no_regression={"passed": False, "diff_total_return": -20.0}, - cross_scenario_gate={"passed": True}, - ) - } - - gate = _gate_ab_no_regression(base, cur) - - assert gate.passed is True - assert gate.details["checks"]["self_no_regression_ok"] is True - assert gate.details["checks"]["cross_scenario_gate_ok"] is True - assert gate.details["warnings"]["robust_vs_baseline_warning"] is True diff --git a/tests/test_scripts/test_run_complexity_report.py b/tests/test_scripts/test_run_complexity_report.py new file mode 100644 index 0000000..3303dfd --- /dev/null +++ b/tests/test_scripts/test_run_complexity_report.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from scripts.run_complexity_report import ( + RADON_REQUIREMENT, + build_radon_command, + isolated_uvx_environment, +) +from scripts.run_ty_advisory import iter_python_files + + +def test_complexity_report_uses_only_declared_active_files() -> None: + files = iter_python_files(scope="active") + command = build_radon_command(uvx="uvx", files=files) + + assert command[:4] == ["uvx", "--from", RADON_REQUIREMENT, "radon"] + assert "scripts/build_ijds_binary_geometry_frontier_v4_evidence.py" in command + assert "scripts/generate_conformal_intervals.py" not in command + assert "src/data/make_dataset.py" not in command + assert command[-3:] == ["--show-complexity", "--min", "C"] + + +def test_complexity_report_isolates_uvx_interpreter_environment() -> None: + cleaned = isolated_uvx_environment( + {"PATH": "bin", "PYTHONHOME": "managed-python", "VIRTUAL_ENV": ".venv"} + ) + + assert cleaned == {"PATH": "bin"} diff --git a/tests/test_scripts/test_run_conformal_reopen_search.py b/tests/test_scripts/test_run_conformal_reopen_search.py deleted file mode 100644 index d1348b3..0000000 --- a/tests/test_scripts/test_run_conformal_reopen_search.py +++ /dev/null @@ -1,397 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pandas as pd - -from scripts.search import run_conformal_reopen_search as reopen -from scripts.search.run_conformal_reopen_search import ( - Phase1ConfirmationResult, - _maybe_apply_phase2, - _phase1_from_resume, - _phase2_metric_blocked, - _phase2_run_reason, - _phase2_top_designs, - _rank_phase2_candidates, - _run_phase1_oot_confirmation, - _run_phase2_search, -) - - -def _design(*, partition: str, alpha: float, width: float, rank: int) -> dict[str, object]: - return { - "partition": partition, - "partition_probability_source": "calibrated", - "n_score_bins": 10, - "fallback_mode": "grade_then_global", - "alpha_used_90": alpha, - "alpha_used_95": 0.05, - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 500, - "calibration_fraction": 0.50, - "avg_width_90": width, - "selection_rank": rank, - } - - -def test_phase2_top_designs_prefers_phase1_oot_confirmed_candidates() -> None: - inner = pd.DataFrame( - [ - _design(partition="score_decile_mondrian", alpha=0.085, width=0.79, rank=1), - _design(partition="grade", alpha=0.095, width=0.77, rank=2), - ] - ) - oot = pd.DataFrame( - [ - _design(partition="grade_x_scoreband_mondrian", alpha=0.075, width=0.78, rank=1), - ] - ) - - top, source = _phase2_top_designs( - aggregated=inner, - phase1_candidates_frame=oot, - top_k=2, - ) - - assert source == "phase1_oot_confirmed" - assert top.iloc[0]["partition"] == "grade_x_scoreband_mondrian" - assert set(top["phase2_design_source"]) == {"phase1_oot_confirmed"} - - -def test_phase2_top_designs_falls_back_to_inner_when_oot_empty() -> None: - inner = pd.DataFrame( - [ - _design(partition="score_decile_mondrian", alpha=0.085, width=0.79, rank=1), - ] - ) - - top, source = _phase2_top_designs( - aggregated=inner, - phase1_candidates_frame=pd.DataFrame(), - top_k=2, - ) - - assert source == "phase1_inner_aggregate" - assert top.iloc[0]["partition"] == "score_decile_mondrian" - assert set(top["phase2_design_source"]) == {"phase1_inner_aggregate"} - - -def _passing_policy(*, width: float = 0.42) -> dict[str, object]: - return { - "overall_pass": True, - "strict_overall_pass": True, - "methodological_justification_pass": True, - "coverage_90": 0.90, - "avg_width_90": width, - "min_group_coverage_90": 0.89, - "warning_alerts": 0, - "total_alerts": 0, - } - - -def test_phase1_resume_result_preserves_source_paths_and_winner( - tmp_path: Path, - monkeypatch: object, -) -> None: - aggregate = pd.DataFrame( - [ - _design(partition="grade", alpha=0.10, width=0.40, rank=1), - _design(partition="score_decile_mondrian", alpha=0.09, width=0.41, rank=2), - ] - ) - source_paths = { - "inner_aggregate": tmp_path / "source_aggregate.parquet", - "inner_search": tmp_path / "source_inner.parquet", - } - aggregate.to_parquet(source_paths["inner_aggregate"], index=False) - - shortlist = aggregate.head(1).copy() - - def fake_paths(_run_tag: str) -> dict[str, Path]: - return source_paths - - def fake_shortlist( - *, source_run_tag: str, top_k_inner: int - ) -> tuple[pd.DataFrame, dict[str, Any]]: - assert source_run_tag == "source-run" - assert top_k_inner == 1 - return shortlist, {"source": "source-run"} - - monkeypatch.setattr(reopen, "_reopen_artifact_paths", fake_paths) - monkeypatch.setattr(reopen, "_build_resume_shortlist", fake_shortlist) - - result = _phase1_from_resume( - resume_from_run_tag="source-run", - top_k_inner=1, - output_paths={"phase1_shortlist": tmp_path / "shortlist.parquet"}, - ) - - assert result.aggregate_path == str(source_paths["inner_aggregate"]) - assert result.inner_search_path == str(source_paths["inner_search"]) - assert result.inner_search_winner["partition"] == "grade" - assert result.resume_meta == {"source": "source-run"} - assert (tmp_path / "shortlist.parquet").exists() - - -def test_phase1_oot_confirmation_writes_ranked_candidate_frame( - tmp_path: Path, - monkeypatch: object, -) -> None: - shortlist = pd.DataFrame( - [ - _design(partition="grade", alpha=0.10, width=0.50, rank=1), - _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), - ] - ) - - def fake_run_candidate(**kwargs: Any) -> dict[str, object]: - rank = int(kwargs["rank"]) - return { - "namespace": f"candidate-{rank}", - "policy_status": _passing_policy(width=0.50 if rank == 1 else 0.42), - "set_status": {"summary": {"set_coverage": 0.91, "singleton_rate": 0.73}}, - "selection_status": {"promotion_pass": rank == 2, "selected_variant": "variant"}, - } - - monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_run_candidate) - - result = _run_phase1_oot_confirmation( - run_tag="test-run", - env={}, - shortlist=shortlist, - output_paths={"phase1_final_candidates": tmp_path / "phase1_final.parquet"}, - alpha_candidates_95=[0.05], - partition_candidates=["grade"], - partition_probability_sources=["calibrated"], - n_score_bins_candidates=[10], - fallback_modes=["grade_then_global"], - score_scale_families=["none"], - calibration_fractions=[1.0], - sidecar_cfg={}, - validation_cfg={}, - ) - - assert result.best_namespace == "candidate-2" - assert result.final_namespace == "candidate-2" - assert result.final_decision == "promotable_for_followup" - assert result.frame.iloc[0]["avg_width_90"] == 0.42 - assert (tmp_path / "phase1_final.parquet").exists() - - -def test_maybe_apply_phase2_respects_phase1_only() -> None: - phase1 = Phase1ConfirmationResult( - candidates=[], - frame=pd.DataFrame([{"namespace": "phase1"}]), - best_namespace="phase1", - final_policy=_passing_policy(), - final_sets={"summary": {"set_coverage": 0.91}}, - final_decision="promotable_for_followup", - final_namespace="phase1", - ) - - result = _maybe_apply_phase2( - run_tag="run", - upstream_run_tag="upstream", - env={}, - aggregated=pd.DataFrame(), - phase1=phase1, - phase1_only=True, - force_phase2=True, - alpha_candidates_95=[], - tuning_holdout_ratios=[], - inner_random_states=[], - partition_candidates=[], - partition_probability_sources=[], - n_score_bins_candidates=[], - fallback_modes=[], - score_scale_families=[], - calibration_fractions=[], - phase2_cfg={"enabled": True}, - sidecar_cfg={}, - validation_cfg={}, - ) - - assert result.final_namespace == "phase1" - assert result.final_decision == "promotable_for_followup" - assert result.phase2_summary is None - assert _phase2_run_reason(force_phase2=True, phase2_always_evaluate=True) == "forced" - - -def test_phase2_metric_gate_and_ranking_are_explicit() -> None: - assert _phase2_metric_blocked( - calibration_metrics={"ece": 0.12, "brier_score": 0.21}, - baseline_metrics={"ece": 0.10, "brier_score": 0.20}, - max_metric_degradation={"ece": 0.01, "brier_score": 0.02}, - ) - assert not _phase2_metric_blocked( - calibration_metrics={"ece": 0.105}, - baseline_metrics={"ece": 0.10}, - max_metric_degradation={"ece": 0.01}, - ) - - ranked = _rank_phase2_candidates( - pd.DataFrame( - [ - { - "artifact_namespace": "wide", - "calibrator_method": "platt", - "holdout_coverage": 0.91, - "holdout_width": 0.30, - "calibrator_ece": 0.01, - "calibrator_adaptive_ece": 0.01, - "calibrator_brier": 0.10, - "calibrator_phi_brier": 0.20, - "selection_rank": 1, - }, - { - "artifact_namespace": "centered", - "calibrator_method": "isotonic", - "holdout_coverage": 0.90, - "holdout_width": 0.50, - "calibrator_ece": 0.02, - "calibrator_adaptive_ece": 0.02, - "calibrator_brier": 0.10, - "calibrator_phi_brier": 0.10, - "selection_rank": 2, - }, - ] - ) - ) - - assert ranked.iloc[0]["artifact_namespace"] == "centered" - - -def _phase2_paths(tmp_path: Path) -> dict[str, Path]: - data_dir = tmp_path / "data" - models_dir = tmp_path / "models" - data_dir.mkdir() - models_dir.mkdir() - return { - "data_dir": data_dir, - "models_dir": models_dir, - "phase2_search": data_dir / "phase2.parquet", - "phase2_progress": models_dir / "phase2_progress.json", - } - - -def test_run_phase2_search_records_metric_gate_skip( - tmp_path: Path, - monkeypatch: Any, -) -> None: - paths = _phase2_paths(tmp_path) - aggregate = pd.DataFrame([_design(partition="grade", alpha=0.10, width=0.40, rank=1)]) - - def fake_fit_calibrator( - *, method: str, output_path: Path, upstream_run_tag: str - ) -> tuple[str, dict[str, float]]: - del output_path, upstream_run_tag - if method == "venn_abers": - return method, {"ece": 0.10} - return method, {"ece": 0.20} - - monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) - monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) - - decision, policy, sets, summary = _run_phase2_search( - run_tag="phase2-skip", - upstream_run_tag="upstream", - env={}, - aggregated=aggregate, - phase1_candidates_frame=pd.DataFrame(), - alpha_candidates_95=[0.05], - tuning_holdout_ratios=[0.2], - inner_random_states=[42], - partition_candidates=["grade"], - partition_probability_sources=["calibrated"], - n_score_bins_candidates=[10], - fallback_modes=["grade_then_global"], - score_scale_families=["none"], - calibration_fractions=[1.0], - phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {"ece": 0.01}}, - sidecar_cfg={}, - validation_cfg={}, - ) - - progress = json.loads(paths["phase2_progress"].read_text(encoding="utf-8")) - assert decision == "policy_review_candidate" - assert policy == {} - assert sets == {} - assert summary is not None - assert summary["status"] == "no_noninferior_calibrator_candidate" - assert progress["skipped"][0]["reason"] == "metric_degradation_gate" - assert paths["phase2_search"].exists() - - -def test_run_phase2_search_ranks_and_confirms_best_candidate( - tmp_path: Path, - monkeypatch: Any, -) -> None: - paths = _phase2_paths(tmp_path) - aggregate = pd.DataFrame( - [ - _design(partition="grade", alpha=0.10, width=0.50, rank=1), - _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), - ] - ) - - def fake_fit_calibrator( - *, method: str, output_path: Path, upstream_run_tag: str - ) -> tuple[str, dict[str, float]]: - del output_path, upstream_run_tag - return method, {"ece": 0.02, "adaptive_ece": 0.03, "brier_score": 0.10} - - def fake_resolve_run_paths(namespace: str) -> dict[str, Path]: - return {"results": tmp_path / f"{namespace}.pkl"} - - def fake_load_pickle(path: Path) -> dict[str, dict[str, float]]: - if "rank-2" in path.name: - return {"metrics_90": {"empirical_coverage": 0.90, "avg_interval_width": 0.44}} - return {"metrics_90": {"empirical_coverage": 0.88, "avg_interval_width": 0.30}} - - def fake_final_candidate(**kwargs: Any) -> dict[str, object]: - assert kwargs["phase_prefix"] == "phase2" - assert kwargs["design"]["selection_rank"] == 2 - assert str(kwargs["calibrator_override_path"]).endswith("platt.pkl") - return { - "namespace": "phase2-final", - "policy_status": _passing_policy(width=0.44), - "set_status": {"summary": {"set_coverage": 0.91}}, - } - - monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) - monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) - monkeypatch.setattr(reopen, "_run_python", lambda *_args, **_kwargs: None) - monkeypatch.setattr(reopen, "_resolve_run_paths", fake_resolve_run_paths) - monkeypatch.setattr(reopen, "_load_pickle", fake_load_pickle) - monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_final_candidate) - - decision, policy, sets, summary = _run_phase2_search( - run_tag="phase2-success", - upstream_run_tag="upstream", - env={}, - aggregated=aggregate, - phase1_candidates_frame=pd.DataFrame(), - alpha_candidates_95=[0.05], - tuning_holdout_ratios=[0.2], - inner_random_states=[42], - partition_candidates=["grade"], - partition_probability_sources=["calibrated"], - n_score_bins_candidates=[10], - fallback_modes=["grade_then_global"], - score_scale_families=["none"], - calibration_fractions=[1.0], - phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {}}, - sidecar_cfg={}, - validation_cfg={}, - ) - - phase2_search = pd.read_parquet(paths["phase2_search"]) - assert decision == "promotable_for_followup" - assert policy["avg_width_90"] == 0.44 - assert sets["summary"]["set_coverage"] == 0.91 - assert summary is not None - assert summary["final_namespace"] == "phase2-final" - assert summary["best_candidate"]["selection_rank"] == 2 - assert phase2_search.iloc[0]["selection_rank"] == 2 diff --git a/tests/test_scripts/test_run_crpto_vs_spo_stability.py b/tests/test_scripts/test_run_crpto_vs_spo_stability.py deleted file mode 100644 index 7f483c0..0000000 --- a/tests/test_scripts/test_run_crpto_vs_spo_stability.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import json -from argparse import Namespace -from pathlib import Path - -import numpy as np -import pandas as pd -import pytest - -from scripts import run_crpto_vs_spo_stability as stability_mod - - -def test_crpto_vs_spo_stability_artifacts_exist() -> None: - status_path = Path("data/processed/crpto_vs_spo_stability.json") - assert status_path.exists() - status = json.loads(status_path.read_text(encoding="utf-8")) - assert status.get("schema_version") - assert Path("reports/crpto/figures/crpto_fig11_crpto_stability.png").exists() - - -def test_period_sample_seed_is_stable_and_distinct_by_period() -> None: - seed = 42 - - assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 - assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 - assert stability_mod._period_sample_seed(seed, "2020") == 500_042 - - -def test_detail_rows_and_summary_payload_preserve_period_contract() -> None: - periods = list(stability_mod.PERIODS) - test = pd.DataFrame({"default_flag": [0, 1, 0, 1, 0]}) - period_masks = { - period: np.array([idx == pos for idx in range(len(periods))]) - for pos, period in enumerate(periods) - } - regrets = stability_mod._init_period_regrets() - regrets["2018H1"]["two_stage"] = [2.0, 4.0] - regrets["2018H1"]["spo_plus"] = [1.0, 2.0] - regrets["2018H1"]["conformal_robust"] = [3.0, float("nan")] - coverage = { - "2018H1": { - "coverage_90": 0.91, - "coverage_95": 0.96, - "avg_width_90": 0.42, - "min_grade_coverage_90": 0.89, - } - } - - rows = stability_mod._detail_rows( - test=test, - period_masks=period_masks, - per_period_regrets=regrets, - period_coverage=coverage, - ) - summary = stability_mod._summary_payload( - run_tag="run-test", - args=Namespace(n_items=50, budget=15, n_train=800, epochs=50, seeds=2), - n_features=3, - feature_names=["a", "b", "c"], - rows=rows, - per_period_regrets=regrets, - total_time=12.34, - ) - - first = rows[0] - assert first["period"] == "2018H1" - assert first["two_stage_mean_regret"] == 3.0 - assert first["spo_plus_mean_regret"] == 1.5 - assert first["conformal_robust_mean_regret"] == 3.0 - assert first["spo_improvement_pct"] == pytest.approx(49.999999983333336) - assert summary["config"]["n_features"] == 3 - assert summary["per_period"]["2018H1"]["coverage_90"] == 0.91 - assert summary["per_period"]["2018H1"]["spo_improvement_vs_ts_pct"] == 50.0 - assert summary["stability_summary"]["coverage_always_above_target"] is True diff --git a/tests/test_scripts/test_run_fairness_audit.py b/tests/test_scripts/test_run_fairness_audit.py deleted file mode 100644 index f3322b5..0000000 --- a/tests/test_scripts/test_run_fairness_audit.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Tests for scripts/run_fairness_audit.py threshold resolution behavior.""" - -from __future__ import annotations - -import json - -import numpy as np -import pandas as pd -import pytest -import yaml - -from scripts import run_fairness_audit as fairness_mod - - -@pytest.fixture(autouse=True) -def _set_run_tag(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PIPELINE_RUN_TAG", "run-fairness-test") - - -def test_run_fairness_uses_threshold_artifact(tmp_path) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - preds = pd.DataFrame({"pd_calibrated": [0.20, 0.60, 0.30, 0.90]}) - test_df = pd.DataFrame( - { - "default_flag": [0, 1, 0, 1], - "home_ownership": ["RENT", "RENT", "OWN", "OWN"], - "annual_inc": [50_000, 60_000, 55_000, 90_000], - "verification_status": ["Verified", "Not Verified", "Verified", "Not Verified"], - } - ) - - pred_path = data_dir / "test_predictions.parquet" - data_path = data_dir / "test_fe.parquet" - preds.to_parquet(pred_path, index=False) - test_df.to_parquet(data_path, index=False) - - threshold_path = model_dir / "decision_threshold.json" - threshold_path.write_text(json.dumps({"selected_threshold": 0.70}), encoding="utf-8") - - cfg = { - "policy": { - "dpd_threshold": 0.5, - "eo_gap_threshold": 0.5, - "dir_threshold": 0.5, - "prediction_threshold": 0.50, - "outcome_mode": "approval", - }, - "threshold_policy": { - "use_artifact": True, - "artifact_path": str(threshold_path), - "selected_threshold_key": "selected_threshold", - }, - "decision_policy": { - "auto_select": False, - "artifact_path": str(model_dir / "missing_fairness_decision_policy.json"), - }, - "attributes": [ - {"name": "home_ownership", "column": "home_ownership"}, - {"name": "annual_inc_quartile", "column": "annual_inc", "binning": "quartile"}, - {"name": "verification_status", "column": "verification_status"}, - ], - "artifacts": { - "test_predictions_path": str(pred_path), - "test_data_path": str(data_path), - }, - "output": { - "audit_parquet": str(data_dir / "fairness_audit.parquet"), - "frontier_parquet": str(data_dir / "fairness_threshold_frontier.parquet"), - "status_json": str(model_dir / "fairness_audit_status.json"), - "threshold_semantics_json": str(model_dir / "threshold_semantics.json"), - }, - "fairlearn_sidecar": { - "enabled": True, - "status_json": str(model_dir / "fairlearn_fairness_status.json"), - "group_metrics_parquet": str(data_dir / "fairlearn_group_metrics.parquet"), - "bootstrap_samples": 5, - }, - "intersectional": {"enabled": True, "max_order": 2, "min_group_size": 1}, - "threshold_frontier": {"enabled": True, "window_radius": 0.10, "step": 0.10}, - } - - cfg_path = tmp_path / "fairness_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - fairness_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "fairness_audit_status.json").read_text(encoding="utf-8")) - fairlearn_status = json.loads( - (model_dir / "fairlearn_fairness_status.json").read_text(encoding="utf-8") - ) - fairlearn_groups = pd.read_parquet(data_dir / "fairlearn_group_metrics.parquet") - frontier = pd.read_parquet(data_dir / "fairness_threshold_frontier.parquet") - assert status["schema_version"] - assert status["generated_at_utc"] - assert status["run_tag"] - assert status["prediction_threshold"] == 0.70 - assert status["primary_threshold"] == 0.70 - assert status["prediction_threshold_source"] == "artifact" - assert status["outcome_mode"] == "approval" - assert status["n_intersectional_attributes"] > 0 - assert fairlearn_status["run_tag"] == "run-fairness-test" - assert fairlearn_status["n_attributes"] >= 1 - assert not fairlearn_groups.empty - assert not frontier.empty - assert frontier["is_primary_threshold"].any() - - -def test_run_fairness_auto_selects_threshold_and_writes_decision_policy(tmp_path) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - preds = pd.DataFrame({"pd_calibrated": [0.35, 0.45, 0.55, 0.65, 0.75, 0.85]}) - test_df = pd.DataFrame( - { - "default_flag": [0, 0, 1, 1, 0, 1], - "home_ownership": ["RENT", "RENT", "OWN", "OWN", "OWN", "RENT"], - "annual_inc": [50_000, 55_000, 60_000, 65_000, 70_000, 75_000], - "verification_status": [ - "Verified", - "Not Verified", - "Verified", - "Not Verified", - "Verified", - "Not Verified", - ], - } - ) - pred_path = data_dir / "test_predictions.parquet" - data_path = data_dir / "test_fe.parquet" - preds.to_parquet(pred_path, index=False) - test_df.to_parquet(data_path, index=False) - - cfg = { - "policy": { - "dpd_threshold": 0.5, - "eo_gap_threshold": 0.5, - "dir_threshold": 0.5, - "prediction_threshold": 0.50, - "outcome_mode": "approval", - }, - "threshold_policy": {"use_artifact": False}, - "decision_policy": { - "auto_select": True, - "artifact_path": str(model_dir / "fairness_decision_policy.json"), - }, - "attributes": [ - {"name": "home_ownership", "column": "home_ownership"}, - {"name": "verification_status", "column": "verification_status"}, - ], - "artifacts": { - "test_predictions_path": str(pred_path), - "test_data_path": str(data_path), - }, - "output": { - "audit_parquet": str(data_dir / "fairness_audit.parquet"), - "frontier_parquet": str(data_dir / "fairness_threshold_frontier.parquet"), - "status_json": str(model_dir / "fairness_audit_status.json"), - "threshold_semantics_json": str(model_dir / "threshold_semantics.json"), - }, - "fairlearn_sidecar": { - "enabled": True, - "status_json": str(model_dir / "fairlearn_fairness_status.json"), - "group_metrics_parquet": str(data_dir / "fairlearn_group_metrics.parquet"), - "bootstrap_samples": 5, - }, - "intersectional": {"enabled": True, "max_order": 2, "min_group_size": 1}, - "threshold_frontier": {"enabled": True, "thresholds": [0.35, 0.40, 0.45, 0.50]}, - } - cfg_path = tmp_path / "fairness_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - fairness_mod.main(str(cfg_path), run_tag="run-fairness-auto") - - status = json.loads((model_dir / "fairness_audit_status.json").read_text(encoding="utf-8")) - decision_policy = json.loads( - (model_dir / "fairness_decision_policy.json").read_text(encoding="utf-8") - ) - assert status["run_tag"] == "run-fairness-auto" - assert status["prediction_threshold_source"] == "decision_policy_artifact_auto_selected" - assert status["decision_policy"]["path"].endswith("fairness_decision_policy.json") - assert decision_policy["global_threshold"] in [0.35, 0.4, 0.45, 0.5] - - -def test_shap_helpers_detect_categorical_columns_and_fill_missing_values() -> None: - class DummyModel: - def __init__(self) -> None: - self.feature_names_ = ["grade", "purpose", "rate"] - - def get_cat_feature_indices(self) -> list[int]: - return [0] - - frame = pd.DataFrame( - { - "grade": ["A", None, "B"], - "purpose": ["debt", "car", None], - "rate": [0.1, np.nan, 0.3], - } - ) - - cat_names = fairness_mod._catboost_cat_feature_names(DummyModel(), frame) - prepared = fairness_mod._prepare_catboost_shap_frame(frame, cat_names) - - assert cat_names == ["grade", "purpose"] - assert prepared.loc[1, "grade"] == "missing" - assert prepared.loc[2, "purpose"] == "missing" - assert prepared.loc[1, "rate"] == 0.0 - - -def test_shap_attribute_result_reports_group_drivers_and_pairwise_diffs() -> None: - shap_matrix = np.vstack( - [ - np.tile([3.0, 1.0, 0.5], (10, 1)), - np.tile([1.0, 4.0, 0.5], (10, 1)), - ] - ) - sample_idx = np.arange(20) - groups = { - "home_ownership": np.array(["A"] * 10 + ["B"] * 10), - "home_ownership__x__grade": np.array(["skip"] * 20), - } - - results = fairness_mod._shap_attribute_results( - groups, - sample_idx=sample_idx, - shap_matrix=shap_matrix, - feature_names=["dti", "income", "grade"], - ) - - assert len(results) == 1 - result = results[0] - assert result["attribute"] == "home_ownership" - assert result["groups_analyzed"] == ["A", "B"] - assert result["top5_per_group"]["A"][0]["feature"] == "dti" - assert result["top5_per_group"]["B"][0]["feature"] == "income" - assert result["pairwise_feature_diffs"][0]["top_driving_features"][0]["feature"] == "income" diff --git a/tests/test_scripts/test_run_portfolio_bound_aware_search.py b/tests/test_scripts/test_run_portfolio_bound_aware_search.py deleted file mode 100644 index 6df7fb5..0000000 --- a/tests/test_scripts/test_run_portfolio_bound_aware_search.py +++ /dev/null @@ -1,351 +0,0 @@ -from __future__ import annotations - -import pandas as pd -import pytest - -from scripts.search.run_portfolio_bound_aware_search import ( - _aggregate_exact_results, - _budget_profiles, - _build_grid_spec, - _build_parser, - _build_stratified_shortlist, - _policy_semantic_key, - _resolve_run_paths, - _sanitize_run_label, - _search_space_payload, - _selection_context_payload, - _targeted_policy_grid, -) -from src.optimization.certificate_semantics import ( - IJDS_DECLARED_ALPHA_GRID, - IJDS_DECLARED_ALPHA_GRID_CSV, -) - - -def _frontier_row( - *, - risk_tolerance: float, - policy_mode: str, - gamma: float, - uncertainty_aversion: float, - min_budget_utilization: float, - pd_cap_slack_penalty: float, - realized_total_return: float, - price_of_robustness: float, - worst_case_pd: float, - point_pd: float, - ab_pass_all: bool = True, -) -> dict[str, object]: - return { - "risk_tolerance": risk_tolerance, - "policy_mode": policy_mode, - "gamma": gamma, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0 if policy_mode != "tail_blended_uncertainty" else 0.85, - "uncertainty_aversion": uncertainty_aversion, - "min_budget_utilization": min_budget_utilization, - "pd_cap_slack_penalty": pd_cap_slack_penalty, - "solver_backend": "highs", - "seed_count": 1, - "sample_random_states": "42", - "ab_pass_all": ab_pass_all, - "ab_pass_rate": 1.0 if ab_pass_all else 0.0, - "realized_total_return": realized_total_return, - "realized_total_return_max": realized_total_return, - "price_of_robustness": price_of_robustness, - "price_of_robustness_pct": price_of_robustness / 1000.0, - "ab_diff_total_return": realized_total_return / 100.0, - "objective_value": realized_total_return / 10.0, - "n_funded": 100.0, - "total_allocated": 1_000_000.0, - "expected_return_net_point": realized_total_return / 5.0, - "worst_case_pd": worst_case_pd, - "point_pd": point_pd, - "pd_cap_slack": 0.0, - } - - -def test_stratified_shortlist_keeps_alpha01_incumbent_region() -> None: - rows = [] - for idx in range(120): - rows.append( - _frontier_row( - risk_tolerance=0.18 + idx * 0.0001, - policy_mode="tail_blended_uncertainty", - gamma=0.7, - uncertainty_aversion=0.1, - min_budget_utilization=0.05, - pd_cap_slack_penalty=1.5, - realized_total_return=150_000.0 - idx, - price_of_robustness=-40_000.0 + idx, - worst_case_pd=0.18 + idx * 1e-5, - point_pd=0.16 + idx * 1e-5, - ) - ) - - incumbent = _frontier_row( - risk_tolerance=0.16, - policy_mode="blended_uncertainty", - gamma=0.5, - uncertainty_aversion=0.0, - min_budget_utilization=0.0, - pd_cap_slack_penalty=0.0, - realized_total_return=90_529.78, - price_of_robustness=-10_261.65, - worst_case_pd=0.077725, - point_pd=0.0700, - ) - rows.append(incumbent) - frontier = pd.DataFrame(rows) - - shortlist = _build_stratified_shortlist( - frontier=frontier, - shortlist_top_k=30, - bucket_return_k=10, - bucket_proxy_k=10, - bucket_family_k=5, - bucket_region_k=5, - incumbent_policy={ - "risk_tolerance": 0.16, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs", - }, - incumbent_risk_neighbors=[0.155, 0.16, 0.165, 0.17], - incumbent_gamma_neighbors=[0.45, 0.5, 0.55], - incumbent_policy_modes=["blended_uncertainty", "capped_blended_uncertainty"], - budget_profiles=[ - {"name": "free_budget", "min_budget_utilization": 0.0, "pd_cap_slack_penalty": 0.0}, - {"name": "floored_budget", "min_budget_utilization": 0.05, "pd_cap_slack_penalty": 1.5}, - ], - solver_backend="highs", - ) - - incumbent_key = _policy_semantic_key(incumbent) - assert incumbent_key in set(shortlist["semantic_policy_key"]) - incumbent_row = shortlist[shortlist["semantic_policy_key"] == incumbent_key].iloc[0] - assert str(incumbent_row["shortlist_bucket"]).startswith("forced_") or str( - incumbent_row["shortlist_bucket"] - ).startswith("incumbent_") - - -def test_aggregate_exact_results_prefers_alpha01_passers() -> None: - shortlist = pd.DataFrame( - [ - { - "candidate_rank": 1, - "risk_tolerance": 0.16, - "policy_mode": "blended_uncertainty", - "gamma": 0.5, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "solver_backend": "highs", - "ab_pass_all": True, - "realized_total_return": 90_000.0, - "price_of_robustness": -10_000.0, - "alpha01_exact_pass": False, - "alpha01_weighted_miscoverage_V": 9.99, - "alpha01_gamma_cp": 9.99, - }, - { - "candidate_rank": 2, - "risk_tolerance": 0.18, - "policy_mode": "tail_blended_uncertainty", - "gamma": 0.7, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 0.85, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.05, - "pd_cap_slack_penalty": 1.5, - "solver_backend": "highs", - "ab_pass_all": True, - "realized_total_return": 120_000.0, - "price_of_robustness": -35_000.0, - "alpha01_exact_pass": True, - "alpha01_weighted_miscoverage_V": 0.01, - "alpha01_gamma_cp": 0.01, - }, - ] - ) - - bound_rows = [] - for eval_seed in [42, 2026]: - for alpha in [0.01, 0.03, 0.10]: - bound_rows.append( - { - "eval_random_state": eval_seed, - "alpha": alpha, - "all_bounds_hold": True, - "gamma_cp": 0.17, - "weighted_miscoverage_V": 0.08, - "violation": 0.0, - "weighted_pd_true": 0.08, - "weighted_pd_constraint_used": 0.16, - "empirical_coverage_funded": 0.95, - **shortlist.iloc[0].to_dict(), - } - ) - bound_rows.append( - { - "eval_random_state": eval_seed, - "alpha": alpha, - "all_bounds_hold": not (alpha == 0.01 and eval_seed == 2026), - "gamma_cp": 0.19, - "weighted_miscoverage_V": 0.11, - "violation": 0.01 if (alpha == 0.01 and eval_seed == 2026) else 0.0, - "weighted_pd_true": 0.11, - "weighted_pd_constraint_used": 0.18, - "empirical_coverage_funded": 0.93, - **shortlist.iloc[1].to_dict(), - } - ) - bound_eval = pd.DataFrame(bound_rows) - - ranked = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) - - assert bool(ranked.iloc[0]["alpha01_exact_pass"]) is True - assert float(ranked.iloc[0]["risk_tolerance"]) == 0.16 - assert float(ranked.iloc[0]["alpha01_weighted_miscoverage_V"]) == 0.08 - assert bool(ranked.iloc[1]["alpha01_exact_pass"]) is False - assert not any(col.endswith(("_x", "_y")) for col in ranked.columns) - - -def test_targeted_policy_grid_includes_segment_tail_families() -> None: - grid = _targeted_policy_grid( - gamma_values=[0.5], - delta_cap_quantiles=[0.75], - tail_focus_quantiles=[0.9], - policy_modes=[ - "blended_uncertainty", - "capped_blended_uncertainty", - "tail_blended_uncertainty", - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - ], - ) - - assert {row[0] for row in grid} == { - "blended_uncertainty", - "capped_blended_uncertainty", - "tail_blended_uncertainty", - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - } - assert grid == [ - ("blended_uncertainty", 0.5, 1.0, 1.0), - ("capped_blended_uncertainty", 0.5, 0.75, 1.0), - ("tail_blended_uncertainty", 0.5, 1.0, 0.9), - ("segment_tail_blended_uncertainty", 0.5, 1.0, 0.9), - ("segment_relative_tail_blended_uncertainty", 0.5, 1.0, 0.9), - ] - - -def test_budget_profiles_are_explicit_and_reject_unknown_tokens() -> None: - profiles = _budget_profiles("free,floored") - - assert [profile["name"] for profile in profiles] == ["free_budget", "floored_budget"] - assert profiles[0]["min_budget_utilization"] == 0.0 - assert profiles[1]["pd_cap_slack_penalty"] == 1.5 - with pytest.raises(ValueError, match="Unsupported budget profile"): - _budget_profiles("free,aggressive") - - -def test_bound_aware_default_uses_declared_ijds_alpha_grid() -> None: - args = _build_parser().parse_args(["--conformal-intervals-path", "intervals.parquet"]) - - assert args.alpha_grid == IJDS_DECLARED_ALPHA_GRID_CSV - - -def test_grid_spec_separates_proxy_and_exact_sampling_contracts() -> None: - args = _build_parser().parse_args( - [ - "--conformal-intervals-path", - "intervals.parquet", - "--risk-grid", - "0.16,0.17", - "--random-states", - "42,52", - "--exact-random-states", - "62,72,82", - "--max-candidates", - "5000", - "--exact-max-candidates", - "0", - ] - ) - - grid = _build_grid_spec(args) - - assert grid.random_states == [42, 52] - assert grid.exact_random_states == [62, 72, 82] - assert grid.exact_max_candidates == 0 - assert grid.alpha_grid == list(IJDS_DECLARED_ALPHA_GRID) - assert grid.bound_total_checks(shortlist_size=5) == 5 * 8 * 3 - assert grid.frontier_total_units == 2 * 2 * (1 + grid.policy_grid_count) - - -def test_selection_context_payload_keeps_frontier_and_exact_paths_together(tmp_path) -> None: - args = _build_parser().parse_args( - [ - "--conformal-intervals-path", - str(tmp_path / "intervals.parquet"), - "--run-label", - "unit/run", - "--output-dir", - str(tmp_path / "out"), - "--model-dir", - str(tmp_path / "model"), - "--random-states", - "42,52", - "--exact-random-states", - "42", - "--policy-modes", - "blended_uncertainty", - ] - ) - run_label = _sanitize_run_label(args.run_label) - paths = _resolve_run_paths(args, run_label=run_label) - search_space = _search_space_payload( - args=args, - risk_values=[0.16], - aversion_values=[0.0], - gamma_values=[0.5], - delta_cap_quantiles=[1.0], - tail_focus_quantiles=[1.0], - budget_profiles=_budget_profiles(args.budget_profiles), - alpha_grid=[0.01], - random_states=[42, 52], - exact_random_states=[42], - exact_max_candidates=0, - policy_modes=["blended_uncertainty"], - cuopt_parameters={"method": "concurrent"}, - incumbent_risk_neighbors=[0.16], - incumbent_gamma_neighbors=[0.5], - incumbent_policy_modes=["blended_uncertainty"], - ) - - context = _selection_context_payload( - args=args, - paths=paths, - run_label=run_label, - search_space=search_space, - exact_max_candidates=0, - random_states=[42, 52], - exact_random_states=[42], - alpha_grid=[0.01], - ) - - assert run_label == "unit_run" - assert context["search_space"]["random_states"] == [42, 52] - assert context["search_space"]["exact_random_states"] == [42] - assert context["shortlist_path"] == str(paths.shortlist_path) - assert context["selection_path"] == str(paths.selection_path) - assert context["resource_snapshot_path"] == str(paths.resource_path) diff --git a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py b/tests/test_scripts/test_run_portfolio_bound_exact_eval.py deleted file mode 100644 index 203aa63..0000000 --- a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py +++ /dev/null @@ -1,230 +0,0 @@ -from __future__ import annotations - -import pandas as pd -import pytest - -import scripts.search.run_portfolio_bound_exact_eval as exact_eval -from scripts.search.build_portfolio_exact_priority_context import build_priority_context -from scripts.search.run_portfolio_bound_exact_eval import ( - ROOT, - _context_exact_threads, - _context_max_candidates, - _context_random_states, - _exact_eval_plan, - _load_completed_bound_eval, - _load_partial_bound_eval, - _repo_relative, - _resume_exact_rows, - _search_space_payload, - _shortlist_exact_path, - _validate_alpha_grid_supported, -) - - -def test_load_completed_bound_eval_reuses_complete_cache(tmp_path) -> None: - path = tmp_path / "bound_eval.parquet" - expected = pd.DataFrame( - { - "alpha": [0.01, 0.03], - "all_bounds_hold": [True, True], - "gamma_cp": [0.18, 0.18], - "weighted_miscoverage_V": [0.03, 0.03], - } - ) - expected.to_parquet(path, index=False) - - cached = _load_completed_bound_eval(bound_eval_path=path, expected_checks=2) - - assert cached is not None - assert cached.equals(expected) - - -def test_load_completed_bound_eval_rejects_incomplete_cache(tmp_path) -> None: - path = tmp_path / "bound_eval.parquet" - pd.DataFrame( - { - "alpha": [0.01], - "all_bounds_hold": [True], - "gamma_cp": [0.18], - "weighted_miscoverage_V": [0.03], - } - ).to_parquet(path, index=False) - - assert _load_completed_bound_eval(bound_eval_path=path, expected_checks=2) is None - - -def test_load_partial_bound_eval_reuses_checkpoint_rows(tmp_path) -> None: - path = tmp_path / "bound_eval.parquet" - pd.DataFrame( - { - "candidate_rank": [1, 1], - "eval_random_state": [42, 42], - "alpha": [0.01, 0.01], - "all_bounds_hold": [False, True], - "gamma_cp": [0.20, 0.18], - "weighted_miscoverage_V": [0.04, 0.03], - "solver_status": ["unknown", "optimal"], - } - ).to_parquet(path, index=False) - - cached = _load_partial_bound_eval(bound_eval_path=path) - - assert len(cached) == 1 - assert bool(cached.iloc[0]["all_bounds_hold"]) is True - assert cached.iloc[0]["allocator_solver_backend"] == "highspy_fallback_highs_sparse" - assert cached.iloc[0]["allocator_native_solver_error"] == "" - - -def test_exact_context_overrides_proxy_sampling() -> None: - context = { - "max_candidates": 100000, - "exact_max_candidates": 0, - "random_states": [42], - "exact_random_states": "42,52,62", - } - - assert _context_max_candidates(context) == 0 - assert _context_random_states(context) == [42, 52, 62] - - -def test_exact_plan_dedupes_full_universe_random_states(monkeypatch) -> None: - monkeypatch.delenv("EXACT_THREADS", raising=False) - context = { - "alpha_grid": [0.01, 0.03], - "max_candidates": 100000, - "exact_max_candidates": 0, - "random_states": [42], - "requested_exact_random_states": "42,52,62", - "exact_checkpoint_every": 7, - "exact_threads": 3, - } - - plan = _exact_eval_plan(context=context, shortlist_rows=5) - - assert plan.requested_random_states == [42, 52, 62] - assert plan.random_states == [42] - assert plan.full_universe_seed_deduped is True - assert plan.expected_checks == 10 - assert plan.exact_threads == 3 - assert plan.checkpoint_every == 7 - - -def test_exact_threads_can_come_from_environment(monkeypatch) -> None: - monkeypatch.setenv("EXACT_THREADS", "8") - - assert _context_exact_threads({"exact_threads": 2}) == 8 - - -def test_shortlist_exact_path_uses_explicit_output() -> None: - expected = ROOT / "data" / "processed" / "portfolio_bound_aware_shortlist_exact.parquet" - context = { - "shortlist_path": str( - ROOT / "data" / "processed" / "portfolio_bound_aware_shortlist.parquet" - ), - "shortlist_exact_path": str(expected), - } - - assert _shortlist_exact_path(context) == expected - - -def test_repo_relative_keeps_artifact_paths_standalone() -> None: - path = ROOT / "models" / "portfolio_bound_aware" / "selection.json" - - assert _repo_relative(path) == "models/portfolio_bound_aware/selection.json" - - -def test_resume_exact_rows_filters_out_irrelevant_seeds(tmp_path) -> None: - path = tmp_path / "bound_eval.parquet" - pd.DataFrame( - { - "candidate_rank": [1, 1, 2], - "eval_random_state": [42, 52, 42], - "alpha": [0.01, 0.01, 0.03], - "all_bounds_hold": [True, True, False], - } - ).to_parquet(path, index=False) - - rows, keys = _resume_exact_rows(bound_eval_path=path, random_states=[42]) - - assert len(rows) == 2 - assert keys == {(1, 42, 0.01), (2, 42, 0.03)} - - -def test_search_space_payload_records_requested_alpha_grid() -> None: - context = {"search_space": {"alpha_grid": [0.01], "mode": ["capped"]}} - - payload = _search_space_payload(context=context, alpha_grid=[0.01, 0.03]) - - assert payload["alpha_grid"] == [0.01, 0.03] - assert payload["requested_alpha_grid"] == [0.01] - assert payload["effective_alpha_grid"] == [0.01, 0.03] - assert payload["mode"] == ["capped"] - - -def test_validate_alpha_grid_supported_accepts_sweep_values(tmp_path, monkeypatch) -> None: - sweep_path = tmp_path / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" - sweep_path.parent.mkdir(parents=True) - pd.DataFrame({"alpha": [0.01, 0.03, 0.05]}).to_parquet(sweep_path, index=False) - monkeypatch.setattr(exact_eval, "ROOT", tmp_path) - - _validate_alpha_grid_supported([0.01, 0.03]) - - -def test_validate_alpha_grid_supported_rejects_missing_sweep_values( - tmp_path, - monkeypatch, -) -> None: - sweep_path = tmp_path / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" - sweep_path.parent.mkdir(parents=True) - pd.DataFrame({"alpha": [0.01, 0.03, 0.05]}).to_parquet(sweep_path, index=False) - monkeypatch.setattr(exact_eval, "ROOT", tmp_path) - - with pytest.raises(ValueError, match="absent"): - _validate_alpha_grid_supported([0.01, 0.02]) - - -def test_priority_context_orders_claim_candidates_and_dedupes_full_universe_seeds( - tmp_path, -) -> None: - shortlist_path = tmp_path / "shortlist.parquet" - pd.DataFrame( - { - "candidate_rank": [1, 2, 3], - "shortlist_bucket": [ - "incumbent_region", - "forced_incumbent_neighbors", - "conservative_proxy", - ], - "risk_tolerance": [0.15, 0.16, 0.20], - "policy_mode": ["blended_uncertainty"] * 3, - "gamma": [0.45, 0.45, 0.0], - "uncertainty_aversion": [0.0, 0.1, 0.0], - "realized_total_return": [160_000.0, 175_000.0, 180_000.0], - "ab_pass_all": [True, True, True], - } - ).to_parquet(shortlist_path, index=False) - context_path = tmp_path / "context.json" - context_path.write_text( - """ -{ - "shortlist_path": "__SHORTLIST__", - "random_states": [42], - "exact_random_states": [42, 52, 62], - "max_candidates": 100000, - "exact_max_candidates": 0, - "selection_policy": {} -} -""".replace("__SHORTLIST__", shortlist_path.as_posix()), - encoding="utf-8", - ) - - payload = build_priority_context( - context_path=context_path, - champion_return=170_464.54, - ) - - priority = pd.read_parquet(payload["priority_shortlist_path"]) - assert priority.iloc[0]["candidate_rank"] == 2 - assert priority.iloc[0]["exact_priority_reason"] == "above_champion_forced_incumbent_region" - assert payload["effective_exact_random_states"] == [42] - assert payload["full_universe_seed_deduped"] is True diff --git a/tests/test_scripts/test_run_regret_auditability_sandbox.py b/tests/test_scripts/test_run_regret_auditability_sandbox.py deleted file mode 100644 index 23b5bbb..0000000 --- a/tests/test_scripts/test_run_regret_auditability_sandbox.py +++ /dev/null @@ -1,264 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest -import yaml - -import scripts.search.run_regret_auditability_sandbox as sandbox -from scripts.search.run_regret_auditability_sandbox import ( - FEATURE_PROFILES, - MONOTONIC_POLICIES, - PORTFOLIO_ALPHA_GRID, - PhaseCommand, - _pd_validation_policy, - _pending_commands_for_group, - _phase_command_groups, - _rank_pd_candidate_rows, - assert_safe_output_path, - build_phase_commands, - compute_auditability_score, - compute_decision_regret, - load_resume_manifest, - materialize_feature_profiles, - materialize_monotonic_policies, -) - - -def test_protected_path_rejection_blocks_frozen_outputs(tmp_path: Path) -> None: - repo_root = tmp_path / "repo" - - with pytest.raises(ValueError, match="protected CRPTO artifact"): - assert_safe_output_path("models/pd_canonical.cbm", repo_root=repo_root) - - with pytest.raises(ValueError, match="protected CRPTO directory"): - assert_safe_output_path( - "data/processed/portfolio_bound_aware/rank1_candidate/frontier.parquet", - repo_root=repo_root, - ) - - external = tmp_path / "outside" / "frontier.parquet" - assert assert_safe_output_path(external, repo_root=repo_root) == external.resolve() - - -def test_monotonic_policy_materialization_matches_sandbox_lanes() -> None: - policies = materialize_monotonic_policies() - - assert set(policies) == set(MONOTONIC_POLICIES) - assert policies["canonical_4"] == { - "installment": 1, - "annual_inc": -1, - "dti": 1, - "loan_to_income": 1, - } - assert policies["affordability_rate_5"]["int_rate"] == 1 - assert policies["credit_history_7"]["delinq_severity"] == 1 - assert policies["credit_history_7"]["delinq_recency"] == -1 - assert policies["bureau_utilization_11"]["bc_util"] == 1 - assert policies["bureau_behavior_15"]["pct_tl_nvr_dlq"] == -1 - assert policies["inquiry_velocity_12"]["mths_since_recent_inq"] == -1 - - -def test_feature_profiles_materialize_expected_lanes() -> None: - profiles = materialize_feature_profiles() - - assert set(profiles) == set(FEATURE_PROFILES) - assert profiles["core_stable"]["stable_core_enabled"] is True - assert "WOE_FEATURES" in profiles["core_woe"]["groups"] - assert "CHALLENGER_FEATURE_POOL_V2" in profiles["full_challenger"]["groups"] - - -def test_decision_regret_is_oracle_minus_policy_return() -> None: - assert compute_decision_regret(1250.0, 1000.0) == 250.0 - assert compute_decision_regret(1000.0, 1250.0) == -250.0 - - -def test_auditability_score_weights_all_checks() -> None: - result = compute_auditability_score( - { - "coverage90": 0.91, - "coverage95": 0.951, - "min_group_coverage": 0.90, - "critical_alerts": 0, - "alpha01_exact_pass": True, - "violation": 0.0, - "weighted_miscoverage_V": 0.05, - "alpha": 0.01, - "monotonic_audit_pass": True, - "reproducible_resume_manifest": True, - } - ) - - assert result["score"] == 100 - assert all(result["checks"].values()) - - -def test_auditability_score_penalizes_failed_guarantees() -> None: - result = compute_auditability_score( - { - "coverage90": 0.89, - "coverage95": 0.94, - "min_group_coverage": 0.80, - "critical_alerts": 1, - "alpha01_exact_pass": False, - "violation": 0.01, - "weighted_miscoverage_V": 0.2, - "alpha": 0.01, - "monotonic_audit_pass": True, - "reproducible_resume_manifest": True, - } - ) - - assert result["score"] == 15 - assert result["checks"]["monotonic_audit_pass"] - assert result["checks"]["reproducible_resume_manifest"] - - -def test_resume_manifest_loading_roundtrip(tmp_path: Path) -> None: - manifest_path = tmp_path / "sandbox_manifest.json" - manifest_path.write_text(json.dumps({"schema_version": "x", "phase": "plan"})) - - assert load_resume_manifest(manifest_path)["phase"] == "plan" - assert load_resume_manifest(tmp_path / "missing.json") == {} - - -def _command(name: str, phase: str, output: Path) -> PhaseCommand: - return PhaseCommand( - name=name, - phase=phase, - command=["python", "-c", "pass"], - outputs=[str(output)], - checkpoint=str(output), - env={}, - max_workers=1, - cpu_threads=1, - ) - - -def test_phase_command_groups_keep_consecutive_phase_batches(tmp_path: Path) -> None: - commands = [ - _command("a", "pd-smoke", tmp_path / "a"), - _command("b", "pd-smoke", tmp_path / "b"), - _command("c", "conformal", tmp_path / "c"), - _command("d", "pd-smoke", tmp_path / "d"), - ] - - groups = _phase_command_groups(commands) - - assert [[command.name for command in group] for group in groups] == [["a", "b"], ["c"], ["d"]] - - -def test_pending_commands_skip_completed_outputs_on_resume( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - completed_output = tmp_path / "done.txt" - completed_output.write_text("ok", encoding="utf-8") - pending_output = tmp_path / "todo.txt" - monkeypatch.setattr(sandbox, "_log_command_to_mlflow", lambda **_: None) - - pending, skipped = _pending_commands_for_group( - artifact_root=tmp_path, - log_path=tmp_path / "command_log.csv", - group=[ - _command("done", "pd-smoke", completed_output), - _command("todo", "pd-smoke", pending_output), - ], - resume=True, - ) - - assert skipped == 1 - assert [command.name for command in pending] == ["todo"] - assert "skipped_completed" in (tmp_path / "command_log.csv").read_text(encoding="utf-8") - - -def test_pd_validation_policy_scales_by_phase() -> None: - smoke_replay, smoke_walk_forward = _pd_validation_policy("pd-smoke") - broad_replay, broad_walk_forward = _pd_validation_policy("pd-broad") - refine_replay, refine_walk_forward = _pd_validation_policy("pd-refine") - - assert smoke_replay["top_k_trials"] == 1 - assert smoke_walk_forward is False - assert broad_replay["seeds"] == [42, 52, 62] - assert broad_walk_forward is True - assert refine_replay["top_k_trials"] == 30 - assert refine_walk_forward is True - - -def test_pd_candidate_ranking_prefers_auc_then_calibration() -> None: - ranked = _rank_pd_candidate_rows( - [ - {"lane_id": "a", "auc_roc": 0.75, "brier_score": 0.11, "ece": 0.03}, - {"lane_id": "b", "auc_roc": 0.76, "brier_score": 0.13, "ece": 0.05}, - {"lane_id": "c", "auc_roc": 0.75, "brier_score": 0.10, "ece": 0.04}, - ] - ) - - assert [row["lane_id"] for row in ranked] == ["b", "c", "a"] - assert [row["selection_rank"] for row in ranked] == [1, 2, 3] - - -def test_build_pd_smoke_commands_write_external_config_snapshots(tmp_path: Path) -> None: - commands = build_phase_commands( - artifact_root=tmp_path, - run_tag="unit_sandbox", - phase="pd-smoke", - max_workers=3, - cpu_threads=6, - ) - - expected_search_lanes = len(FEATURE_PROFILES) * len(MONOTONIC_POLICIES) - assert len(commands) == expected_search_lanes + 1 - assert {command.phase for command in commands} == {"pd-smoke"} - search_commands = [ - command for command in commands if command.lane_id != "incumbent__frozen_champion" - ] - assert len(search_commands) == expected_search_lanes - assert all("--hpo_n_trials" in command.command for command in search_commands) - assert all("12" in command.command for command in search_commands) - assert {command.env["PIPELINE_RUN_TAG"] for command in commands} == {"unit_sandbox"} - assert {command.env["CRPTO_RUN_TAG"] for command in commands} == {"unit_sandbox"} - config_path = tmp_path / "configs" / "pd_core_stable__canonical_4_pd-smoke.yaml" - assert config_path.exists() - assert (tmp_path / "configs" / "feature_profiles" / "full_challenger_woe.pkl").exists() - assert all(str(tmp_path) in output for command in commands for output in command.outputs) - config = yaml.safe_load(config_path.read_text(encoding="utf-8")) - assert config["hpo"]["enqueue_trials"] - assert config["sandbox_search"]["hpo_warm_start"]["sources"] == ["frozen_champion_pd_config"] - - -def test_build_portfolio_command_uses_external_output_dirs(tmp_path: Path) -> None: - commands = build_phase_commands( - artifact_root=tmp_path, - run_tag="unit_sandbox", - phase="portfolio", - max_workers=4, - cpu_threads=4, - ) - - assert len(commands) == 1 - command = commands[0] - assert "--output-dir" in command.command - assert "--model-dir" in command.command - alpha_grid_index = command.command.index("--alpha-grid") + 1 - assert command.command[alpha_grid_index] == PORTFOLIO_ALPHA_GRID - assert PORTFOLIO_ALPHA_GRID == "0.01,0.03,0.05,0.07,0.10,0.12,0.15,0.20" - assert all(str(tmp_path) in output for output in command.outputs) - - -def test_build_conformal_command_accepts_model_override(tmp_path: Path) -> None: - commands = build_phase_commands( - artifact_root=tmp_path, - run_tag="unit_sandbox", - phase="conformal", - max_workers=6, - cpu_threads=1, - ) - - assert len(commands) == 1 - command = commands[0] - assert "--artifact_root" in command.command - assert "--model_override_path" in command.command - assert "--calibrator_override_path" in command.command - assert all(str(tmp_path) in output for output in command.outputs) diff --git a/tests/test_scripts/test_run_spo_real.py b/tests/test_scripts/test_run_spo_real.py deleted file mode 100644 index 854ae6b..0000000 --- a/tests/test_scripts/test_run_spo_real.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Tests for scripts/run_spo_real.py — unit tests for key components. - -Tests the building blocks independently of real data files (which are too -large to load in unit tests). Covers the LP model, point-wise MLP architecture, -regret computation, and instance sampling. -""" - -from __future__ import annotations - -import numpy as np -import pytest - -torch = pytest.importorskip("torch", reason="torch not installed (needs spo extra)") - -# ── CreditPortfolioLP ──────────────────────────────────────────────────────── - - -def test_portfolio_lp_selects_exactly_budget_items() -> None: - from scripts.run_spo_real import CreditPortfolioLP - - n_items, budget = 10, 3 - lp = CreditPortfolioLP(n_items=n_items, budget=budget) - # Costs: items 0-2 cheapest - costs = np.array([0.1, 0.2, 0.3, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.95], dtype=float) - lp.setObj(costs) - sol, obj = lp.solve() - - assert len(sol) == n_items - assert abs(sum(sol) - budget) < 1e-6, f"Expected {budget} selected, got {sum(sol):.4f}" - - -def test_portfolio_lp_selects_cheapest_items() -> None: - from scripts.run_spo_real import CreditPortfolioLP - - n_items, budget = 6, 2 - lp = CreditPortfolioLP(n_items=n_items, budget=budget) - costs = np.array([-0.5, -0.3, 0.1, 0.2, 0.3, 0.4], dtype=float) - lp.setObj(costs) - sol, _ = lp.solve() - - selected = [i for i, v in enumerate(sol) if v > 0.5] - # Items 0 and 1 are cheapest (most negative → best) - assert 0 in selected - assert 1 in selected - - -def test_portfolio_lp_copy_is_independent() -> None: - from scripts.run_spo_real import CreditPortfolioLP - - lp = CreditPortfolioLP(n_items=5, budget=2) - lp_copy = lp.copy() - assert lp_copy.n_items == 5 - assert lp_copy.budget == 2 - assert lp_copy is not lp - - -# ── PDPredictorMLP ──────────────────────────────────────────────────────────── - - -def test_mlp_output_shape() -> None: - """Point-wise MLP must output (B, n_items) from (B, n_items * n_features).""" - from scripts.run_spo_real import PDPredictorMLP - - n_features, n_items, batch = 10, 20, 8 - model = PDPredictorMLP(n_features=n_features, n_items=n_items) - x = torch.randn(batch, n_items * n_features) - out = model(x) - assert out.shape == (batch, n_items), f"Expected ({batch}, {n_items}), got {out.shape}" - - -def test_mlp_permutation_equivariance() -> None: - """Shuffling items within an instance should consistently shuffle predictions.""" - from scripts.run_spo_real import PDPredictorMLP - - n_features, n_items = 8, 15 - model = PDPredictorMLP(n_features=n_features, n_items=n_items) - model.eval() - - rng = np.random.RandomState(0) - feats = rng.randn(n_items, n_features).astype(np.float32) # (n_items, n_features) - perm = np.random.permutation(n_items) - - x_orig = torch.tensor(feats.reshape(1, -1)) - x_perm = torch.tensor(feats[perm].reshape(1, -1)) - - with torch.no_grad(): - out_orig = model(x_orig)[0].numpy() # (n_items,) - out_perm = model(x_perm)[0].numpy() # (n_items,) - - # out_perm should equal out_orig reordered by perm - np.testing.assert_allclose( - out_perm, out_orig[perm], atol=1e-5, err_msg="Point-wise MLP is not permutation-equivariant" - ) - - -def test_mlp_shared_weights_across_items() -> None: - """Same loan features at different positions should get same cost prediction.""" - from scripts.run_spo_real import PDPredictorMLP - - n_features, n_items = 6, 10 - model = PDPredictorMLP(n_features=n_features, n_items=n_items) - model.eval() - - # Build instance where item 0 and item 3 have identical features - feats = torch.randn(n_items, n_features) - feats[3] = feats[0] - x = feats.reshape(1, -1) - - with torch.no_grad(): - out = model(x)[0].numpy() - - assert abs(out[0] - out[3]) < 1e-5, "Identical loan features should produce identical costs" - - -# ── Instance sampling ───────────────────────────────────────────────────────── - - -def test_sample_instances_shapes() -> None: - from scripts.run_spo_real import _sample_instances - - n_loans, n_feats, n_items, n_inst = 500, 10, 20, 50 - rng = np.random.RandomState(42) - X = np.random.randn(n_loans, n_feats).astype(np.float32) - c = np.random.randn(n_loans).astype(np.float32) - - feats, costs, indices = _sample_instances(X, c, n_items, n_inst, rng) - - assert feats.shape == (n_inst, n_items, n_feats) - assert costs.shape == (n_inst, n_items) - assert indices.shape == (n_inst, n_items) - - -def test_sample_instances_no_duplicates_within_instance() -> None: - from scripts.run_spo_real import _sample_instances - - n_loans, n_feats, n_items, n_inst = 200, 5, 30, 20 - rng = np.random.RandomState(7) - X = np.random.randn(n_loans, n_feats).astype(np.float32) - c = np.random.randn(n_loans).astype(np.float32) - - _, _, indices = _sample_instances(X, c, n_items, n_inst, rng) - - for i in range(n_inst): - assert len(set(indices[i])) == n_items, f"Instance {i} has duplicate loan indices" - - -# ── Regret computation ──────────────────────────────────────────────────────── - - -def test_zero_regret_with_true_costs() -> None: - """Regret must be 0 when predicted costs == true costs (optimal prediction).""" - from scripts.run_spo_real import CreditPortfolioLP, _compute_regret, _compute_true_optima - - n_items, budget, n_inst = 8, 3, 5 - rng = np.random.RandomState(99) - lp = CreditPortfolioLP(n_items=n_items, budget=budget) - - c_true = rng.randn(n_inst, n_items).astype(np.float32) - true_optima = _compute_true_optima(c_true, lp) - regrets = _compute_regret(c_true, c_true, lp.copy(), true_optima) - - assert np.allclose(regrets, 0.0, atol=1e-5), f"Expected zero regret, got {regrets}" - - -def test_nonnegative_regret() -> None: - """Regret is always >= 0 (any prediction is at least as bad as optimal).""" - from scripts.run_spo_real import CreditPortfolioLP, _compute_regret, _compute_true_optima - - n_items, budget, n_inst = 10, 4, 20 - rng = np.random.RandomState(123) - lp = CreditPortfolioLP(n_items=n_items, budget=budget) - - c_true = rng.randn(n_inst, n_items).astype(np.float32) - c_pred = rng.randn(n_inst, n_items).astype(np.float32) # random prediction - - true_optima = _compute_true_optima(c_true, lp) - regrets = _compute_regret(c_pred, c_true, lp.copy(), true_optima) - - assert (regrets >= -1e-6).all(), f"Found negative regret: {regrets.min():.6f}" - - -# ── index_costs helper ──────────────────────────────────────────────────────── - - -def test_index_costs_shape_and_values() -> None: - from scripts.run_spo_real import _index_costs - - n_loans, n_items, n_inst = 100, 10, 15 - rng = np.random.RandomState(0) - c_all = rng.randn(n_loans).astype(np.float32) - indices = rng.choice(n_loans, size=(n_inst, n_items), replace=True) - - result = _index_costs(c_all, indices) - assert result.shape == (n_inst, n_items) - # Verify a specific value - assert result[3, 7] == c_all[indices[3, 7]] diff --git a/tests/test_scripts/test_run_ty_advisory.py b/tests/test_scripts/test_run_ty_advisory.py index 43db7b3..3054ac1 100644 --- a/tests/test_scripts/test_run_ty_advisory.py +++ b/tests/test_scripts/test_run_ty_advisory.py @@ -1,41 +1,33 @@ from __future__ import annotations -from scripts.run_ty_advisory import TY_REQUIREMENT, build_ty_command, iter_python_files +from scripts.run_ty_advisory import ( + TY_REQUIREMENT, + _diagnostic_lines, + build_ty_command, + iter_python_files, +) -def test_active_ty_scope_excludes_archived_optional_and_protected_paths() -> None: - files = set(iter_python_files(scope="active")) - - assert "scripts/generate_conformal_intervals.py" not in files - assert "scripts/train_pd_model.py" not in files - assert "scripts/run_spo_real.py" not in files - assert "src/optimization/cuopt_adapter.py" not in files - assert all(not path.startswith("scripts/archive/") for path in files) - assert "scripts/experiments/ijds_policy_support.py" in files - assert "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py" in files - assert "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py" in files - assert all( - not (path.startswith("scripts/search/run_") and path.endswith(".py")) for path in files - ) - - -def test_active_ty_scope_keeps_live_ijds_helpers() -> None: +def test_active_ty_scope_is_the_complete_current_code_surface() -> None: files = set(iter_python_files(scope="active")) assert "scripts/compile_ijds_submission.py" in files - assert "scripts/build_ijds_calibration_selected_evidence.py" in files - assert "scripts/search/build_pool93_body_allocation_audit.py" in files - assert "src/models/conformal_alpha_grid.py" in files + assert "scripts/build_ijds_binary_geometry_frontier_v4_evidence.py" in files + assert "scripts/experiments/run_ijds_binary_geometry_frontier_v4.py" in files + assert "src/models/binary_conformal_guardrail.py" in files + assert "src/optimization/cuopt_adapter.py" in files assert "src/optimization/policy_selection.py" in files assert "src/optimization/portfolio_model.py" in files + assert all("/archive/" not in path and "/search/" not in path for path in files) -def test_full_ty_scope_keeps_every_python_file_under_src_and_scripts() -> None: - files = set(iter_python_files(scope="full")) +def test_full_ty_scope_adds_only_sealed_compatibility_sources() -> None: + active = set(iter_python_files(scope="active")) + full = set(iter_python_files(scope="full")) - assert "scripts/generate_conformal_intervals.py" in files - assert "scripts/train_pd_model.py" in files - assert "src/optimization/cuopt_adapter.py" in files + assert active < full + assert "src/data/make_dataset.py" in full - active + assert "scripts/train_pd_model.py" in full - active def test_ty_command_pins_version_and_keeps_daily_scope_advisory() -> None: @@ -58,3 +50,10 @@ def test_ty_command_can_block_the_submission_gate() -> None: ) assert "--exit-zero" not in command + + +def test_ty_diagnostic_parser_keeps_every_error() -> None: + first = r"src\first.py:4:2: error[invalid-return-type] first failure" + second = r"src\second.py:10:2: error[invalid-argument-type] second failure" + + assert _diagnostic_lines(f"{first}\ninformation\n{second}\n") == [first, second] diff --git a/tests/test_scripts/test_select_economic_portfolio_policy.py b/tests/test_scripts/test_select_economic_portfolio_policy.py deleted file mode 100644 index 432c2f6..0000000 --- a/tests/test_scripts/test_select_economic_portfolio_policy.py +++ /dev/null @@ -1,380 +0,0 @@ -from __future__ import annotations - -import json - -import pandas as pd - -from scripts import select_economic_portfolio_policy as sel_mod - - -def _write_common_inputs(tmp_path) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - pd.DataFrame( - { - "id": ["a", "b", "c"], - "int_rate": [10.0, 12.0, 9.0], - "loan_amnt": [1000.0, 1000.0, 1000.0], - "default_flag": [0, 0, 0], - } - ).to_parquet(data_dir / "test_fe.parquet", index=False) - pd.DataFrame( - { - "id": ["a", "b", "c"], - "pd_calibrated": [0.05, 0.08, 0.07], - "pd_low": [0.04, 0.07, 0.06], - "pd_high": [0.09, 0.11, 0.10], - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - pd.DataFrame({"id": ["a", "b", "c"], "sample_order": [0, 1, 2]}).to_parquet( - data_dir / "champion_candidate_universe.parquet", index=False - ) - frontier = pd.DataFrame( - [ - { - "policy": "robust", - "policy_mode": "blended_uncertainty", - "gamma": 0.0, - "delta_cap_quantile": 1.0, - "risk_tolerance": 0.1, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "realized_total_return": 100.0, - "price_of_robustness_pct": 0.0, - "selected_for_champion": True, - "selected_for_balanced_robustness": False, - "selected_for_guardrail_robustness": False, - "eligible_for_canonical_selection": True, - }, - { - "policy": "robust", - "policy_mode": "capped_blended_uncertainty", - "gamma": 0.1, - "delta_cap_quantile": 0.75, - "risk_tolerance": 0.1, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "realized_total_return": 95.0, - "price_of_robustness_pct": -5.0, - "selected_for_champion": False, - "selected_for_balanced_robustness": True, - "selected_for_guardrail_robustness": True, - "eligible_for_canonical_selection": True, - }, - ] - ) - frontier.to_parquet(data_dir / "portfolio_robustness_frontier.parquet", index=False) - (model_dir / "portfolio_research_policy.json").write_text( - json.dumps({"selected_policy": {"gamma": 0.0}}), - encoding="utf-8", - ) - (tmp_path / "configs").mkdir(parents=True, exist_ok=True) - (tmp_path / "configs" / "optimization.yaml").write_text( - """ -portfolio: - total_budget: 1000 -portfolio_selection: - canonical_selector: economic_actual_ab_v2 - actual_ab_top_k: 20 - min_funded_ratio: 0.88 - min_total_allocated_ratio: 0.98 - min_breadth_score: 0.995 - breadth_weight_funded_ratio: 0.50 - breadth_weight_allocation_ratio: 0.30 - breadth_weight_allocation_similarity: 0.20 - max_price_of_robustness_pct: -15.0 - canonical_policy_modes: [blended_uncertainty, capped_blended_uncertainty] -""".strip(), - encoding="utf-8", - ) - - -def _set_run_tag(monkeypatch) -> None: - monkeypatch.setenv("PIPELINE_RUN_TAG", "run-policy-test") - - -def test_selector_promotes_robust_candidate_when_one_passes(tmp_path, monkeypatch) -> None: - _write_common_inputs(tmp_path) - monkeypatch.chdir(tmp_path) - _set_run_tag(monkeypatch) - - def fake_run_strategy(*, robust, robust_policy=None, **kwargs): - _ = kwargs - if not robust: - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - gamma = float((robust_policy or {}).get("gamma", 0.0)) - if gamma > 0: - return { - "allocation": {0: 0.98, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - - def fake_candidate_metrics(*, solution, **kwargs): - _ = kwargs - gamma_like = 0.98 if solution["allocation"][0] < 1.0 else 1.0 - total_return = 100.0 if gamma_like == 1.0 else 99.5 - return None, { - "total_return": total_return, - "n_funded": solution["n_funded"], - "total_allocated": solution["total_allocated"], - "avg_return_per_funded": total_return, - } - - monkeypatch.setattr(sel_mod, "_run_strategy", fake_run_strategy) - monkeypatch.setattr(sel_mod, "_candidate_metrics", fake_candidate_metrics) - - sel_mod.main(config_path="configs/optimization.yaml") - - payload = json.loads((tmp_path / "models" / "champion_portfolio_policy.json").read_text()) - status = json.loads((tmp_path / "models" / "champion_policy_selection_status.json").read_text()) - assert payload["selection_outcome"] == "robust_selected" - assert payload["selection_stage"] == "economic_actual_ab_v2" - assert payload["selected_policy"]["gamma"] == 0.1 - assert status["fallback_applied"] is False - assert status["selected_candidate"]["breadth_score"] >= 0.93 - - -def test_selector_falls_back_when_no_robust_candidate_passes(tmp_path, monkeypatch) -> None: - _write_common_inputs(tmp_path) - monkeypatch.chdir(tmp_path) - _set_run_tag(monkeypatch) - - def fake_run_strategy(*, robust, robust_policy=None, **kwargs): - _ = kwargs - if not robust: - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - gamma = float((robust_policy or {}).get("gamma", 0.0)) - return { - "allocation": {0: max(0.7, 1.0 - gamma), 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - - def fake_candidate_metrics(*, solution, **kwargs): - _ = kwargs - total_return = 100.0 if solution["allocation"][0] == 1.0 else 80.0 - return None, { - "total_return": total_return, - "n_funded": solution["n_funded"], - "total_allocated": solution["total_allocated"], - "avg_return_per_funded": total_return, - } - - monkeypatch.setattr(sel_mod, "_run_strategy", fake_run_strategy) - monkeypatch.setattr(sel_mod, "_candidate_metrics", fake_candidate_metrics) - - sel_mod.main(config_path="configs/optimization.yaml") - - payload = json.loads((tmp_path / "models" / "champion_portfolio_policy.json").read_text()) - status = json.loads((tmp_path / "models" / "champion_policy_selection_status.json").read_text()) - assert payload["selection_outcome"] == "fallback_nonrobust" - assert payload["selected_policy"]["gamma"] == 0.0 - assert status["fallback_applied"] is True - - -def test_selector_v2_prefers_breadth_aware_candidate(tmp_path, monkeypatch) -> None: - _write_common_inputs(tmp_path) - monkeypatch.chdir(tmp_path) - _set_run_tag(monkeypatch) - - def fake_run_strategy(*, robust, robust_policy=None, **kwargs): - _ = kwargs - if not robust: - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - gamma = float((robust_policy or {}).get("gamma", 0.0)) - if gamma > 0: - return { - "allocation": {0: 0.99, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 990.0, - }, None - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - - def fake_candidate_metrics(*, solution, **kwargs): - _ = kwargs - total_allocated = float(solution["total_allocated"]) - total_return = 100.0 if total_allocated >= 999.0 else 108.0 - return None, { - "total_return": total_return, - "n_funded": solution["n_funded"], - "total_allocated": total_allocated, - "avg_return_per_funded": total_return, - } - - monkeypatch.setattr(sel_mod, "_run_strategy", fake_run_strategy) - monkeypatch.setattr(sel_mod, "_candidate_metrics", fake_candidate_metrics) - - sel_mod.main(config_path="configs/optimization.yaml") - - payload = json.loads((tmp_path / "models" / "champion_portfolio_policy.json").read_text()) - status = json.loads((tmp_path / "models" / "champion_policy_selection_status.json").read_text()) - assert payload["selection_outcome"] == "robust_selected" - assert payload["selected_policy"]["gamma"] == 0.1 - assert status["selected_candidate"]["total_allocated_ratio"] == 0.99 - assert status["selected_candidate"]["breadth_score"] >= 0.93 - - -def test_selector_v3_uses_ab_like_ranking(tmp_path, monkeypatch) -> None: - _write_common_inputs(tmp_path) - monkeypatch.chdir(tmp_path) - _set_run_tag(monkeypatch) - (tmp_path / "configs" / "optimization.yaml").write_text( - """ -portfolio: - total_budget: 1000 -portfolio_selection: - canonical_selector: economic_actual_ab_v3 - actual_ab_top_k: 20 - ab_like_top_m: 2 - ab_like_bootstrap_n: 10 - ab_like_seed: 7 - min_funded_ratio: 0.88 - min_total_allocated_ratio: 0.98 - min_breadth_score: 0.93 - breadth_weight_funded_ratio: 0.50 - breadth_weight_allocation_ratio: 0.30 - breadth_weight_allocation_similarity: 0.20 - max_price_of_robustness_pct: -15.0 - canonical_policy_modes: [blended_uncertainty, capped_blended_uncertainty] -""".strip(), - encoding="utf-8", - ) - - def fake_run_strategy(*, robust, robust_policy=None, **kwargs): - _ = kwargs - if not robust: - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - gamma = float((robust_policy or {}).get("gamma", 0.0)) - alloc0 = 0.99 if gamma > 0.09 else 0.98 - return { - "allocation": {0: alloc0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - - def fake_candidate_metrics(*, solution, **kwargs): - _ = kwargs - alloc0 = solution["allocation"][0] - if abs(alloc0 - 0.99) < 1e-9: - returns = pd.Series([100.0, 0.0, 0.0]).to_numpy() - total_return = 100.0 - else: - returns = pd.Series([120.0, -40.0, 0.0]).to_numpy() - total_return = 80.0 - return returns, { - "total_return": total_return, - "n_funded": solution["n_funded"], - "total_allocated": solution["total_allocated"], - "avg_return_per_funded": total_return, - } - - monkeypatch.setattr(sel_mod, "_run_strategy", fake_run_strategy) - monkeypatch.setattr(sel_mod, "_candidate_metrics", fake_candidate_metrics) - - sel_mod.main(config_path="configs/optimization.yaml") - - payload = json.loads((tmp_path / "models" / "champion_portfolio_policy.json").read_text()) - status = json.loads((tmp_path / "models" / "champion_policy_selection_status.json").read_text()) - assert payload["selection_stage"] == "economic_actual_ab_v3" - assert payload["selected_policy"]["gamma"] == 0.1 - assert status["selected_candidate"]["ab_like_passed_no_regression"] is True - - -def test_selector_v3_respects_breadth_hard_filters(tmp_path, monkeypatch) -> None: - _write_common_inputs(tmp_path) - monkeypatch.chdir(tmp_path) - _set_run_tag(monkeypatch) - (tmp_path / "configs" / "optimization.yaml").write_text( - """ -portfolio: - total_budget: 1000 -portfolio_selection: - canonical_selector: economic_actual_ab_v3 - actual_ab_top_k: 20 - ab_like_top_m: 2 - ab_like_bootstrap_n: 10 - ab_like_seed: 7 - min_funded_ratio: 0.88 - min_total_allocated_ratio: 0.98 - min_breadth_score: 0.93 - breadth_weight_funded_ratio: 0.50 - breadth_weight_allocation_ratio: 0.30 - breadth_weight_allocation_similarity: 0.20 - max_price_of_robustness_pct: -15.0 - canonical_policy_modes: [blended_uncertainty, capped_blended_uncertainty] -""".strip(), - encoding="utf-8", - ) - - def fake_run_strategy(*, robust, robust_policy=None, **kwargs): - _ = kwargs - if not robust: - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - gamma = float((robust_policy or {}).get("gamma", 0.0)) - if gamma > 0: - return { - "allocation": {0: 0.8, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 900.0, - }, None - return { - "allocation": {0: 1.0, 1: 0.0, 2: 0.0}, - "n_funded": 1, - "total_allocated": 1000.0, - }, None - - def fake_candidate_metrics(*, solution, **kwargs): - _ = kwargs - alloc0 = solution["allocation"][0] - total_return = 120.0 if alloc0 < 1.0 else 100.0 - return pd.Series([total_return, 0.0, 0.0]).to_numpy(), { - "total_return": total_return, - "n_funded": solution["n_funded"], - "total_allocated": solution["total_allocated"], - "avg_return_per_funded": total_return, - } - - monkeypatch.setattr(sel_mod, "_run_strategy", fake_run_strategy) - monkeypatch.setattr(sel_mod, "_candidate_metrics", fake_candidate_metrics) - - sel_mod.main(config_path="configs/optimization.yaml") - - payload = json.loads((tmp_path / "models" / "champion_portfolio_policy.json").read_text()) - status = json.loads((tmp_path / "models" / "champion_policy_selection_status.json").read_text()) - assert payload["selection_outcome"] == "fallback_nonrobust" - assert payload["selected_policy"]["gamma"] == 0.0 - assert status["fallback_applied"] is True diff --git a/tests/test_scripts/test_simulate_ab_test.py b/tests/test_scripts/test_simulate_ab_test.py deleted file mode 100644 index 93907e1..0000000 --- a/tests/test_scripts/test_simulate_ab_test.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -import json - -import pandas as pd -import pytest - -from scripts.simulate_ab_test import _resolve_robust_policy - - -def test_resolve_robust_policy_uses_guardrail_champion_priority(tmp_path, monkeypatch) -> None: - monkeypatch.chdir(tmp_path) - model_dir = tmp_path / "models" - model_dir.mkdir() - (model_dir / "champion_portfolio_policy.json").write_text( - json.dumps( - { - "selected_policy": {"risk_tolerance": 0.10, "gamma": 0.1}, - "selected_policy_balanced_robustness": { - "risk_tolerance": 0.11, - "policy_mode": "balanced", - "gamma": 0.2, - }, - "selected_policy_guardrail_robustness": { - "risk_tolerance": 0.12, - "policy_mode": "guardrail", - "gamma": 0.3, - }, - } - ), - encoding="utf-8", - ) - - policy = _resolve_robust_policy( - max_portfolio_pd=0.20, - policy_selector="guardrail_robustness", - ) - - assert policy["source"] == "champion_policy_artifact::guardrail_robustness" - assert policy["policy_mode"] == "guardrail" - assert policy["risk_tolerance"] == 0.12 - assert policy["gamma"] == 0.3 - - -def test_resolve_robust_policy_uses_summary_when_champion_missing(tmp_path, monkeypatch) -> None: - monkeypatch.chdir(tmp_path) - summary_path = tmp_path / "data" / "processed" / "portfolio_robustness_summary.parquet" - summary_path.parent.mkdir(parents=True) - pd.DataFrame( - [ - { - "risk_tolerance": 0.15, - "best_robust_lambda": 0.2, - "best_robust_min_budget_utilization": 0.90, - "best_robust_pd_cap_slack_penalty": 1.0, - "best_robust_return": 100.0, - "best_robust_policy_mode": "hard_worst_case", - "best_robust_gamma": 0.8, - "best_robust_delta_cap_quantile": 0.95, - }, - { - "risk_tolerance": 0.18, - "best_robust_lambda": 0.4, - "best_robust_min_budget_utilization": 0.91, - "best_robust_pd_cap_slack_penalty": 2.0, - "best_robust_return": 200.0, - "best_robust_policy_mode": "blended_uncertainty", - "best_robust_gamma": 0.6, - "best_robust_delta_cap_quantile": 0.90, - }, - ] - ).to_parquet(summary_path, index=False) - - policy = _resolve_robust_policy(max_portfolio_pd=0.17) - - assert policy["source"] == "portfolio_robustness_summary" - assert policy["risk_tolerance"] == 0.15 - assert policy["uncertainty_aversion"] == 0.2 - assert policy["gamma"] == 0.8 - - -def test_resolve_robust_policy_explicit_champion_only_requires_artifact( - tmp_path, monkeypatch -) -> None: - monkeypatch.chdir(tmp_path) - - with pytest.raises(FileNotFoundError): - _resolve_robust_policy( - max_portfolio_pd=0.20, - policy_selector="explicit_champion_only", - ) diff --git a/tests/test_scripts/test_tail_satisficing_challenger_audit.py b/tests/test_scripts/test_tail_satisficing_challenger_audit.py deleted file mode 100644 index fa2874f..0000000 --- a/tests/test_scripts/test_tail_satisficing_challenger_audit.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pandas as pd - -from scripts.build_tail_satisficing_challenger_audit import _build_cluster_bound_table - - -def test_tail_satisficing_audit_artifacts_exist() -> None: - status_path = Path("models/crpto_tail_satisficing_audit_status.json") - assert status_path.exists() - status = json.loads(status_path.read_text(encoding="utf-8")) - - assert status["schema_version"] == "2026-05-12.2" - assert status["generated_at_utc"] == "2026-06-07T00:00:00+00:00" - assert status["elapsed_sec"] == 0.0 - assert status["timestamp_policy"] == "fixed_for_bit_reproducible_manifest" - assert status["champion_promotion_changed"] is False - assert status["tail_satisficing_audit"]["n_policies_audited"] == 45 - assert status["tail_satisficing_audit"]["promotion_status"] == ( - "journal_audit_only_not_champion" - ) - - for artifact in status["generated_artifacts"]: - assert Path(artifact).exists() - - -def test_cluster_bound_table_is_not_tighter_than_markov_for_concentrated_fixture() -> None: - funded = pd.DataFrame( - { - "period": ["a", "a", "b"], - "original_grade": ["C", "D", "D"], - "portfolio_weight": [0.45, 0.35, 0.20], - "miscovered_alpha01": [False, True, False], - } - ) - - table = _build_cluster_bound_table(funded) - - assert set(table["cluster_type"]) == {"period", "grade", "period_grade"} - assert not bool(table["cluster_bound_tighter_than_markov"].any()) - assert table["empirical_weighted_miscoverage_V"].max() == 0.35 diff --git a/tests/test_scripts/test_train_pd_model_config_overrides.py b/tests/test_scripts/test_train_pd_model_config_overrides.py deleted file mode 100644 index 93799aa..0000000 --- a/tests/test_scripts/test_train_pd_model_config_overrides.py +++ /dev/null @@ -1,489 +0,0 @@ -from __future__ import annotations - -import json -import sys -import types - -import numpy as np -import pytest - -import scripts.train_pd_model as pd_train -from scripts.train_pd_model import ( - _apply_cli_overrides, - _apply_pd_replay_manifest, - _gate_tier, - _load_training_splits, - _normalize_sample_size, - _prepare_training_inputs, - _replay_selection_policy, - _sample_training_splits, - _select_calibration_from_backtest, - _summarize_replayed_trials, -) - - -def _base_config() -> dict[str, object]: - return { - "training_regime": {"mode": "full", "recent_window_quarters": 8}, - "stable_core": {"enabled": False}, - "hpo": {"enabled": True, "n_trials": 100}, - "challenger_pipeline": {"enabled": True}, - "validation": { - "walk_forward": {"enabled": True}, - "seed_replay": {"enabled": True}, - }, - "model": {"params": {"iterations": 500, "learning_rate": 0.05}}, - } - - -def test_apply_cli_overrides_keeps_nested_config_intent() -> None: - base = _base_config() - - updated = _apply_cli_overrides( - base, - training_regime_mode="recent", - recent_window_quarters=4, - half_life_quarters=2, - stable_core_enabled=True, - hpo_n_trials=12, - hpo_enabled=False, - challenger_enabled=False, - walk_forward_enabled=False, - seed_replay_enabled=False, - catboost_iterations=250, - ) - - assert base["training_regime"] == {"mode": "full", "recent_window_quarters": 8} - assert updated["training_regime"] == { - "mode": "recent", - "recent_window_quarters": 4, - "half_life_quarters": 2, - } - assert updated["stable_core"] == {"enabled": True} - assert updated["hpo"] == {"enabled": False, "n_trials": 12} - assert updated["challenger_pipeline"] == {"enabled": False} - assert updated["validation"] == { - "walk_forward": {"enabled": False}, - "seed_replay": {"enabled": False}, - } - assert updated["model"] == {"params": {"iterations": 250, "learning_rate": 0.05}} - - -def test_apply_pd_replay_manifest_forces_replay_without_dropping_base_params() -> None: - replay_cfg = {"selected_params": {"depth": 6, "learning_rate": 0.0573}} - - updated = _apply_pd_replay_manifest(_base_config(), replay_cfg) - - assert updated["hpo"]["enabled"] is False - assert updated["validation"]["seed_replay"]["enabled"] is False - assert updated["challenger_pipeline"]["enabled"] is False - assert updated["model"]["params"] == { - "iterations": 500, - "learning_rate": 0.0573, - "depth": 6, - } - - -def test_apply_pd_replay_manifest_requires_selected_params() -> None: - with pytest.raises(ValueError, match="selected_params"): - _apply_pd_replay_manifest(_base_config(), {"selected_params": {}}) - - -def test_select_calibration_from_backtest_uses_search_candidates( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[str] = [] - - def fake_evaluate( - method: str, - y_true: np.ndarray, - y_prob_raw: np.ndarray, - splits: list[tuple[np.ndarray, np.ndarray]], - ) -> dict[str, object]: - calls.append(method) - return { - "method": method, - "folds_used": 1, - "mean_brier": 0.1 if method == "isotonic" else 0.2, - "mean_log_loss": 0.3, - "mean_ece": 0.01, - "mean_auc_drop": 0.0, - "brier_variance": 0.0, - "ece_variance": 0.0, - "stability": 0.0, - "degradation_rate": 0.0, - "folds": [{"n": len(y_true), "p": len(y_prob_raw), "splits": len(splits)}], - } - - monkeypatch.setattr(pd_train, "_evaluate_calibration_method", fake_evaluate) - - selected, report, reports = _select_calibration_from_backtest( - run_mode="search", - replay_cfg={}, - cal_cfg={"candidates": ["platt", "isotonic"]}, - y_cal=np.array([0.0, 1.0, 0.0, 1.0]), - y_prob_tuned_cal=np.array([0.2, 0.8, 0.3, 0.7]), - cal_splits=[(np.array([0, 1]), np.array([2, 3]))], - ) - - assert calls == ["platt", "isotonic"] - assert selected == "isotonic" - assert report["selection_reason"] == "feasible_multi_metric" - assert [row["method"] for row in reports] == ["platt", "isotonic"] - - -def test_select_calibration_from_backtest_keeps_replay_method_on_diagnostic_failure( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_evaluate(*args, **kwargs): - raise RuntimeError("diagnostic fold failed") - - monkeypatch.setattr(pd_train, "_evaluate_calibration_method", fake_evaluate) - - selected, report, reports = _select_calibration_from_backtest( - run_mode="replay", - replay_cfg={"selected_calibration_method": "venn_abers"}, - cal_cfg={"candidates": ["platt", "isotonic"]}, - y_cal=np.array([0.0, 1.0]), - y_prob_tuned_cal=np.array([0.25, 0.75]), - cal_splits=[], - ) - - assert selected == "venn_abers" - assert report["selection_reason"] == "frozen_replay_manifest" - assert report["feasible_candidates"] == reports - assert reports[0]["method"] == "venn_abers" - assert reports[0]["error"] == "diagnostic fold failed" - - -def test_resolve_training_features_filters_splits_and_disables_stable_core_in_replay( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_resolve_feature_sets(*args, **kwargs): - return { - "feature_source": "fixture", - "catboost_features": ["a", "b", "missing", "rev_utilization"], - "logreg_features": ["a", "b", "rev_utilization"], - "categorical_features": ["b", "missing"], - } - - monkeypatch.setattr(pd_train, "resolve_feature_sets", fake_resolve_feature_sets) - train = pd_train.pd.DataFrame({"a": [1], "b": ["x"], "rev_utilization": [0.2]}) - cal = pd_train.pd.DataFrame({"a": [1], "b": ["x"], "rev_utilization": [0.2]}) - test = pd_train.pd.DataFrame({"a": [1], "b": ["x"], "rev_utilization": [0.2]}) - - ( - feature_source, - feature_config_path, - catboost_features, - logreg_features, - categorical_features, - stable_core_meta, - ) = pd_train._resolve_training_features( - config={ - "feature_source": {"mode": "auto", "feature_config_path": "feature_config.yml"}, - "stable_core": {"enabled": True, "exclude_features": ["rev_utilization"]}, - }, - train=train, - cal=cal, - test=test, - run_mode="search", - replay_cfg={}, - ) - - assert feature_source == "fixture" - assert feature_config_path == "feature_config.yml" - assert catboost_features == ["a", "b"] - assert logreg_features == ["a", "b"] - assert categorical_features == ["b"] - assert stable_core_meta["enabled"] is True - - ( - _replay_feature_source, - _replay_feature_config_path, - replay_catboost_features, - replay_logreg_features, - replay_categorical_features, - replay_stable_core_meta, - ) = pd_train._resolve_training_features( - config={ - "feature_source": {"mode": "auto", "feature_config_path": "feature_config.yml"}, - "stable_core": {"enabled": True, "exclude_features": ["rev_utilization"]}, - }, - train=train, - cal=cal, - test=test, - run_mode="replay", - replay_cfg={ - "feature_names": ["a", "rev_utilization"], - "categorical_features": ["rev_utilization"], - }, - ) - - assert replay_catboost_features == ["a", "rev_utilization"] - assert replay_logreg_features == ["a", "rev_utilization"] - assert replay_categorical_features == ["rev_utilization"] - assert replay_stable_core_meta == {"enabled": False, "excluded_features": []} - - -def test_replay_trial_summary_prioritizes_gate_then_ece_then_auc() -> None: - rows = [ - { - "trial_number": 1, - "validation_auc": 0.80, - "validation_brier": 0.20, - "validation_ece": 0.05, - "gate_attrs_present": True, - "gate_all_pass": False, - }, - { - "trial_number": 2, - "validation_auc": 0.79, - "validation_brier": 0.21, - "validation_ece": 0.06, - "gate_attrs_present": True, - "gate_all_pass": True, - }, - { - "trial_number": 3, - "validation_auc": 0.90, - "validation_brier": 0.19, - "validation_ece": 0.01, - "gate_attrs_present": False, - "gate_all_pass": None, - }, - ] - - summary = _summarize_replayed_trials(rows, prioritize_gate_pass=True) - - assert summary["trial_number"].tolist() == [2, 3, 1] - assert summary["gate_tier"].tolist() == [0, 1, 2] - assert _gate_tier(True, prioritize_gate_pass=True) == 0 - assert _gate_tier(None, prioritize_gate_pass=True) == 1 - assert _gate_tier(False, prioritize_gate_pass=True) == 2 - assert _gate_tier(False, prioritize_gate_pass=False) == 1 - assert _replay_selection_policy(True)["rank_order"][0] == "gate_tier(pass->unknown->fail)" - - -def test_load_training_splits_uses_config_paths_and_normalizes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[str] = [] - - def fake_read_split(path: str) -> pd_train.pd.DataFrame: - calls.append(path) - return pd_train.pd.DataFrame({"source": [path]}) - - def fake_normalize(df: pd_train.pd.DataFrame) -> pd_train.pd.DataFrame: - out = df.copy() - out["normalized"] = True - return out - - monkeypatch.setattr(pd_train, "read_split_with_fe_fallback", fake_read_split) - monkeypatch.setattr(pd_train, "_normalize_percent_columns", fake_normalize) - - train_split, cal_split, test_split = _load_training_splits( - { - "train_path": "train.parquet", - "test_path": "test.parquet", - "calibration_path": "calibration.parquet", - } - ) - - assert calls == ["train.parquet", "test.parquet", "calibration.parquet"] - assert train_split.to_dict(orient="records") == [ - {"source": "train.parquet", "normalized": True} - ] - assert test_split.to_dict(orient="records") == [{"source": "test.parquet", "normalized": True}] - assert cal_split.to_dict(orient="records") == [ - {"source": "calibration.parquet", "normalized": True} - ] - - -def test_sample_training_splits_is_deterministic_and_keeps_small_splits() -> None: - train = pd_train.pd.DataFrame({"row": range(10)}) - cal = pd_train.pd.DataFrame({"row": range(10, 20)}) - test = pd_train.pd.DataFrame({"row": [100, 101]}) - - sampled_train, sampled_cal, sampled_test = _sample_training_splits((train, cal, test), 3) - - assert sampled_train["row"].tolist() == train.sample(n=3, random_state=42)["row"].tolist() - assert sampled_cal["row"].tolist() == cal.sample(n=3, random_state=42)["row"].tolist() - assert sampled_test.equals(test) - assert sampled_train.index.tolist() == [0, 1, 2] - assert sampled_cal.index.tolist() == [0, 1, 2] - - -@pytest.mark.parametrize("raw_sample_size", [None, 0, -5]) -def test_normalize_sample_size_treats_non_positive_as_full_data( - raw_sample_size: int | None, -) -> None: - assert _normalize_sample_size(raw_sample_size) is None - - -def test_normalize_sample_size_keeps_positive_integer() -> None: - assert _normalize_sample_size(50) == 50 - - -def test_prepare_training_inputs_builds_model_ready_frames() -> None: - train = pd_train.pd.DataFrame( - { - "issue_d": pd_train.pd.date_range("2018-01-01", periods=10, freq="MS"), - "default_flag": [0, 1] * 5, - "score": [0.1, 0.2, None, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], - "grade": ["A", "B", None, "C", "A", "B", "C", "A", "B", "C"], - "_recency_weight": [1.0 + i for i in range(10)], - } - ) - cal = pd_train.pd.DataFrame( - { - "issue_d": pd_train.pd.date_range("2019-01-01", periods=2, freq="MS"), - "default_flag": [0, 1], - "score": [0.3, 0.4], - "grade": ["A", "C"], - } - ) - test = pd_train.pd.DataFrame( - { - "issue_d": pd_train.pd.date_range("2020-01-01", periods=2, freq="MS"), - "default_flag": [1, 0], - "score": [None, 0.9], - "grade": ["B", None], - } - ) - - ( - train_val, - train_fit_weights, - train_val_weights, - y_train_fit, - _y_val, - y_cal, - y_test, - x_train_fit_cb, - _x_val_cb, - _x_cal_cb, - x_test_cb, - x_train_fit_lr, - x_test_lr, - lr_fill, - ) = _prepare_training_inputs( - train, - cal, - test, - catboost_features=["score", "grade"], - categorical_features=["grade"], - logreg_features=["score"], - val_fraction=0.2, - ) - - assert len(x_train_fit_cb) == 8 - assert len(train_val) == 2 - assert train_fit_weights.tolist() == [1.0 + i for i in range(8)] - assert train_val_weights.tolist() == [9.0, 10.0] - assert pd_train.pd.api.types.is_integer_dtype(y_train_fit) - assert y_cal.tolist() == [0, 1] - assert y_test.tolist() == [1, 0] - assert x_train_fit_cb.columns.tolist() == ["score", "grade"] - assert x_test_cb["grade"].tolist() == ["B", "UNKNOWN"] - assert x_test_lr.isna().sum().sum() == 0 - assert lr_fill["score"] == x_train_fit_lr["score"].median() - - -def test_evaluate_walk_forward_stage_reports_disabled_without_training( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fail_if_called(*args, **kwargs): - raise AssertionError("walk-forward evaluator should not run when disabled") - - monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fail_if_called) - - report = pd_train._evaluate_walk_forward_stage( - enabled=False, - walk_cfg={"n_windows": 5}, - train=pd_train.pd.DataFrame(), - catboost_features=[], - categorical_features=[], - model_params={}, - ) - - assert report == { - "enabled": False, - "reason": "disabled_in_config", - "n_windows_requested": 5, - "n_windows_used": 0, - "folds": [], - } - - -def test_evaluate_walk_forward_stage_normalizes_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, object] = {} - - def fake_evaluate(train_df: pd_train.pd.DataFrame, **kwargs): - captured["train_rows"] = len(train_df) - captured.update(kwargs) - return {"enabled": True, "n_windows_used": 1} - - monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fake_evaluate) - - report = pd_train._evaluate_walk_forward_stage( - enabled=True, - walk_cfg={ - "n_windows": "4", - "min_train_rows": "12", - "window_rows": "6", - "date_col": "issue_d", - "max_rows": "0", - }, - train=pd_train.pd.DataFrame({"x": [1, 2]}), - catboost_features=["x"], - categorical_features=[], - model_params={"depth": 4}, - ) - - assert report == {"enabled": True, "n_windows_used": 1} - assert captured["features"] == ["x"] - assert captured["target"] == pd_train.TARGET - assert captured["params"] == {"depth": 4} - assert captured["n_windows"] == 4 - assert captured["min_train_rows"] == 12 - assert captured["window_rows"] == 6 - assert captured["max_rows"] is None - - -def test_export_shap_feature_importance_writes_summary( - tmp_path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_catboost = types.ModuleType("catboost") - - class FakePool: - def __init__(self, frame, cat_features): - self.frame = frame - self.cat_features = cat_features - - fake_catboost.Pool = FakePool - monkeypatch.setitem(sys.modules, "catboost", fake_catboost) - - class FakeModel: - def get_feature_importance(self, *, type: str, data: FakePool): - assert type == "ShapValues" - assert data.cat_features == ["grade"] - return np.array([[0.1, 0.4, 0.5], [0.3, -0.2, 0.5]]) - - shap_dir = tmp_path / "shap" - result = pd_train._export_shap_feature_importance( - cb_tuned_model=FakeModel(), - X_test_cb=pd_train.pd.DataFrame({"score": [1.0, 2.0], "grade": ["A", "B"]}), - categorical_features=["grade"], - catboost_features=["score", "grade"], - shap_dir=shap_dir, - ) - - summary = json.loads((shap_dir / "shap_feature_importance.json").read_text(encoding="utf-8")) - assert result == {"exported": True, "n_features": 2, "path": str(shap_dir)} - assert (shap_dir / "shap_values_test.npz").exists() - assert summary["expected_value"] == 0.5 - assert summary["top_features"][0]["feature"] == "grade" diff --git a/tests/test_scripts/test_validate_conformal_experiment.py b/tests/test_scripts/test_validate_conformal_experiment.py deleted file mode 100644 index 72a9d94..0000000 --- a/tests/test_scripts/test_validate_conformal_experiment.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for scripts/validate_conformal_experiment.py.""" - -from __future__ import annotations - -from scripts.validate_conformal_experiment import _build_paths - - -def test_build_paths_uses_namespaced_artifact_locations() -> None: - paths = _build_paths("abc/def") - - assert paths["data_dir"].as_posix().endswith("data/processed/conformal_gap/abc_def") - assert paths["models_dir"].as_posix().endswith("models/conformal_gap/abc_def") - assert ( - paths["policy_status"] - .as_posix() - .endswith("models/conformal_gap/abc_def/conformal_policy_status.json") - ) diff --git a/tests/test_scripts/test_validate_conformal_policy.py b/tests/test_scripts/test_validate_conformal_policy.py deleted file mode 100644 index 2078000..0000000 --- a/tests/test_scripts/test_validate_conformal_policy.py +++ /dev/null @@ -1,715 +0,0 @@ -"""Tests for conformal policy validation v2 checks.""" - -from __future__ import annotations - -import json -import pickle - -import numpy as np -import pandas as pd -import pytest -import yaml - -from scripts import validate_conformal_policy as policy_mod - - -@pytest.fixture(autouse=True) -def _set_run_tag(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PIPELINE_RUN_TAG", "run-conformal-test") - - -def test_compute_mapie_mwi_score_uses_current_mapie_signature( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def _fake_current_score( - y_true: np.ndarray, - y_pis: np.ndarray, - *, - confidence_level: float, - ) -> float: - assert y_true.shape == (3,) - assert y_pis.shape == (3, 2, 1) - assert confidence_level == pytest.approx(0.90) - return 1.23 - - monkeypatch.setattr(policy_mod, "_mapie_mwi_score", _fake_current_score) - - score = policy_mod._compute_mapie_mwi_score( - np.array([0.1, 0.2, 0.3]), - np.array([0.0, 0.1, 0.2]), - np.array([0.2, 0.3, 0.4]), - confidence_level=0.90, - ) - - assert score == pytest.approx(1.23) - - -def test_compute_mapie_mwi_score_keeps_legacy_alpha_signature( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def _fake_legacy_score(*_args, **_kwargs): - if "confidence_level" in _kwargs: - raise TypeError("legacy signature") - assert _kwargs["alpha_"] == pytest.approx(0.10) - return np.array([1.10, 1.30]) - - monkeypatch.setattr(policy_mod, "_mapie_mwi_score", _fake_legacy_score) - - score = policy_mod._compute_mapie_mwi_score( - np.array([0.1, 0.2]), - np.array([0.0, 0.1]), - np.array([0.2, 0.3]), - confidence_level=0.90, - ) - - assert score == pytest.approx(1.20) - - -def test_interval_arrays_filters_invalid_interval_rows() -> None: - intervals = pd.DataFrame( - { - "y_true": [0.1, None, 0.3, 0.4], - "pd_low_90": [0.0, 0.1, np.nan, 0.2], - "pd_high_90": [0.2, 0.3, 0.4, 0.6], - } - ) - - y_true, lower, upper = policy_mod._interval_arrays( - intervals, - lower_col="pd_low_90", - upper_col="pd_high_90", - ) - - assert list(y_true) == [0.1, 0.4] - assert list(lower) == [0.0, 0.2] - assert list(upper) == [0.2, 0.6] - - -def test_winkler_90_check_allows_documented_compensated_band() -> None: - policy = { - "target_coverage_90_min": 0.90, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.80, - "max_critical_alerts": 0, - "max_winkler_90": 1.00, - "enable_compensated_winkler_90": True, - "compensated_winkler_90_max": 1.20, - } - metrics = { - "winkler_90": 1.10, - "coverage_90": 0.91, - "min_group_coverage_90": 0.89, - "avg_width_90": 0.70, - "critical_alerts": 0.0, - } - - check = policy_mod._winkler_90_check(policy, metrics) - - assert check["passed"] is True - assert check["raw_passed"] is False - assert check["compensated_passed"] is True - assert check["policy_mode"] == "compensated_band" - - -def test_validate_conformal_policy_includes_material_gate_checks(tmp_path) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.91, "avg_interval_width": 0.4}, - "metrics_95": {"empirical_coverage": 0.96, "avg_interval_width": 0.6}, - }, - f, - ) - - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.9, 0.89]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01"]), - "coverage_90": [0.9, 0.91], - "coverage_95": [0.95, 0.96], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - - rng = np.random.RandomState(42) - y_true = rng.uniform(0.0, 1.0, 200) - intervals = pd.DataFrame( - { - "y_true": y_true, - "pd_low_90": np.clip(y_true - 0.15, 0.0, 1.0), - "pd_high_90": np.clip(y_true + 0.15, 0.0, 1.0), - "pd_low_95": np.clip(y_true - 0.20, 0.0, 1.0), - "pd_high_95": np.clip(y_true + 0.20, 0.0, 1.0), - } - ) - intervals.to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 10.0, - "max_winkler_95": 10.0, - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - policy_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - checks = pd.read_parquet(data_dir / "conformal_policy_checks.parquet") - - assert status["schema_version"] - assert status["generated_at_utc"] - assert status["run_tag"] - assert "winkler_90" in status - assert "kupiec_pvalue_90" not in status - assert "christoffersen_pvalue_90" not in status - assert status["checks_total"] == 9 - assert "statistical_coverage" not in set(checks["scope"]) - assert "lgd_ead_conformal_status" in status - assert status["strict_overall_pass"] is True - assert status["diagnostic_checks_total"] == 0 - assert "methodological_justification_pass" in status - assert status["failing_statistical_checks"] == [] - assert "sample_size_context" in status - - -def test_validate_conformal_policy_falls_back_to_official_baseline_run_tag( - tmp_path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.delenv("PIPELINE_RUN_TAG", raising=False) - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.91, "avg_interval_width": 0.4}, - "metrics_95": {"empirical_coverage": 0.96, "avg_interval_width": 0.6}, - }, - f, - ) - pd.DataFrame({"group": ["A"], "coverage_90": [0.9]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01"]), - "coverage_90": [0.91], - "coverage_95": [0.96], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - pd.DataFrame( - { - "y_true": np.linspace(0.1, 0.9, 10), - "pd_low_90": np.linspace(0.0, 0.8, 10), - "pd_high_90": np.linspace(0.2, 1.0, 10), - "pd_low_95": np.linspace(0.0, 0.75, 10), - "pd_high_95": np.linspace(0.25, 1.0, 10), - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 10.0, - "max_winkler_95": 10.0, - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - monkeypatch.setattr(policy_mod, "resolve_official_baseline_run_tag", lambda: "run-official") - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - policy_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - assert status["run_tag"] == "run-official" - - -def test_validate_conformal_policy_supports_artifact_namespace( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - data_dir = tmp_path / "data" / "processed" / "conformal_gap" / "shadow_ns" - model_dir = tmp_path / "models" / "conformal_gap" / "shadow_ns" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.91, "avg_interval_width": 0.4}, - "metrics_95": {"empirical_coverage": 0.96, "avg_interval_width": 0.6}, - }, - f, - ) - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.91, 0.90]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01"]), - "coverage_90": [0.91, 0.92], - "coverage_95": [0.95, 0.96], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - pd.DataFrame( - { - "y_true": np.linspace(0.1, 0.9, 20), - "pd_low_90": np.linspace(0.0, 0.75, 20), - "pd_high_90": np.linspace(0.25, 1.0, 20), - "pd_low_95": np.linspace(0.0, 0.70, 20), - "pd_high_95": np.linspace(0.30, 1.0, 20), - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 10.0, - "max_winkler_95": 10.0, - }, - "artifacts": {}, - "output": {}, - } - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - policy_mod.main(str(cfg_path), artifact_namespace="shadow_ns") - - status_path = ( - tmp_path / "models" / "conformal_gap" / "shadow_ns" / "conformal_policy_status.json" - ) - assert status_path.exists() - status = json.loads(status_path.read_text(encoding="utf-8")) - assert status["artifact_namespace"] == "shadow_ns" - assert not (tmp_path / "models" / "conformal_policy_status.json").exists() - - -def test_validate_conformal_policy_ignores_retired_backtest_thresholds( - tmp_path, monkeypatch: pytest.MonkeyPatch -) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.915, "avg_interval_width": 0.4}, - "metrics_95": {"empirical_coverage": 0.958, "avg_interval_width": 0.6}, - }, - f, - ) - - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.91, 0.90]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01"]), - "coverage_90": [0.91, 0.92], - "coverage_95": [0.95, 0.96], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - pd.DataFrame( - { - "y_true": np.linspace(0.05, 0.95, 50), - "pd_low_90": np.linspace(0.00, 0.80, 50), - "pd_high_90": np.linspace(0.20, 1.00, 50), - "pd_low_95": np.linspace(0.00, 0.75, 50), - "pd_high_95": np.linspace(0.25, 1.00, 50), - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 10.0, - "max_winkler_95": 10.0, - "allow_methodological_justification": True, - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - def _fake_winkler(_y_true, _lower, _upper, *, alpha): - return np.full(120, 1.203 if float(alpha) == 0.10 else 1.10) - - monkeypatch.setattr(policy_mod, "winkler_interval_score", _fake_winkler) - - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - - policy_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - - assert status["overall_pass"] is True - assert status["gate_overall_pass"] is True - assert status["strict_overall_pass"] is True - assert status["non_statistical_checks_pass"] is True - assert status["diagnostic_statistical_pass"] is True - assert status["methodological_justification_pass"] is True - assert status["methodological_justification_status"] == "not_needed_material_gate_pass" - assert status["gate_checks_passed"] == status["gate_checks_total"] == 9 - assert status["diagnostic_checks_passed"] == 0 - assert status["diagnostic_checks_total"] == 0 - assert "retired_backtest_checks" in status["methodological_justification"] - assert status["failing_non_statistical_checks"] == [] - assert status["failing_statistical_checks"] == [] - - -def test_validate_conformal_policy_reports_winkler_sensitivity_without_overwriting_main_status( - tmp_path, -) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.912, "avg_interval_width": 0.75}, - "metrics_95": {"empirical_coverage": 0.955, "avg_interval_width": 0.90}, - }, - f, - ) - - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.90, 0.89]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01", "2025-03-01"]), - "coverage_90": [0.91, 0.92, 0.90], - "coverage_95": [0.95, 0.96, 0.95], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - - y_true = np.linspace(0.10, 0.90, 120) - pd_low_90 = np.clip(y_true - 0.15, 0.0, 1.0) - pd_high_90 = np.clip(y_true + 0.15, 0.0, 1.0) - pd_low_95 = np.clip(y_true - 0.20, 0.0, 1.0) - pd_high_95 = np.clip(y_true + 0.20, 0.0, 1.0) - - miss_idx = np.arange(108, 120) - pd_low_90[miss_idx] = np.clip(y_true[miss_idx] - 0.758, 0.0, 1.0) - pd_high_90[miss_idx] = np.clip(y_true[miss_idx] - 0.458, 0.0, 1.0) - pd_low_95[miss_idx] = np.clip(y_true[miss_idx] - 0.80, 0.0, 1.0) - pd_high_95[miss_idx] = np.clip(y_true[miss_idx] - 0.42, 0.0, 1.0) - - intervals = pd.DataFrame( - { - "y_true": y_true, - "pd_low_90": pd_low_90, - "pd_high_90": pd_high_90, - "pd_low_95": pd_low_95, - "pd_high_95": pd_high_95, - } - ) - intervals.to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 1.00, - "max_winkler_95": 10.0, - }, - "policy_sensitivity": { - "max_winkler_90_values": [1.20, 1.22, 1.25], - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - policy_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - - assert status["overall_pass"] is False - assert status["failing_non_statistical_checks"] == ["winkler_90"] - assert "policy_sensitivity" in status - results = status["policy_sensitivity"]["results"] - assert [row["max_winkler_90"] for row in results] == [1.2, 1.22, 1.25] - assert results[0]["strict_overall_pass"] is False - assert any(row["strict_overall_pass"] for row in results[1:]) - - -def test_validate_conformal_policy_allows_compensated_winkler_band( - tmp_path, -) -> None: - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.928, "avg_interval_width": 0.756}, - "metrics_95": {"empirical_coverage": 0.960, "avg_interval_width": 0.936}, - }, - f, - ) - - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.887, 0.92]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01", "2025-03-01"]), - "coverage_90": [0.92, 0.93, 0.94], - "coverage_95": [0.96, 0.96, 0.97], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - - y_true = np.linspace(0.10, 0.90, 120) - low90 = np.clip(y_true - 0.378, 0.0, 1.0) - high90 = np.clip(y_true + 0.378, 0.0, 1.0) - low95 = np.clip(y_true - 0.468, 0.0, 1.0) - high95 = np.clip(y_true + 0.468, 0.0, 1.0) - pd.DataFrame( - { - "y_true": y_true, - "pd_low_90": low90, - "pd_high_90": high90, - "pd_low_95": low95, - "pd_high_95": high95, - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.80, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 1.00, - "enable_compensated_winkler_90": True, - "compensated_winkler_90_max": 1.22, - "compensated_min_coverage_90": 0.92, - "compensated_min_group_coverage_90": 0.885, - "compensated_max_avg_width_90": 0.80, - "max_winkler_95": 10.0, - "allow_methodological_justification": True, - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - policy_mod.main(str(cfg_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - - assert status["strict_overall_pass"] is True - assert status["non_statistical_checks_pass"] is True - assert status["methodological_justification_pass"] is True - assert status["failing_non_statistical_checks"] == [] - assert status["winkler_90_policy_pass"] is True - - -def test_validate_conformal_policy_sensitivity_config_overrides_policy_sensitivity( - tmp_path, -) -> None: - """--sensitivity-config flag overrides policy_sensitivity from the main config.""" - data_dir = tmp_path / "data" / "processed" - model_dir = tmp_path / "models" - data_dir.mkdir(parents=True) - model_dir.mkdir(parents=True) - - with open(model_dir / "conformal_results_mondrian.pkl", "wb") as f: - pickle.dump( - { - "metrics_90": {"empirical_coverage": 0.912, "avg_interval_width": 0.75}, - "metrics_95": {"empirical_coverage": 0.955, "avg_interval_width": 0.90}, - }, - f, - ) - - pd.DataFrame({"group": ["A", "B"], "coverage_90": [0.90, 0.89]}).to_parquet( - data_dir / "conformal_group_metrics_mondrian.parquet", index=False - ) - pd.DataFrame( - { - "month": pd.to_datetime(["2025-01-01", "2025-02-01"]), - "coverage_90": [0.91, 0.92], - "coverage_95": [0.95, 0.96], - } - ).to_parquet(data_dir / "conformal_backtest_monthly.parquet", index=False) - pd.DataFrame(columns=["severity"]).to_parquet( - data_dir / "conformal_backtest_alerts.parquet", index=False - ) - - y_true = np.linspace(0.10, 0.90, 120) - pd.DataFrame( - { - "y_true": y_true, - "pd_low_90": np.clip(y_true - 0.378, 0.0, 1.0), - "pd_high_90": np.clip(y_true + 0.378, 0.0, 1.0), - "pd_low_95": np.clip(y_true - 0.468, 0.0, 1.0), - "pd_high_95": np.clip(y_true + 0.468, 0.0, 1.0), - } - ).to_parquet(data_dir / "conformal_intervals_mondrian.parquet", index=False) - - # Main config has NO policy_sensitivity (or different values) - cfg = { - "policy": { - "target_coverage_90_min": 0.90, - "target_coverage_95_min": 0.95, - "min_group_coverage_90_min": 0.88, - "max_avg_width_90": 0.8, - "max_critical_alerts": 0, - "max_total_alerts": 5, - "max_warning_alerts": 5, - "max_winkler_90": 1.00, - "max_winkler_95": 10.0, - }, - "artifacts": { - "conformal_results_path": str(model_dir / "conformal_results_mondrian.pkl"), - "group_metrics_path": str(data_dir / "conformal_group_metrics_mondrian.parquet"), - "backtest_monthly_path": str(data_dir / "conformal_backtest_monthly.parquet"), - "backtest_alerts_path": str(data_dir / "conformal_backtest_alerts.parquet"), - "intervals_path": str(data_dir / "conformal_intervals_mondrian.parquet"), - }, - "output": { - "policy_status_json": str(model_dir / "conformal_policy_status.json"), - "policy_checks_parquet": str(data_dir / "conformal_policy_checks.parquet"), - }, - } - - # Sensitivity config injects thresholds to test - sensitivity_cfg = {"policy_sensitivity": {"max_winkler_90_values": [1.20, 1.25]}} - - cfg_path = tmp_path / "conformal_policy.yaml" - cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8") - sens_path = tmp_path / "conformal_policy_sensitivity.yaml" - sens_path.write_text(yaml.safe_dump(sensitivity_cfg), encoding="utf-8") - - policy_mod.main(str(cfg_path), sensitivity_config_path=str(sens_path)) - - status = json.loads((model_dir / "conformal_policy_status.json").read_text(encoding="utf-8")) - - assert "policy_sensitivity" in status - results = status["policy_sensitivity"]["results"] - assert [row["max_winkler_90"] for row in results] == [1.20, 1.25] diff --git a/tests/test_submission_preview_layout.py b/tests/test_submission_preview_layout.py new file mode 100644 index 0000000..08d4308 --- /dev/null +++ b/tests/test_submission_preview_layout.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_print_css_does_not_create_a_trailing_grid_page() -> None: + css = (ROOT / "paper" / "ijds.css").read_text(encoding="utf-8") + + assert "#quarto-content {\n display: block !important;" in css + assert "grid-template-rows: none !important;" in css + assert "break-after: auto !important;" in css + assert "page-break-after: auto !important;" in css + assert "break-after: avoid-page" not in css diff --git a/tests/test_supplement_table_sync.py b/tests/test_supplement_table_sync.py index a425dcd..a32f404 100644 --- a/tests/test_supplement_table_sync.py +++ b/tests/test_supplement_table_sync.py @@ -1,71 +1,269 @@ -"""Drift guards for the diagnostic multidataset evidence. - -Chapter 30 retains the detailed A25/A34 values. The submitted body and -supplement deliberately keep only the scope boundary: these older replications -are static transfer evidence, not active-policy certificates. Tests preserve -both contracts without forcing retired detail back into the IJDS narrative. -""" +"""Sync hand-authored supplement summaries to active IJDS evidence tables.""" from __future__ import annotations import csv +import re from pathlib import Path -import pytest - REPO = Path(__file__).resolve().parents[1] -TABLES = REPO / "reports" / "crpto" / "tables" +TABLES = REPO / "reports/crpto/tables" +SUPPLEMENT = REPO / "paper/supplement_ijds.qmd" -SUPPLEMENT = REPO / "paper" / "supplement_ijds.qmd" -PAPER = REPO / "paper" / "CRPTO_ijds.qmd" -BOOK_CH30 = REPO / "book" / "chapters" / "30-replicacion-multidataset.qmd" +def _rows(name: str) -> list[dict[str, str]]: + with (TABLES / name).open(encoding="utf-8") as handle: + return list(csv.DictReader(handle)) -def _read_rows(name: str) -> list[dict[str, str]]: - path = TABLES / name - if not path.is_file(): - pytest.skip(f"{name} not present locally.") - with path.open(encoding="utf-8") as fh: - return list(csv.DictReader(fh)) +def test_all_primary_coverage_bounds_are_visible_in_supplement() -> None: + rows = _rows("crpto_ijds_v4_table1_coverage_windows.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") -def _text(path: Path) -> str: - if not path.is_file(): - pytest.skip(f"{path} not present locally.") - return path.read_text(encoding="utf-8") + assert len(rows) == 16 + assert {row["learner"] for row in rows} == { + "catboost_platt", + "numeric_logistic_platt", + } + for row in rows: + assert f"{float(row['coverage_lower']):.6f}" in supplement + assert f"{float(row['coverage_upper']):.6f}" in supplement + assert f"{float(row['coverage_resolved']):.6f}" in supplement -def test_a25_robust_objectives_match_book_and_diagnostic_status() -> None: - """A25 values remain in the book while IJDS surfaces retain their boundary.""" - rows = _read_rows("crpto_tableA25_external_replication_gate.csv") - book = _text(BOOK_CH30) - missing: list[str] = [] +def test_complete_phase_path_is_visible_in_supplement() -> None: + rows = _rows("crpto_ijds_v4_table2_phase_transition.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(rows) == 8 for row in rows: - dollars = f"${round(float(row['robust_objective'])):,}" - if dollars not in book: - missing.append(f"{row['dataset']} {dollars} missing in {BOOK_CH30.name}") - assert not missing, "A25 robust objective drift:\n" + "\n".join(missing) - - supplement = _text(SUPPLEMENT) - paper = _text(PAPER) - assert "A25--A34" in supplement - assert "Static transfer evidence; not active Lending Club certificates." in supplement - assert "Prosper" in paper and "Freddie/Mendeley" in paper - assert "retained as diagnostics or external context" in paper - - -def test_a34_price_of_robustness_matches_historical_book_surface() -> None: - """A34 signed price-of-robustness values remain traceable in chapter 30.""" - rows = _read_rows("crpto_tableA34_price_of_robustness_cross_dataset.csv") - book = _text(BOOK_CH30) - missing: list[str] = [] + for field in ("fit_prevalence", "fit_residual_quantile", "mean_width"): + assert f"{float(row[field]):.6f}" in supplement + + +def test_credit_control_metrics_and_shift_diagnostics_are_visible() -> None: + controls = _rows("crpto_ijds_v4_table6_credit_controls.csv") + woe = _rows("crpto_ijds_v4_tableS3_woe_iv_psi.csv") + score_psi = _rows("crpto_ijds_v4_tableS4_score_psi.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(controls) == 5 + assert len(woe) == 45 + assert len(score_psi) == 25 + for row in controls: + assert row["learner_label"] in supplement + assert f"{float(row['roc_auc']):.6f}" in supplement + assert f"{float(row['brier']):.6f}" in supplement + assert f"{float(row['calibration_slope']):.6f}" in supplement + assert f"{float(row['coverage_upper_max']):.6f}" in supplement + + top_iv = sorted(woe, key=lambda row: float(row["iv"]), reverse=True)[:5] + for row in top_iv: + assert row["feature"] in supplement + assert f"{float(row['iv']):.6f}" in supplement + assert f"{float(row['primary_oot_psi']):.6f}" in supplement + + primary_psi = [row for row in score_psi if row["comparison_role"] == "primary_oot"] + assert len(primary_psi) == 5 + for row in primary_psi: + assert f"{float(row['psi']):.6f}" in supplement + + +def test_named_and_exact_direction_counts_are_visible_in_supplement() -> None: + named = _rows("crpto_ijds_v4_tableS1_named_comparators.csv") + directions = _rows("crpto_ijds_v4_table4_direction_summary.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(named) == 9 + assert len(directions) == 5 + for row in named: + for field in ("guardrail_lower", "crosses_zero", "guardrail_higher"): + assert f"| {int(row[field])} |" in supplement + labels = { + "standardized_payoff": "Status-indexed payoff proxy", + "terminal_default": "Terminal default", + "funded_miscoverage": "Funded miscoverage", + } + by_metric = { + metric: { + row["direction"]: int(row["cells"]) for row in directions if row["metric"] == metric + } + for metric in labels + } + for metric, label in labels.items(): + counts = by_metric[metric] + lower = counts.get("guardrail_lower", 0) + crossing = counts.get("crosses_zero", 0) + higher = counts.get("guardrail_higher", 0) + assert f"| {label} | {lower} | {crossing} | {higher} | 72 |" in supplement + + +def test_two_ruler_tracks_and_repeated_quarter_contrast_are_visible() -> None: + rows = _rows("crpto_ijds_v4_table5_two_ruler_tracks.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(rows) == 6 + assert {(row["ruler"], float(row["coordinate"])) for row in rows} == { + (ruler, coordinate) + for ruler in ("objective_matched", "normalized_score") + for coordinate in (0.25, 0.5, 0.75) + } for row in rows: - pct = f"+{float(row['price_of_robustness_pct']) * 100:.2f}%" - if pct not in book: - missing.append(f"{row['application']} {pct} missing in {BOOK_CH30.name}") - assert not missing, "A34 price-of-robustness drift:\n" + "\n".join(missing) - - supplement = _text(SUPPLEMENT) - assert "older frozen replication contracts" in supplement - assert "cannot be quoted as direct" in supplement - assert "replications of the active midpoint policy" in supplement + assert f"{float(row['payoff_bound_usd_lower_min']):,.2f}" in supplement + assert f"{float(row['payoff_bound_usd_upper_max']):,.2f}" in supplement + assert f"{float(row['default_bound_pp_lower_min']):.4f}" in supplement + assert f"{float(row['default_bound_pp_upper_max']):.4f}" in supplement + assert f"{float(row['payoff_identification_width_usd_min']):,.0f}" in supplement + assert f"{float(row['payoff_identification_width_usd_max']):,.0f}" in supplement + normalized = re.sub(r"\s+", " ", supplement.lower()) + assert "44 loan-month positions" in normalized + assert "155,937.27" in normalized + assert "one repeated allocation, not eight independent confirmations" in normalized + assert "all three sharp intervals cross zero in all eight windows" in normalized + assert "exact identification-width ranges" in normalized + assert "endpoint-recovery direction reconciliation" not in normalized + + +def test_label_lag_sensitivity_is_visible_in_supplement() -> None: + rows = _rows("crpto_ijds_v4_tableS5_label_lag_sensitivity.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(rows) == 40 + assert {int(row["charged_off_lag_months"]) for row in rows} == {0, 3, 6, 8, 12} + scoped = [row for row in rows if row["window_id"].startswith(("w07_", "w08_"))] + assert len(scoped) == 10 + for row in scoped: + assert f"{float(row['phase_prevalence']):.6f}" in supplement + assert f"{float(row['phase_residual_quantile']):.6f}" in supplement + + +def test_endpoint_availability_grid_is_visible_and_kept_separate() -> None: + rows = _rows("crpto_ijds_v4_tableS6_endpoint_availability_sensitivity.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(rows) == 5 + assert {int(row["charged_off_lag_months"]) for row in rows} == {0, 3, 6, 8, 12} + for row in rows: + lag = int(row["charged_off_lag_months"]) + resolved = int(row["primary_resolved"]) + unresolved = int(row["primary_unresolved"]) + below = int(row["coverage_upper_below_0_90_cells"]) + maximum = float(row["coverage_upper_max"]) + payoff_lower = int(row["two_ruler_payoff_gamma_1_lower_cells"]) + payoff_cross = int(row["two_ruler_payoff_crosses_zero_cells"]) + default_higher = int(row["two_ruler_default_gamma_1_higher_cells"]) + default_cross = int(row["two_ruler_default_crosses_zero_cells"]) + miscoverage_higher = int(row["two_ruler_miscoverage_gamma_1_higher_cells"]) + miscoverage_cross = int(row["two_ruler_miscoverage_crosses_zero_cells"]) + expected = ( + f"| {lag} | {resolved:,} / {unresolved:,} | {below} / 40 | {maximum:.6f} | " + f"{payoff_lower} / {payoff_cross} | {default_higher} / {default_cross} | " + f"{miscoverage_higher} / {miscoverage_cross} |" + ) + assert expected in supplement + + normalized = re.sub(r"\s+", " ", supplement.lower()) + assert "fit-label-by-endpoint combinations had been evaluated" in normalized + assert "active six-month result remains the declared endpoint" in normalized + + +def test_complete_portfolio_structure_grid_is_visible_in_supplement() -> None: + rows = _rows("crpto_ijds_v4_tableS7_portfolio_structure_sensitivity.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(rows) == 36 + assert {float(row["budget"]) for row in rows} == {500_000.0, 1_000_000.0, 2_000_000.0} + assert {float(row["purpose_cap"]) for row in rows} == {0.2, 0.25, 0.3, 1.0} + assert {float(row["lgd"]) for row in rows} == {0.25, 0.45, 0.65} + assert {int(row["activity_portfolios"]) for row in rows} == {1440} + assert {float(row["activity_frontier_constraint_binding_share"]) for row in rows} == {1.0} + for row in rows: + payoff = "/".join( + row[column] + for column in ( + "standardized_payoff_gamma_1_lower_cells", + "standardized_payoff_gamma_1_higher_cells", + "standardized_payoff_crosses_zero_cells", + "standardized_payoff_exact_zero_cells", + ) + ) + default = "/".join( + row[column] + for column in ( + "funded_default_gamma_1_higher_cells", + "funded_default_gamma_1_lower_cells", + "funded_default_crosses_zero_cells", + "funded_default_exact_zero_cells", + ) + ) + miscoverage = "/".join( + row[column] + for column in ( + "funded_binary_miscoverage_gamma_1_higher_cells", + "funded_binary_miscoverage_gamma_1_lower_cells", + "funded_binary_miscoverage_crosses_zero_cells", + "funded_binary_miscoverage_exact_zero_cells", + ) + ) + expected = ( + f"| {float(row['budget']) / 1_000_000:.1f} | " + f"{float(row['purpose_cap']):.2f} | {float(row['lgd']):.2f} | " + f"{payoff} | {default} | {miscoverage} | " + f"{float(row['activity_purpose_cap_binding_share']):.0%} |" + ) + assert expected in supplement + + normalized = re.sub(r"\s+", " ", supplement.lower()) + assert "zero scenarios are favorable on all three metrics" in normalized + assert "zero are adverse on all three metrics" in normalized + assert "share of the 1,440 portfolios" in normalized + assert "baseline scenario reproduces the active two-ruler bounds exactly" in normalized + + +def test_endpoint_reason_missingness_and_second_origin_tables_are_visible() -> None: + endpoint = _rows("crpto_ijds_v4_tableS8_endpoint_resolution.csv") + missingness = _rows("crpto_ijds_v4_tableS9_missingness_encoding_sensitivity.csv") + rolling = _rows("crpto_ijds_v4_tableS10_rolling_origin_recurrence.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(endpoint) == 5 + assert sum(int(row["candidate_rows"]) for row in endpoint) == 376890 + for row in endpoint: + assert f"{int(row['candidate_rows']):,}" in supplement + + assert len(missingness) == 3 + for row in missingness: + assert f"{float(row['roc_auc']):.6f}" in supplement + assert f"{float(row['coverage_upper_max']):.6f}" in supplement + + assert len(rolling) == 16 + assert {row["origin"] for row in rolling} == {"primary_2016", "rolling_2017"} + for row in rolling: + assert f"{float(row['coverage_lower']):.6f}" in supplement + assert f"{float(row['coverage_upper']):.6f}" in supplement + + normalized = re.sub(r"\s+", " ", supplement.lower()) + assert "not an independent replication" in normalized + assert "does not identify a missingness mechanism" in normalized + + +def test_fit_completion_and_allocation_granularity_tables_are_visible() -> None: + fit = _rows("crpto_ijds_v4_tableS11_fit_label_completion.csv") + granularity = _rows("crpto_ijds_v4_tableS12_allocation_granularity.csv") + supplement = SUPPLEMENT.read_text(encoding="utf-8") + + assert len(fit) == 4 + assert sum(row["w7_w8_stratum2_crossing"] == "True" for row in fit) == 3 + for row in fit: + assert f"{float(row['coverage_lower_min']):.6f}" in supplement + assert f"{float(row['coverage_upper_max']):.6f}" in supplement + + assert len(granularity) == 1 + row = granularity[0] + assert int(row["portfolios"]) == 1440 + assert int(row["changed_rows"]) == 2985 + assert f"{float(row['default_rate_perturbation_abs_max']):.9f}" in supplement + + normalized = re.sub(r"\s+", " ", supplement.lower()) + assert "declared stresses rather than sharp bounds" in normalized + assert "does not establish integer optimality" in normalized diff --git a/tests/test_utils/test_mlflow_tracing.py b/tests/test_utils/test_mlflow_tracing.py deleted file mode 100644 index 54b603a..0000000 --- a/tests/test_utils/test_mlflow_tracing.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Smoke tests for ``src.utils.mlflow_tracing``. - -The module must degrade gracefully when MLflow is not installed AND remain -importable even with MLflow present. Real run/artifact logging is exercised by -the scripts themselves; here we only test the public surface contracts. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import pytest - -from src.utils import mlflow_tracing -from src.utils.mlflow_tracing import ( - PAPER_RUN_TAG, - paper_run, - register_parquet_dataset, - set_paper_tags, - trace, -) - - -def test_paper_run_tag_constant_matches_champion() -> None: - assert PAPER_RUN_TAG == "ijds-rebaseline-2026-06-07" - - -def test_trace_decorator_is_transparent_when_mlflow_missing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """When MLflow is unavailable, ``@trace`` must return the function unchanged.""" - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", False) - - @trace(name="op") - def f(x: int) -> int: - return x * 2 - - assert f(3) == 6 - # Must not wrap the function (same identity, same docstring/name) - assert f.__name__ == "f" - - -def test_set_paper_tags_no_op_when_mlflow_missing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", False) - # Should silently no-op without raising. - set_paper_tags(section="results", policy="bound_aware_276k_economic_champion") - - -def test_paper_run_yields_none_when_mlflow_missing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", False) - with paper_run("smoke") as run: - assert run is None - - -def test_register_parquet_dataset_no_op_when_mlflow_missing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", False) - # Even if MLflow is present at install time, _HAS_MLFLOW=False must short-circuit. - assert register_parquet_dataset(tmp_path / "absent.parquet") is None - - -def test_sha256_of_file_is_deterministic(tmp_path: Path) -> None: - path = tmp_path / "blob.bin" - path.write_bytes(b"crpto" * 1024) - a = mlflow_tracing._sha256_of_file(path) - b = mlflow_tracing._sha256_of_file(path) - assert a == b - assert len(a) == 64 # SHA256 hex - - -def test_set_paper_tags_uses_extra(monkeypatch: pytest.MonkeyPatch) -> None: - """When MLflow is present, ``extra`` keys must be prefixed with ``paper.``.""" - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", True) - captured: dict[str, Any] = {} - - class _StubMlflow: - @staticmethod - def set_tags(tags: dict[str, Any]) -> None: - captured.update(tags) - - @staticmethod - def log_param(key: str, value: Any) -> None: - captured[key] = value - - monkeypatch.setattr(mlflow_tracing, "mlflow", _StubMlflow) - set_paper_tags(section="results", policy="point", extra={"variant": "A"}) - assert captured["paper.run_tag"] == PAPER_RUN_TAG - assert captured["paper.section"] == "results" - assert captured["paper.policy"] == "point" - assert captured["paper.variant"] == "A" - - -def test_set_paper_tags_rejects_unknown_run_tag(monkeypatch: pytest.MonkeyPatch) -> None: - """Non-canonical run_tags must be opted in via allow_new_run_tag=True.""" - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", True) - monkeypatch.setattr(mlflow_tracing, "mlflow", object()) - - with pytest.raises(ValueError, match="canonical paper tag"): - set_paper_tags(run_tag="paper-thesis-revalidation-2027") - - -def test_set_paper_tags_accepts_new_tag_with_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: - """allow_new_run_tag=True must let revalidation cohorts proceed.""" - monkeypatch.setattr(mlflow_tracing, "_HAS_MLFLOW", True) - captured: dict[str, Any] = {} - - class _StubMlflow: - @staticmethod - def set_tags(tags: dict[str, Any]) -> None: - captured.update(tags) - - @staticmethod - def log_param(key: str, value: Any) -> None: - captured[key] = value - - monkeypatch.setattr(mlflow_tracing, "mlflow", _StubMlflow) - set_paper_tags(run_tag="paper-thesis-revalidation-2027", allow_new_run_tag=True) - assert captured["paper.run_tag"] == "paper-thesis-revalidation-2027" diff --git a/tests/test_utils/test_optuna_storage.py b/tests/test_utils/test_optuna_storage.py deleted file mode 100644 index 5ae55f0..0000000 --- a/tests/test_utils/test_optuna_storage.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Tests for ``src.utils.optuna_storage``. - -Exercise URL resolution and journal fallback without spinning up Optuna trials. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from src.utils import optuna_storage - - -def test_is_journal_url_recognises_prefixes() -> None: - assert optuna_storage._is_journal_url("journal:/tmp/x.log") - assert optuna_storage._is_journal_url("journal+file:/tmp/x.log") - assert optuna_storage._is_journal_url("journalfile:/tmp/x.log") - assert optuna_storage._is_journal_url("journalFile:/tmp/x.log") - assert not optuna_storage._is_journal_url("sqlite:///tmp/x.db") - assert not optuna_storage._is_journal_url("postgresql://u:p@h/db") - - -@pytest.mark.parametrize( - ("url", "expected_suffix"), - [ - ("journal:/tmp/x.log", "/tmp/x.log"), - ("journal:///tmp/x.log", "/tmp/x.log"), - ("journal:tmp/x.log", "tmp/x.log"), - ("journal+file:/foo/y.log", "/foo/y.log"), - ("journalfile:foo/y.log", "foo/y.log"), - ("journalFile:C:/Users/carlos/optuna.log", "C:/Users/carlos/optuna.log"), - ], -) -def test_journal_path_from_url(url: str, expected_suffix: str) -> None: - out = optuna_storage._journal_path_from_url(url) - assert out == expected_suffix - - -def test_make_storage_journal_default(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """No env var ⇒ JournalStorage under /.log.""" - monkeypatch.delenv("OPTUNA_STORAGE", raising=False) - storage = optuna_storage.make_storage("smoke_study", directory=tmp_path) - # We don't import JournalStorage at module level to avoid version coupling. - assert storage.__class__.__name__ in {"JournalStorage", "RDBStorage"} - - -def test_make_storage_env_journal_url_resolves( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - target = tmp_path / "explicit.log" - monkeypatch.setenv("OPTUNA_STORAGE", f"journal:{target}") - storage = optuna_storage.make_storage("ignored_when_env_present") - assert storage.__class__.__name__ in {"JournalStorage", "RDBStorage"} - - -def test_make_storage_explicit_sqlite(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("OPTUNA_STORAGE", raising=False) - db_dir = tmp_path / "optuna" - db_dir.mkdir() - monkeypatch.chdir(tmp_path) - storage = optuna_storage.make_storage("sqlite_smoke", backend="sqlite") - assert storage.__class__.__name__ == "RDBStorage" - - -def test_make_study_returns_study(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """End-to-end smoke: a TPE study can be created. Uses in-memory storage to - avoid Windows symlink-lock issues — the journal path itself is exercised in - other tests via :func:`make_storage`.""" - monkeypatch.delenv("OPTUNA_STORAGE", raising=False) - monkeypatch.setattr(optuna_storage, "DEFAULT_JOURNAL_DIR", tmp_path) - - # Patch make_storage to return an in-memory store for this smoke test. - import optuna - - monkeypatch.setattr( - optuna_storage, - "make_storage", - lambda *_, **__: optuna.storages.InMemoryStorage(), - ) - - study = optuna_storage.make_study(name="smoke", direction="maximize", pruner="none") - assert study.study_name == "smoke" - assert study.sampler is not None diff --git a/tests/test_utils/test_pipeline_runtime.py b/tests/test_utils/test_pipeline_runtime.py index a6b3e86..9f8d5ac 100644 --- a/tests/test_utils/test_pipeline_runtime.py +++ b/tests/test_utils/test_pipeline_runtime.py @@ -3,8 +3,11 @@ import json from pathlib import Path +import pytest + from src.utils.pipeline_runtime import ( atomic_write_json, + atomic_write_strict_json, load_runtime_status, runtime_checkpoint_dir, runtime_last_artifact_path, @@ -30,6 +33,16 @@ def test_atomic_write_json_replaces_existing_file_without_tmp_leak(tmp_path: Pat assert _tmp_files(target) == [] +def test_atomic_write_strict_json_rejects_nan_and_unsupported_values(tmp_path: Path) -> None: + target = tmp_path / "strict.json" + + with pytest.raises(ValueError, match="Out of range float"): + atomic_write_strict_json(target, {"value": float("nan")}) + with pytest.raises(TypeError, match="not JSON serializable"): + atomic_write_strict_json(target, {"value": tmp_path}) + assert not target.exists() + + def test_runtime_status_helpers_write_named_artifacts(tmp_path: Path) -> None: assert runtime_status_path("unit_stage") == Path("models/unit_stage_runtime_status.json") diff --git a/tests/test_utils/test_pipeline_state.py b/tests/test_utils/test_pipeline_state.py deleted file mode 100644 index 8e8ed8c..0000000 --- a/tests/test_utils/test_pipeline_state.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Smoke tests for ``src.utils.pipeline_state``. - -The module is read-only and tolerates missing files. These tests verify that -behaviour: no crashes on a partial environment, namespaced lookups work, and -``EXTRACTION_MANIFEST.json`` is merged when present. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from src.utils.pipeline_state import PipelineState, load_pipeline_state - - -def test_load_pipeline_state_against_repo_root() -> None: - state = load_pipeline_state() - assert isinstance(state, PipelineState) - # Repo manifest exists in the curated CRPTO export; if it's absent the test - # still passes as long as the loader did not crash. - if (state.repo_root / "EXTRACTION_MANIFEST.json").exists(): - assert state.manifest, "manifest should be loaded when the file exists" - - -def test_get_returns_default_for_missing_path() -> None: - state = load_pipeline_state() - assert state.get("does", "not", "exist", default="fallback") == "fallback" - assert state.get("paper", "evidence", "this_key_is_made_up", default=None) is None - - -def test_to_dict_is_json_serialisable() -> None: - state = load_pipeline_state() - snap = state.to_dict() - json.dumps(snap) # must not raise - assert {"state", "missing", "manifest", "repo_root"} <= snap.keys() - - -def test_missing_dir_yields_all_files_missing(tmp_path: Path) -> None: - """A non-existent models dir should report every namespace as missing without raising.""" - state = load_pipeline_state(repo_root=tmp_path, models_dir="nope") - assert state.state == {} - assert state.missing, "all configured namespaces should be marked missing" - - -def test_synthetic_namespace_round_trip(tmp_path: Path) -> None: - """When a single status JSON exists in tmp_path, it must surface in the namespaced state.""" - models = tmp_path / "models" - models.mkdir(parents=True) - payload = {"coverage": {"target": 0.9, "observed": 0.9123}} - (models / "conformal_policy_status.json").write_text(json.dumps(payload)) - - state = load_pipeline_state(repo_root=tmp_path) - assert state.get("conformal", "policy") == payload - assert "conformal/policy" not in state.missing - # The other namespaces remain reported as missing. - assert state.missing - - -@pytest.mark.parametrize( - "namespace", - [ - ("paper", "evidence"), - ("paper", "promotion"), - ("conformal", "policy"), - ("portfolio", "optimization"), - ], -) -def test_namespaces_are_addressable(namespace: tuple[str, str]) -> None: - state = load_pipeline_state() - # No assertion on truthiness — these may be missing in a partial extract, - # we just make sure addressing them doesn't blow up. - _ = state.get(*namespace, default=None) - - -# --------------------------------------------------------------------------- -# write_pipeline_state -# --------------------------------------------------------------------------- - - -def test_write_pipeline_state_round_trip(tmp_path: Path) -> None: - """Writing a known namespace and reading it back must return the payload verbatim.""" - from src.utils.pipeline_state import write_pipeline_state - - payload = {"coverage": {"target": 0.9, "observed": 0.913}} - out_path = write_pipeline_state( - ("conformal", "policy"), - payload, - repo_root=tmp_path, - allow_protected=True, - ) - assert out_path.name == "conformal_policy_status.json" - assert out_path.is_file() - state = load_pipeline_state(repo_root=tmp_path) - assert state.get("conformal", "policy") == payload - - -def test_write_pipeline_state_unknown_namespace_falls_back(tmp_path: Path) -> None: - from src.utils.pipeline_state import write_pipeline_state - - out = write_pipeline_state("unknown/topic", {"value": 1}, repo_root=tmp_path) - assert out.name == "unknown_topic_status.json" - - -def test_write_pipeline_state_merge_combines_keys(tmp_path: Path) -> None: - from src.utils.pipeline_state import write_pipeline_state - - write_pipeline_state( - "paper/evidence", {"status": "pass", "nested": {"a": 1}}, repo_root=tmp_path - ) - write_pipeline_state( - "paper/evidence", - {"checked_at": "2026-05-10", "nested": {"b": 2}}, - repo_root=tmp_path, - merge=True, - ) - state = load_pipeline_state(repo_root=tmp_path) - payload = state.get("paper", "evidence") - assert payload["status"] == "pass" - assert payload["checked_at"] == "2026-05-10" - assert payload["nested"] == {"a": 1, "b": 2} - - -def test_write_pipeline_state_string_namespace_accepted(tmp_path: Path) -> None: - from src.utils.pipeline_state import write_pipeline_state - - out = write_pipeline_state( - "paper/promotion", {"run_tag": "rt"}, repo_root=tmp_path, allow_protected=True - ) - assert out.name == "final_project_promotion.json" - - -def test_write_pipeline_state_refuses_protected_status_by_default(tmp_path: Path) -> None: - from src.utils.pipeline_state import write_pipeline_state - - with pytest.raises(PermissionError): - write_pipeline_state("paper/promotion", {"run_tag": "rt"}, repo_root=tmp_path) diff --git a/tests/test_utils/test_replay_manifest.py b/tests/test_utils/test_replay_manifest.py deleted file mode 100644 index c747a1d..0000000 --- a/tests/test_utils/test_replay_manifest.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -import hashlib -from pathlib import Path - -import pytest - -from src.utils import replay_manifest - - -def test_replay_manifest_round_trips_sections_without_aliasing(tmp_path: Path) -> None: - manifest_path = tmp_path / "configs" / "baselines" / "unit_manifest.json" - payload = {"pd": {"model_path": "models/pd.cbm"}, "conformal": {"alpha": 0.1}} - - saved = replay_manifest.save_replay_manifest(payload, manifest_path) - loaded = replay_manifest.load_replay_manifest(saved) - section = replay_manifest.manifest_section(loaded, "pd") - - assert saved == manifest_path.resolve() - assert loaded == payload - assert section == {"model_path": "models/pd.cbm"} - section["model_path"] = "changed" - assert loaded["pd"]["model_path"] == "models/pd.cbm" - assert replay_manifest.manifest_section(loaded, "missing") == {} - assert replay_manifest.load_replay_manifest(tmp_path / "missing.json") == {} - - -def test_artifact_descriptor_reports_repo_relative_hash( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(replay_manifest, "ROOT", tmp_path) - artifact = tmp_path / "models" / "artifact.txt" - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"frozen\n") - - descriptor = replay_manifest.artifact_descriptor("models/artifact.txt") - - assert descriptor["path"] == "models/artifact.txt" - assert descriptor["exists"] is True - assert descriptor["sha256"] == hashlib.sha256(b"frozen\n").hexdigest() - assert replay_manifest.artifact_descriptor("models/missing.txt")["sha256"] is None diff --git a/tests/test_utils/test_script_helpers.py b/tests/test_utils/test_script_helpers.py deleted file mode 100644 index 201d56d..0000000 --- a/tests/test_utils/test_script_helpers.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Tests for the consolidated script helpers in ``src/utils/script_helpers.py``.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np -import pandas as pd -import pytest - -from src.utils.script_helpers import ( - artifact_path, - first_existing, - load_json, - load_yaml, - parse_percent_series, - policy_matches, - resolve_interval_columns, - resolve_repo_artifact_path, - try_load_json, - write_json, - write_table, -) - - -def test_load_json_roundtrip(tmp_path: Path) -> None: - path = tmp_path / "payload.json" - write_json(path, {"b": 2, "a": 1}) - assert load_json(path) == {"a": 1, "b": 2} - - -def test_write_json_is_lf_only_and_key_sorted(tmp_path: Path) -> None: - path = tmp_path / "nested" / "payload.json" - write_json(path, {"z": 1, "a": {"y": 2, "b": 3}}) - raw = path.read_bytes() - assert b"\r" not in raw - assert raw.endswith(b"\n") - assert ( - raw.decode("utf-8") - == json.dumps({"z": 1, "a": {"y": 2, "b": 3}}, indent=2, sort_keys=True) + "\n" - ) - - -def test_try_load_json_missing_returns_empty(tmp_path: Path) -> None: - assert try_load_json(tmp_path / "missing.json") == {} - - -def test_load_yaml(tmp_path: Path) -> None: - path = tmp_path / "config.yaml" - path.write_text("alpha: 0.01\nmodes: [a, b]\n", encoding="utf-8") - assert load_yaml(path) == {"alpha": 0.01, "modes": ["a", "b"]} - - -def test_write_table_emits_lf_csv_and_tex(tmp_path: Path) -> None: - frame = pd.DataFrame({"metric": ["auc", "brier"], "value": [0.7127, 0.1546]}) - paths = write_table("t_demo", frame, table_dir=tmp_path, root=tmp_path) - assert [p.name for p in paths] == ["t_demo.csv", "t_demo.tex"] - for path in paths: - assert b"\r" not in path.read_bytes() - - -def test_write_table_is_idempotent(tmp_path: Path) -> None: - frame = pd.DataFrame({"a": [1, 2]}) - first = write_table("t_idem", frame, table_dir=tmp_path, root=tmp_path) - mtimes = [p.stat().st_mtime_ns for p in first] - second = write_table("t_idem", frame, table_dir=tmp_path, root=tmp_path) - assert [p.stat().st_mtime_ns for p in second] == mtimes - - -def test_artifact_path_without_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("GPU_REPLAY_ARTIFACT_ROOT", raising=False) - assert artifact_path("data/processed/x.parquet") == Path("data/processed/x.parquet") - - -def test_artifact_path_with_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("GPU_REPLAY_ARTIFACT_ROOT", str(tmp_path)) - assert artifact_path("models/x.json") == tmp_path / "models/x.json" - - -def test_resolve_repo_artifact_path_maps_wsl_manifest_path(tmp_path: Path) -> None: - root = tmp_path / "Paper_CRPTO" - expected = root / "data" / "processed" / "intervals.parquet" - - resolved = resolve_repo_artifact_path( - "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/intervals.parquet", - root=root, - ) - - assert resolved == expected - - -def test_resolve_repo_artifact_path_anchors_relative_path(tmp_path: Path) -> None: - root = tmp_path / "repo" - assert resolve_repo_artifact_path("models/a.json", root=root) == root / "models" / "a.json" - - -def test_first_existing_prefers_existing(tmp_path: Path) -> None: - existing = tmp_path / "exact.parquet" - existing.touch() - missing = tmp_path / "fallback.parquet" - assert first_existing(existing, missing) == existing - assert first_existing(missing, existing) == existing - assert first_existing(missing) == missing - with pytest.raises(ValueError, match="at least one"): - first_existing() - - -def test_parse_percent_series_numeric_and_strings() -> None: - numeric = parse_percent_series(pd.Series([12.0, np.nan, 25.0])) - np.testing.assert_allclose(numeric, [0.12, 0.12, 0.25]) - strings = parse_percent_series(pd.Series([" 12.5% ", "bad", "7%"])) - np.testing.assert_allclose(strings, [0.125, 0.12, 0.07]) - - -def test_resolve_interval_columns_variants() -> None: - modern = pd.DataFrame(columns=["y_pred", "pd_low_90", "pd_high_90"]) - assert resolve_interval_columns(modern) == ("y_pred", "pd_low_90", "pd_high_90") - legacy = pd.DataFrame(columns=["pd_point", "pd_low", "pd_high"]) - assert resolve_interval_columns(legacy) == ("pd_point", "pd_low", "pd_high") - - -def test_policy_matches_full_fields() -> None: - policy = { - "risk_tolerance": 0.175, - "policy_mode": "blended_uncertainty", - "gamma": 0.45, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.1, - "min_budget_utilization": 0.5, - "pd_cap_slack_penalty": 0.0, - } - row = pd.Series(dict(policy)) - assert policy_matches(row, policy) - # within tolerance - near = pd.Series({**policy, "gamma": 0.45 + 5e-10}) - assert policy_matches(near, policy) - # numeric mismatch - off = pd.Series({**policy, "risk_tolerance": 0.17}) - assert not policy_matches(off, policy) - # string mismatch - mode = pd.Series({**policy, "policy_mode": "hard_worst_case"}) - assert not policy_matches(mode, policy) - # missing field on the row side - partial = pd.Series({k: v for k, v in policy.items() if k != "gamma"}) - assert not policy_matches(partial, policy) - # non-coercible numeric - bad = pd.Series({**policy, "gamma": None}) - assert not policy_matches(bad, policy) - - -def test_policy_matches_narrow_fields() -> None: - policy = {"risk_tolerance": 0.175, "policy_mode": "blended_uncertainty", "gamma": 0.45} - row = {"risk_tolerance": 0.175, "policy_mode": "blended_uncertainty", "gamma": 0.45} - fields = ("risk_tolerance", "policy_mode", "gamma") - assert policy_matches(row, policy, fields=fields) - assert not policy_matches(row, {**policy, "gamma": 0.55}, fields=fields) diff --git a/uv.lock b/uv.lock index 50dbe3a..2b6b661 100644 --- a/uv.lock +++ b/uv.lock @@ -2,35 +2,21 @@ version = 1 revision = 3 requires-python = ">=3.11, <3.13" resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version < '3.12'", + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] name = "absl-py" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4", size = 116543, upload-time = "2026-01-28T10:17:05.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, -] - -[[package]] -name = "agate" -version = "1.9.1" +version = "2.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "babel" }, - { name = "isodate" }, - { name = "leather" }, - { name = "parsedatetime" }, - { name = "python-slugify" }, - { name = "pytimeparse" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/29/77/6f5df1c68bf056f5fdefc60ccc616303c6211e71cd6033c830c12735f605/agate-1.9.1.tar.gz", hash = "sha256:bc60880c2ee59636a2a80cd8603d63f995be64526abf3cbba12f00767bcd5b3d", size = 202303, upload-time = "2023-12-21T20:05:24.316Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/4f/d79676ab82f2e42fc3611618139f13a9c4c31d0cff4b486982047679a802/absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f", size = 118119, upload-time = "2026-07-03T10:57:48.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/53/89b197cb472a3175d73384761a3413fd58e6b65a794c1102d148b8de87bd/agate-1.9.1-py2.py3-none-any.whl", hash = "sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50", size = 95085, upload-time = "2023-12-21T20:05:21.954Z" }, + { url = "https://files.pythonhosted.org/packages/58/0a/a10b45aab35b175aded078a462dc8d0c698f5b13946e7cb0869097b78bb6/absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba", size = 137410, upload-time = "2026-07-03T10:57:46.735Z" }, ] [[package]] @@ -53,11 +39,11 @@ wheels = [ [[package]] name = "aiohappyeyeballs" -version = "2.6.1" +version = "2.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, ] [[package]] @@ -148,20 +134,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] -[[package]] -name = "alembic" -version = "1.18.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mako" }, - { name = "sqlalchemy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, -] - [[package]] name = "amqp" version = "5.3.1" @@ -174,15 +146,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, ] -[[package]] -name = "annotated-doc" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, -] - [[package]] name = "annotated-types" version = "0.7.0" @@ -198,19 +161,6 @@ version = "4.9.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } -[[package]] -name = "anyio" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, -] - [[package]] name = "appdirs" version = "1.4.4" @@ -221,89 +171,41 @@ wheels = [ ] [[package]] -name = "appnope" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, -] - -[[package]] -name = "argon2-cffi" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi-bindings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, -] - -[[package]] -name = "argon2-cffi-bindings" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, - { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, - { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, -] - -[[package]] -name = "arrow" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, -] - -[[package]] -name = "asttokens" -version = "3.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, -] - -[[package]] -name = "async-lru" -version = "2.3.0" +name = "ast-serialize" +version = "0.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/1f/989ecfef8e64109a489fff357450cb73fa73a865a92bd8c272170a6922c2/async_lru-2.3.0.tar.gz", hash = "sha256:89bdb258a0140d7313cf8f4031d816a042202faa61d0ab310a0a538baa1c24b6", size = 16332, upload-time = "2026-03-19T01:04:32.413Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/ad/0d70a3a2d6e01968d985415259e8ec7ad3f777903f9b1c1f3c8c44642c60/ast_serialize-0.6.0.tar.gz", hash = "sha256:aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe", size = 61489, upload-time = "2026-06-30T20:02:55.555Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/e2/c2e3abf398f80732e58b03be77bde9022550d221dd8781bf586bd4d97cc1/async_lru-2.3.0-py3-none-any.whl", hash = "sha256:eea27b01841909316f2cc739807acea1c623df2be8c5cfad7583286397bb8315", size = 8403, upload-time = "2026-03-19T01:04:30.883Z" }, + { url = "https://files.pythonhosted.org/packages/52/19/ac8348ae8711c9b5ae834634f635780cab62a0f5e6f988882e048b89c2ae/ast_serialize-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596", size = 1185367, upload-time = "2026-06-30T20:02:30.427Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f6/ec7ec652c51db77c2f61d8573338e13e4704303265ccc658cb4031d9f354/ast_serialize-0.6.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad", size = 1178657, upload-time = "2026-06-30T20:02:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/6f/02/613a7534a41d0122f37d1e0c64aa8ac78bfb831f8c92f6db057a311abb3c/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a", size = 1238620, upload-time = "2026-06-30T20:02:33.664Z" }, + { url = "https://files.pythonhosted.org/packages/4d/21/087957bba486242afc52f49b2d9e21c9dad00289356cf9efe67084015a9d/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081", size = 1236075, upload-time = "2026-06-30T20:02:34.936Z" }, + { url = "https://files.pythonhosted.org/packages/82/04/78128bbb170071c2c72a210a181f1c00e11cc1cec60a8beef747b07f9201/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77", size = 1441348, upload-time = "2026-06-30T20:02:36.245Z" }, + { url = "https://files.pythonhosted.org/packages/64/64/62fb99d6faf199b4c3e5b08a07136e9a0d7664bb249c6de3670e5b63e9b6/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790", size = 1258580, upload-time = "2026-06-30T20:02:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/b4d6c38e0ccd5e85dc54cecdf933a152c60b28fe5d993a6d8a72fa6d5896/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7", size = 1261693, upload-time = "2026-06-30T20:02:39.123Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/3676ca2191f39bafb75f93f99b2f429ec464586158fece2165f3572805dc/ast_serialize-0.6.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b", size = 1252517, upload-time = "2026-06-30T20:02:40.511Z" }, + { url = "https://files.pythonhosted.org/packages/f3/58/494ef8c4b4acb2f4a265ac934caf45f792a08fe27d6b853de35ad991941a/ast_serialize-0.6.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b", size = 1304843, upload-time = "2026-06-30T20:02:41.961Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f2/13736d920ab3d49bbee80ef1a277dd7b7aaf3b3545efd9d2a8114fe05525/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32", size = 1413698, upload-time = "2026-06-30T20:02:44.179Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5a/e046f3899e2acba4677d7427b76431443a1aa1a0e583dfb05b55b69d55cf/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b", size = 1512209, upload-time = "2026-06-30T20:02:45.584Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c7/e42aaca7bb2d22a7c06d5a8c7930086c5a334e93d716e6fa5e6647a4515f/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a", size = 1508464, upload-time = "2026-06-30T20:02:46.942Z" }, + { url = "https://files.pythonhosted.org/packages/95/93/5524a3dc6c3f593de3228ed9cbef73afa047625b7000ec21b7f58e6eb4d4/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e", size = 1457164, upload-time = "2026-06-30T20:02:48.294Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c0/36a6ffb4d653cf621427b4c4928671f53ad800c453474de2b82564a44ad9/ast_serialize-0.6.0-cp39-abi3-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b", size = 863014, upload-time = "2026-06-30T20:02:49.742Z" }, + { url = "https://files.pythonhosted.org/packages/09/c7/7d5ad8b49e1278e1c2a1e0274bd7850560b3f09313aa00c13bc8d5544792/ast_serialize-0.6.0-cp39-abi3-win32.whl", hash = "sha256:82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1", size = 1063165, upload-time = "2026-06-30T20:02:50.98Z" }, + { url = "https://files.pythonhosted.org/packages/47/ae/6710c14ecb276031cf10249f6adf5a59e2d3fdb3b5183bd59f70524067ee/ast_serialize-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e", size = 1101444, upload-time = "2026-06-30T20:02:52.554Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, ] [[package]] name = "asyncssh" -version = "2.23.0" +version = "2.24.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/fd/c34fe7e30838b4b9cc91903da26a62c6d33b673c731b3d951fcd70ab1889/asyncssh-2.23.0.tar.gz", hash = "sha256:8c54760953c1f2cf282591bcba5c8c70efc48d645bbf26bd2307a9c66a0ed1a7", size = 542154, upload-time = "2026-05-09T03:15:01.856Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/7f/2d79247bacc562104f312d27efe541673aa177feac89de291bd61bca52be/asyncssh-2.24.0.tar.gz", hash = "sha256:4064c590e59ce2e8d82a2f66d35f3120d765828b4df5e3dbfb07b4a8c24686c9", size = 550148, upload-time = "2026-06-27T20:34:44.755Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/b5/b1a3979f4840d1271ca8e0978dbccfb18ad2d33b4ece85cf77122fb46e5f/asyncssh-2.23.0-py3-none-any.whl", hash = "sha256:14108bfdaae17457f0c1841e883ad934271bbfdd46458aa4c4d0973451940ad0", size = 375687, upload-time = "2026-05-09T03:15:00.221Z" }, + { url = "https://files.pythonhosted.org/packages/3e/29/908ce0ca5e8cae76662e354a0f08df552d6d221844748b9e5ca06051cc44/asyncssh-2.24.0-py3-none-any.whl", hash = "sha256:9abd46300adcb6d4b73269b34c53cd0d17a138b9a22b5b38008ce7d5808734b7", size = 381237, upload-time = "2026-06-27T20:34:43.198Z" }, ] [[package]] @@ -324,50 +226,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] -[[package]] -name = "babel" -version = "2.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, -] - -[[package]] -name = "backoff" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, -] - -[[package]] -name = "beautifulsoup4" -version = "4.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, -] - -[[package]] -name = "betacal" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "scikit-learn" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/76/edaa7f9688a8a1458063ec2495ef552d99a9186b01bbbd752e61391d8f8d/betacal-1.1.0.tar.gz", hash = "sha256:fc425a0bf78a3b368d60d8b1e465ee4bc953c52440e04836eb5098a1b32d98cc", size = 3663, upload-time = "2021-04-01T21:16:54.94Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/41/4c21097cbf63a8081ab637f1ce064238ad3de9dd000ecd783e2fc81e90e3/betacal-1.1.0-py3-none-any.whl", hash = "sha256:b7063c65fb7eb1b8d5bba4498f1f91aceb727e1ed0758e3bfc4b4b455715d023", size = 5265, upload-time = "2021-04-01T21:16:53.659Z" }, -] - [[package]] name = "billiard" version = "4.2.4" @@ -377,46 +235,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, ] -[[package]] -name = "bleach" -version = "6.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/9d/40b6267367182187139a4000b82a3b287d84d745bccd808e75d916920e9d/bleach-6.4.0-py3-none-any.whl", hash = "sha256:4b6b6a54fff2e69a3dde9d21cc6301220bee3c3cb792187d11403fd795031081", size = 165109, upload-time = "2026-06-05T13:01:12.504Z" }, -] - -[package.optional-dependencies] -css = [ - { name = "tinycss2" }, -] - -[[package]] -name = "blinker" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, -] - -[[package]] -name = "boto3" -version = "1.43.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/65/47670987f2f9e181397872c7ee6415b7b95156d711b7eab6c55f66e575bc/boto3-1.43.0.tar.gz", hash = "sha256:80d44a943ef90aba7958ab31d30c155c198acc8a9581b5846b3878b2c8951086", size = 113143, upload-time = "2026-04-29T22:07:49.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/a0/3e6a0b1c1ea6bec76f71473727ef27abf3cd40e9709b3ebcbfbcfaae6f79/boto3-1.43.0-py3-none-any.whl", hash = "sha256:8ebe03754a4b73a5cb6ec2f14cca03ac33bd4760d0adea53da4724845130258b", size = 140497, upload-time = "2026-04-29T22:07:46.216Z" }, -] - [[package]] name = "botocore" version = "1.43.0" @@ -431,15 +249,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/4b/afc1fef8a43bafb139f57f73bbd70df82807af5934321e8112ae50668827/botocore-1.43.0-py3-none-any.whl", hash = "sha256:cc5b15eaec3c6eac05d8012cb5ef17ebe891beb88a16ca13c374bfaece1241e6", size = 14970102, upload-time = "2026-04-29T22:07:27Z" }, ] -[[package]] -name = "cachetools" -version = "7.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ff/e2/85f227594656000ff4d8adadae91a21f536d4a84c6c716a86bd6685874be/cachetools-7.1.1.tar.gz", hash = "sha256:27bdf856d68fd3c71c26c01b5edc312124ed427524d1ddb31aa2b7746fe20d4b", size = 40202, upload-time = "2026-05-03T20:00:29.391Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/0f/f897abe4ea0a8c408ae65c8c83bffab4936ad65d6032d4fb4cd35bbdc3ee/cachetools-7.1.1-py3-none-any.whl", hash = "sha256:0335cd7a0952d2b22327441fb0628139e234c565559eeb91a8a4ac7551c5353d", size = 16775, upload-time = "2026-05-03T20:00:27.857Z" }, -] - [[package]] name = "catboost" version = "1.2.10" @@ -447,10 +256,12 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "graphviz" }, { name = "matplotlib" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pandas" }, { name = "plotly" }, - { name = "scipy" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/0e/09e8fa0858570fda88090bc3f441b69c18ea3d6f4a02fd41aa5426c157bf/catboost-1.2.10.tar.gz", hash = "sha256:26ae6d423acaf0e9d8160f2477a990431057ed04522d993c2f42dac62743b4f7", size = 39925863, upload-time = "2026-02-18T16:13:29.092Z" } @@ -487,47 +298,47 @@ wheels = [ [[package]] name = "certifi" -version = "2026.5.20" +version = "2026.6.17" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, ] [[package]] name = "cffi" -version = "2.0.0" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, - { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, - { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, - { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, - { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, - { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, - { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/67/85c89a59ba36a671e79638f44d466749f08179266a57e4f2ffdf92174072/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc", size = 183845, upload-time = "2026-07-06T21:32:26.32Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dd/e3b0baa2d3d6a857ac72b7efbf18e32e487c9cdafcc13049ad765495b15e/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7", size = 184186, upload-time = "2026-07-06T21:32:28.025Z" }, + { url = "https://files.pythonhosted.org/packages/65/68/9f3ef890cf3c6ab97bd531c5677f67613d302165d16f8142b2811782a614/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93", size = 211892, upload-time = "2026-07-06T21:32:29.565Z" }, + { url = "https://files.pythonhosted.org/packages/22/d7/1a74539db16d8bfd839ff1515948948efbb162e574650fd3d846896eea95/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2", size = 218793, upload-time = "2026-07-06T21:32:30.951Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d1/9a5b7169499e8e8d8e636de70b97ac7c9447104d2ff1a2cd94790cea5162/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c", size = 205737, upload-time = "2026-07-06T21:32:32.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b0/e131a9c41f10607926278453d9596163594fe1c4ebc46efe3b5e5b34eb84/cffi-2.1.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f", size = 204909, upload-time = "2026-07-06T21:32:33.655Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d2/4398416cd699b35167947c6e22aca52c47e69ad5695073c9f1f2c52e04aa/cffi-2.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565", size = 217883, upload-time = "2026-07-06T21:32:35.173Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a5/d4fe77b589e5e82d43ebc809bf2e6474afe8e48e32ea050b9357645b6471/cffi-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c", size = 221251, upload-time = "2026-07-06T21:32:36.527Z" }, + { url = "https://files.pythonhosted.org/packages/22/f0/a2fc43084c0433caf7f461bccc013e28f848d04ee1c5ed7fce71423cf4d9/cffi-2.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02", size = 214250, upload-time = "2026-07-06T21:32:37.852Z" }, + { url = "https://files.pythonhosted.org/packages/04/8c/b925975448cf20634a9fbd5efceb807219db452653648d2897c0989cab2d/cffi-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e", size = 219441, upload-time = "2026-07-06T21:32:39.146Z" }, + { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, + { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, + { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, ] [[package]] @@ -541,43 +352,37 @@ wheels = [ [[package]] name = "charset-normalizer" -version = "3.4.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, - { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, - { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, - { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, - { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, - { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, - { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, - { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, - { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, - { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, - { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, - { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, - { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, - { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, ] [[package]] @@ -586,8 +391,10 @@ version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, - { name = "numpy" }, - { name = "scipy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/81/e2/47f692161779dbd98876015de934943effb667a014e6f79a6d746b3e4c2a/clarabel-0.11.1.tar.gz", hash = "sha256:e7c41c47f0e59aeab99aefff9e58af4a8753ee5269bbeecbd5526fc6f41b9598", size = 253949, upload-time = "2025-06-11T16:49:05.864Z" } wheels = [ @@ -600,14 +407,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -647,15 +454,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -665,27 +463,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "colorlog" -version = "6.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/61/f083b5ac52e505dfc1c624eafbf8c7589a0d7f32daa398d2e7590efa5fda/colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321", size = 17162, upload-time = "2025-10-16T16:14:11.978Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/c1/e419ef3723a074172b68aaa89c9f3de486ed4c2399e2dbd8113a4fdcaf9e/colorlog-6.10.1-py3-none-any.whl", hash = "sha256:2d7e8348291948af66122cff006c9f8da6255d224e7cf8e37d8de2df3bad8c9c", size = 11743, upload-time = "2025-10-16T16:14:10.512Z" }, -] - -[[package]] -name = "comm" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, -] - [[package]] name = "configobj" version = "5.0.9" @@ -695,28 +472,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/c4/0679472c60052c27efa612b4cd3ddd2a23e885dcdc73461781d2c802d39e/configobj-5.0.9-py2.py3-none-any.whl", hash = "sha256:1ba10c5b6ee16229c79a05047aeda2b55eb4e80d7c7d8ecf17ec1ca600c79882", size = 35615, upload-time = "2024-11-26T14:03:32.972Z" }, ] -[[package]] -name = "configspace" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, - { name = "numpy" }, - { name = "pyparsing" }, - { name = "scipy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/51/7a63132fdc3a71eea66d9f81b451a61a38631af0a73bdcda1ec784cea32e/configspace-1.2.2.tar.gz", hash = "sha256:b5cc981c145ef9632104d230acf963c2f68fe5a375983946e941c2eec5dd08e4", size = 134158, upload-time = "2025-12-19T12:52:39.628Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/67/01bf8795064d2132d7244341908f75bb2ea87dc114364cd8c5b20c606a8f/configspace-1.2.2-py3-none-any.whl", hash = "sha256:9dd5090fab08fbfc0fa8a66affb53e991e9df65ae20959fcee8e544bb3c0a09d", size = 117584, upload-time = "2025-12-19T12:52:37.545Z" }, -] - [[package]] name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -751,41 +513,41 @@ wheels = [ [[package]] name = "coverage" -version = "7.14.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, - { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, - { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, - { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, - { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, - { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, - { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, - { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, - { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, - { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, - { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, - { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, - { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, +version = "7.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d0/55fe630f4cf94e3fcba868240fad8c8cdd1f764e2a932f8926347e6ec4cd/coverage-7.15.2.tar.gz", hash = "sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d", size = 927741, upload-time = "2026-07-15T18:56:19.558Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/3a/54536704f507d4573bf9161c4d0dd3dd59b6d85e48c664e901b6844d8e33/coverage-7.15.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f1ec6f304b156669cfde653b4e9a953f5de87e247ea02ac599bce0ab2744036", size = 221414, upload-time = "2026-07-15T18:53:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/b6/d9/8ba925d29743e3577b21e4d8c11a702b76bc93c41e7fdfd1177af63d4b8d/coverage-7.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d3361879d736f469f45723c11ea1a5bbdaf1f6928f0e632c940378b5aa9b660", size = 221913, upload-time = "2026-07-15T18:53:53.682Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/a855f3aa0187f2b431ade4e4791b77b56282cfb5d201c83ec26a31b5b36a/coverage-7.15.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c6a98d698f9e2c8008d0370ec7fc452ebfcc530002ae2d0061170d768b992589", size = 252332, upload-time = "2026-07-15T18:53:55.467Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d3/13ac97b4370640ba3452fc8559b06cc2f479ce3ba4a0b632a73e44c38a7d/coverage-7.15.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d50dd325e18ec25bfcc10cd7f99b04df1ab9ec76b0918c260e60817ad0643dee", size = 254243, upload-time = "2026-07-15T18:53:57.055Z" }, + { url = "https://files.pythonhosted.org/packages/88/83/5eca144942d8d0659d3f55176517f4a59cdc65eefd17146a0770935a3ebd/coverage-7.15.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67d7602480a47bdf5b675635403625553ebaa70d5a62a657c035149fd401cea0", size = 256352, upload-time = "2026-07-15T18:53:58.83Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ba/d3db2e01a50fc88cdb4c0f19542bcf6f61489e34dc9aa3538413e2459a38/coverage-7.15.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cee0f89f4767a6057c8fbf168f8135f18be651300496086bd873e3189fed0487", size = 258313, upload-time = "2026-07-15T18:54:00.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/b3/aba83416e9177df28e5186d856c19158c59fc0e7e814aaa61a4a2354ad1b/coverage-7.15.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a29ec5305a7335aacee2d799e3422e91e1c8a12474986e2b3b07e315c91be82f", size = 252449, upload-time = "2026-07-15T18:54:02.456Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a5/4b00ecac0194431ab451b0f6710f8e2517d04cef60f821b14dec4637d575/coverage-7.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:48ccc6395958eda89093ecdc35644c86f23a8b23a7f4d44958812b721aad67c1", size = 254043, upload-time = "2026-07-15T18:54:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/75/b6/cfa209b4313ee7f1b34da47efcd789ea51c024ad35af390e00f5a3c10a2e/coverage-7.15.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:81f382c5a94b434ec1f6da607edb904c76d7212e618cd4d1bc9f97bed4120ef5", size = 252107, upload-time = "2026-07-15T18:54:06.745Z" }, + { url = "https://files.pythonhosted.org/packages/36/67/e8cac5a6954038c98d7fe7eb9802afe7ab3ecb637bb7cc00e69b4148b56d/coverage-7.15.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bbc808daf4f5cd567af8075ecc72d21c6dfef9a254709a621a84c217c935ebc0", size = 255873, upload-time = "2026-07-15T18:54:08.48Z" }, + { url = "https://files.pythonhosted.org/packages/2c/92/395cca9f330a86c3fe3471d73e2c102116c4c58fdc619dbbc125c6e93a54/coverage-7.15.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a4c46b247b5d4b78f613bd89fea926d32b25c6cc61a50bd1e99ba310348f3dad", size = 251826, upload-time = "2026-07-15T18:54:10.083Z" }, + { url = "https://files.pythonhosted.org/packages/51/60/3e91b20295439652424f426b7086ec5bf4fbe3f604c73eda22b986c4fd6b/coverage-7.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:094dd37f3ef7b2da8b068b583d1f4c40f91c65197e16c52a71962d5d537fc5db", size = 252735, upload-time = "2026-07-15T18:54:11.878Z" }, + { url = "https://files.pythonhosted.org/packages/a5/eb/8c07839005e5e3c6b3877d3a6e2a80ce766589f31dd2b6882b78d59a7b8c/coverage-7.15.2-cp311-cp311-win32.whl", hash = "sha256:a63b9e190711134d581c4d703df5df09851b1acf99792c7aacbbe9f41f0283c9", size = 223500, upload-time = "2026-07-15T18:54:13.525Z" }, + { url = "https://files.pythonhosted.org/packages/2e/98/59d83c257cd59f0fbaf9d9ddb26b744a576760dfd1ae16e516408894a02b/coverage-7.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:8bb9f4b4279187560796a4cdaca3b0a93dd97e48ee667df005f4ed9a97403688", size = 223973, upload-time = "2026-07-15T18:54:15.163Z" }, + { url = "https://files.pythonhosted.org/packages/ea/09/2d285c8bef5c4f695d120c1c96dc11715638aa8e134069f210bb6a62a9fe/coverage-7.15.2-cp311-cp311-win_arm64.whl", hash = "sha256:8c726b232659cbd2ae57ade46509eb068c9bd7a06df9fcbff6fe484870006934", size = 223519, upload-time = "2026-07-15T18:54:16.803Z" }, + { url = "https://files.pythonhosted.org/packages/6a/50/eb5bf42e531611a9f8d272556b1ed4de503f84a91413584094487cf69f8f/coverage-7.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9", size = 221587, upload-time = "2026-07-15T18:54:18.439Z" }, + { url = "https://files.pythonhosted.org/packages/06/d1/da99af464c335d4e023a6efcd7ec30f63b88a43c93745154ab74ffb31cea/coverage-7.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73", size = 221943, upload-time = "2026-07-15T18:54:20.062Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8a/13c42723d61ca447eafa18732e8141dd6a63f2732e1c7e1502c182dd88d7/coverage-7.15.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d", size = 253450, upload-time = "2026-07-15T18:54:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/29/99021303f98fbdcb63504b4d07bea4cc025b9b2dd907c4f07c85d50a0dab/coverage-7.15.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b", size = 256187, upload-time = "2026-07-15T18:54:23.4Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a8/fd503715ed6ca9c5d742923aa5209257340b367a867b2ced0c7d4ba8a0b9/coverage-7.15.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296", size = 257301, upload-time = "2026-07-15T18:54:25.183Z" }, + { url = "https://files.pythonhosted.org/packages/da/40/3f4b8fb409810036ebc2857d36adc0498c6e957b5df0290c5036b2e143f1/coverage-7.15.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6", size = 259562, upload-time = "2026-07-15T18:54:27.204Z" }, + { url = "https://files.pythonhosted.org/packages/0b/8a/9bdffbef47db77cce3d6b02a28f7e919b19f0106c4b080c2c2246040f885/coverage-7.15.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098", size = 253841, upload-time = "2026-07-15T18:54:29.134Z" }, + { url = "https://files.pythonhosted.org/packages/1b/1e/9031efde019d31a06646261fce6dfc5c3c74e951e27a71e5c9a424563178/coverage-7.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a", size = 255221, upload-time = "2026-07-15T18:54:31.142Z" }, + { url = "https://files.pythonhosted.org/packages/56/db/787acde872389fc84a9ef9d8cd1ccc658e391ab4cb5b28092a714426a394/coverage-7.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b", size = 253366, upload-time = "2026-07-15T18:54:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/2f/9b/6f57bc4b93c842eef1695f8cdaf2318e35e7ba54f5ba80d84be213ab7858/coverage-7.15.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2", size = 257434, upload-time = "2026-07-15T18:54:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/88/26/b3186a21b2acc83e451118978905c81c7072c3333707804db09a78c096a2/coverage-7.15.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440", size = 252935, upload-time = "2026-07-15T18:54:36.548Z" }, + { url = "https://files.pythonhosted.org/packages/20/c2/c9f3376b2e717ea69ed7a6e9a5fcab968fb0b290db6cf4bd9a1fc7541b75/coverage-7.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e", size = 254807, upload-time = "2026-07-15T18:54:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e1/dfc15401f4a8aaeb486e1ba3e9e3c40522a6e38bd0ecf0b3f29cb8082957/coverage-7.15.2-cp312-cp312-win32.whl", hash = "sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd", size = 223641, upload-time = "2026-07-15T18:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/91/40/81b6d809d320cd366ec5bdf8176575e897dcb8efe7fb4b489ef9e93e4d13/coverage-7.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40", size = 224172, upload-time = "2026-07-15T18:54:41.882Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/9f14ec438149f7de557f45518f09b4a7917b795cc37083aa7db482693f8c/coverage-7.15.2-cp312-cp312-win_arm64.whl", hash = "sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3", size = 223556, upload-time = "2026-07-15T18:54:43.674Z" }, + { url = "https://files.pythonhosted.org/packages/ec/82/32e3bd191d498e64f6f911ad55d14006a0861e54869d2d32452326399e65/coverage-7.15.2-py3-none-any.whl", hash = "sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c", size = 213375, upload-time = "2026-07-15T18:56:17.305Z" }, ] [package.optional-dependencies] @@ -798,250 +560,134 @@ name = "crpto" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "betacal" }, { name = "catboost" }, - { name = "dagshub" }, - { name = "dbt-duckdb" }, { name = "duckdb" }, { name = "dvc", extra = ["s3"] }, - { name = "fairlearn" }, { name = "highspy" }, - { name = "ipykernel" }, - { name = "joblib" }, { name = "loguru" }, - { name = "mapie" }, { name = "matplotlib" }, - { name = "mlflow" }, - { name = "nbclient" }, - { name = "nbformat" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "optbinning" }, - { name = "optuna" }, { name = "ortools" }, { name = "pandas" }, - { name = "pandera" }, - { name = "plotly" }, { name = "pyarrow" }, { name = "pyomo" }, - { name = "pytest" }, - { name = "python-dotenv" }, { name = "pyyaml" }, { name = "scikit-learn" }, - { name = "scipy" }, - { name = "seaborn" }, - { name = "skops" }, - { name = "tqdm" }, - { name = "venn-abers" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] [package.optional-dependencies] dev = [ { name = "hypothesis" }, - { name = "jupyter" }, - { name = "jupytext" }, { name = "mypy" }, - { name = "nbstripout" }, - { name = "optuna-integration" }, { name = "pre-commit" }, + { name = "pypdf" }, + { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, ] -docs = [ - { name = "jupyterlab" }, -] -search = [ - { name = "optuna-integration" }, -] -spo = [ - { name = "pyepo" }, - { name = "torch" }, -] [package.metadata] requires-dist = [ - { name = "betacal", specifier = ">=1.1" }, { name = "catboost", specifier = ">=1.2,<1.3" }, - { name = "dagshub", specifier = ">=0.7" }, - { name = "dbt-duckdb", specifier = ">=1.10" }, { name = "duckdb", specifier = ">=1.3" }, { name = "dvc", extras = ["s3"], specifier = ">=3.60" }, - { name = "fairlearn", specifier = ">=0.12" }, { name = "highspy", specifier = ">=1.10" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.115" }, - { name = "ipykernel", specifier = ">=6.29" }, - { name = "joblib", specifier = ">=1.5" }, - { name = "jupyter", marker = "extra == 'dev'", specifier = ">=1.1" }, - { name = "jupyterlab", marker = "extra == 'docs'", specifier = ">=4.5" }, - { name = "jupytext", marker = "extra == 'dev'", specifier = ">=1.19" }, { name = "loguru", specifier = ">=0.7" }, - { name = "mapie", specifier = ">=1.4" }, { name = "matplotlib", specifier = ">=3.10" }, - { name = "mlflow", specifier = ">=3.10" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13" }, - { name = "nbclient", specifier = ">=0.10" }, - { name = "nbformat", specifier = ">=5.10" }, - { name = "nbstripout", marker = "extra == 'dev'", specifier = ">=0.9" }, { name = "numpy", specifier = ">=2.0,<3" }, { name = "optbinning", specifier = ">=0.21" }, - { name = "optuna", specifier = ">=4.0" }, - { name = "optuna-integration", marker = "extra == 'dev'", specifier = ">=4.0" }, - { name = "optuna-integration", marker = "extra == 'search'", specifier = ">=4.0" }, - { name = "ortools", specifier = ">=9.10" }, + { name = "ortools", specifier = ">=9.10,<9.12" }, { name = "pandas", specifier = ">=2.3,<4" }, - { name = "pandera", specifier = ">=0.25" }, - { name = "plotly", specifier = ">=6.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0" }, { name = "pyarrow", specifier = ">=18" }, - { name = "pyepo", marker = "extra == 'spo'", specifier = ">=1.0" }, { name = "pyomo", specifier = ">=6.10" }, - { name = "pytest", specifier = ">=9.0" }, + { name = "pypdf", marker = "extra == 'dev'", specifier = ">=6,<7" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0" }, - { name = "python-dotenv", specifier = ">=1.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.10" }, { name = "scikit-learn", specifier = ">=1.6,<2" }, { name = "scipy", specifier = ">=1.14,<2" }, - { name = "seaborn", specifier = ">=0.13" }, - { name = "skops", specifier = ">=0.12" }, - { name = "torch", marker = "extra == 'spo'", specifier = ">=2.5" }, - { name = "tqdm", specifier = ">=4.66" }, - { name = "venn-abers", specifier = ">=1.5" }, ] -provides-extras = ["dev", "search", "spo", "docs"] +provides-extras = ["dev"] [[package]] name = "cryptography" -version = "48.0.1" +version = "49.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, - { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, - { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, - { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, - { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" }, - { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" }, - { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" }, - { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" }, - { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" }, - { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, - { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, - { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, - { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, - { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, - { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, - { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, - { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" }, - { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" }, - { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" }, - { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" }, - { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, - { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, - { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, - { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, - { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, - { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, - { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, - { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, - { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, - { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, -] - -[[package]] -name = "cuda-bindings" -version = "13.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/3a8241c6e19483ac1f1dcf5c10238205dcb8a6e9d0d4d4709240dff28ff4/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d", size = 5730273, upload-time = "2026-03-11T00:12:37.18Z" }, - { url = "https://files.pythonhosted.org/packages/e9/94/2748597f47bb1600cd466b20cab4159f1530a3a33fe7f70fee199b3abb9e/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1", size = 6313924, upload-time = "2026-03-11T00:12:39.462Z" }, - { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, - { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7", size = 51657, upload-time = "2026-04-27T22:42:07.712Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, -] - -[package.optional-dependencies] -cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, + { url = "https://files.pythonhosted.org/packages/63/d3/4a83af35d65e3fad632c926fad684c193ea4398569ccb0bbbc7fe8f5dc9a/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b", size = 3993685, upload-time = "2026-06-12T20:02:14.883Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f9dac0ab7f80368c56993a7bf638ef9935f825c91902798481fac0898138/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838", size = 4676239, upload-time = "2026-06-12T20:02:28.793Z" }, + { url = "https://files.pythonhosted.org/packages/d7/70/2ba3769dd0ae167e2f33dfa9592d45db6ff9a61d62ca1a5b3d1bdd09068f/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5", size = 4715584, upload-time = "2026-06-12T20:01:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/94/64/2923570ac1c0bd3a737aa366ac3abbbbde273042308b8cde95e2364a6e6a/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615", size = 4675885, upload-time = "2026-06-12T20:01:55.49Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/614dc7e051418cfe53d55173c1e24c6b0085e89996fe90508c2fdf769aef/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6", size = 4715449, upload-time = "2026-06-12T20:02:05.469Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] [[package]] name = "cvxpy" -version = "1.9.1" +version = "1.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "clarabel" }, { name = "highspy" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "osqp" }, { name = "qdldl" }, - { name = "scipy" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "scs" }, { name = "sparsediffpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a0/52/be03c63aa2f101ea6b50d676bd4618c77b17d4b0ce3e173cf306fb805dc4/cvxpy-1.9.1.tar.gz", hash = "sha256:4504d44011e0ef21348db1d8d1d2784d39742c646853fed13894d9b2292ae853", size = 1889168, upload-time = "2026-05-26T22:15:56.597Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/b7/209c6df38f3621fc2f32298c93e7d0310b8a1ee4ef15e5fa59d90492fa6f/cvxpy-1.9.2.tar.gz", hash = "sha256:b2e939f197a7081a300d5a95812fec8643fabaf23a149abf7e67ca7f89671d92", size = 1916772, upload-time = "2026-06-22T04:37:31.975Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/25/7384a92ca7079a902717b53bb6da674953e744cdaf46114727d73b2cc896/cvxpy-1.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0ed5e3af294f4cc060dbcdf9ecf21a36247bc4ff89b4b390f5def989e42a23db", size = 1604339, upload-time = "2026-05-26T22:09:18.463Z" }, - { url = "https://files.pythonhosted.org/packages/23/a9/98359fc64042152c1692929b32e17461667e69075316bc1a7aef8bf0d6ab/cvxpy-1.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e78eda914db92954ac63c58cd3176349f344bf120f3733fab1dbb5ce7dc88ce9", size = 1394712, upload-time = "2026-05-26T22:09:19.888Z" }, - { url = "https://files.pythonhosted.org/packages/85/ca/228f24cc89d578e66e3bf89f295878d687883c86349362ebd33a9946bb92/cvxpy-1.9.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee74bee6a9179e57f80993556afd1f7588d9afedd8fcca707b32a8dcb84a67fa", size = 4330363, upload-time = "2026-05-26T22:13:47.306Z" }, - { url = "https://files.pythonhosted.org/packages/11/3a/d8e7b83678e8ca959b8409db265deaccb0f28dfc2f32a30e3d4037cba111/cvxpy-1.9.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41bba9aea431568fe863bd1f461659b66efc3154f8d9699a2fef329c5d96bc01", size = 4379645, upload-time = "2026-05-26T22:13:49.153Z" }, - { url = "https://files.pythonhosted.org/packages/90/5f/90666e73bc409693a0edf4f10c1eb47d6d134754bc44ca6770bdde89c347/cvxpy-1.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8589b159deba642cd339c4bc0903ebf9ef8c6d3c318ea7ed6b81269828a5e", size = 1356842, upload-time = "2026-05-26T22:05:15.276Z" }, - { url = "https://files.pythonhosted.org/packages/59/d6/ef27ea596e3bf9393aa3d0f9fbb49876a0d658b4d9bc8e989ba6f6141e66/cvxpy-1.9.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ab4900f6ee50ea89a7778b15b7361bdc23fb2a40bc8ed34863d69edb1e8db25", size = 1606249, upload-time = "2026-05-26T22:09:31.931Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/2fb792fc990e3dd73710610e5df6fab51e421ce51f6190602cd3af73220c/cvxpy-1.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab98f4e5b31aef375efee96c22a499b3d6deed34b9b92ab852854745865e35ec", size = 1396105, upload-time = "2026-05-26T22:09:33.148Z" }, - { url = "https://files.pythonhosted.org/packages/b8/34/8ab5203ab71bf1f4a4679f04949bf28a9e4175b69f88fee23eef77520266/cvxpy-1.9.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5a0f0e8c7999b682aee3c1bcb3d65e6102591ce4ced7c99ea3672c9d993c63d", size = 4348501, upload-time = "2026-05-26T22:15:53.31Z" }, - { url = "https://files.pythonhosted.org/packages/b0/19/588ff6966046a553ed8ca1cb429cecbc34ac7bb7945b954bdf1b3cae1bb8/cvxpy-1.9.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df7f2d60d80be2b76617c68c26c241c7208bd4336edc6d726e048ce039570016", size = 4397460, upload-time = "2026-05-26T22:15:55.02Z" }, - { url = "https://files.pythonhosted.org/packages/a7/fd/c7f778cc2c2d5efed895a0fca50650ad9a9ed7c3bf0506bd5d13b62e89b1/cvxpy-1.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:d54e298739afffe97a4f0e7c2584f70cf65ba5c0107cfd6166293873db80d311", size = 1356961, upload-time = "2026-05-26T22:09:25.23Z" }, + { url = "https://files.pythonhosted.org/packages/e7/bf/5e5bee80310c7edca68b7b1831533c692920d133fd0f5c98fb6a6c7310ca/cvxpy-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e6d448befa8607ee12a037ffb5045039c0e08e8a23633c5289ee62eef5042ccc", size = 1633262, upload-time = "2026-06-22T04:30:57.683Z" }, + { url = "https://files.pythonhosted.org/packages/03/70/ff364ee91203768224874f65d85ae8751a2e2dc97005931d22a2248f0a97/cvxpy-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d307b3fd7e289b1bde01a4cfb536a9353b9739967faf8f90269f60365e76a252", size = 1423642, upload-time = "2026-06-22T04:30:58.973Z" }, + { url = "https://files.pythonhosted.org/packages/24/a9/8d5cd3b0c77069ca2bba80f4d0558f632a9df134e7eb9597b2f0795f86b7/cvxpy-1.9.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee5338332f70b6bdb28a9f291b8b1e8a42d3f7ee129667eb30f9060259351d9", size = 4359277, upload-time = "2026-06-22T04:38:20.652Z" }, + { url = "https://files.pythonhosted.org/packages/23/2f/e672607671d092f7ee2788aecf93a03674ea1e38f9daee89f41ffc5d54f0/cvxpy-1.9.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f3c7a067446e27af669bd29c9868f6ee74aeab73c98a52dc3535a367c40267", size = 4408555, upload-time = "2026-06-22T04:38:22.369Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b2/344372aed15f767a6aa141501543280a80f236a5d2bcef8c186ff78ae232/cvxpy-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:d62d01a07dbed2a59d4e059e610e11db600a91bc0be86b2d8dbc1ec2de43ac5e", size = 1385907, upload-time = "2026-06-22T04:27:28.548Z" }, + { url = "https://files.pythonhosted.org/packages/b4/43/929526a0801cdd56fbb8350f2200cec49739344d12f310f0aafea1f3506e/cvxpy-1.9.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc9082110ac7f9d9a121f4dea73936f9b65ec043af91562677478edf4962905b", size = 1635177, upload-time = "2026-06-22T04:32:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0b/3be49d69c0a90e22572b5ed1c568d4f2357f14f013597a4314f4ea98a2da/cvxpy-1.9.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba59da053c4d15fffa54921dc4b8b929d5a11937692703133858c2be6329107b", size = 1425036, upload-time = "2026-06-22T04:32:08.444Z" }, + { url = "https://files.pythonhosted.org/packages/28/70/80e2ec67a9588cf6e2b99f3e528c4508457cba16c5f5a2a8195c255d043e/cvxpy-1.9.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fa6e86a6ecd19a63751b470a3ed44d5fba6663ab9e05d6040bb954e80a4f7551", size = 4379465, upload-time = "2026-06-22T04:37:29.121Z" }, + { url = "https://files.pythonhosted.org/packages/56/6b/f0f804c7b626917edfe1801baddb85076cdb2fc0e52ed49ff112db82f980/cvxpy-1.9.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:074897ae8f2378174099c082feef5a93b44f4c8dc9b3e960397c31fa3cebf722", size = 4427785, upload-time = "2026-06-22T04:37:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/60/54/81871ee6599d33e3a60907829aafff78e713e1c3a34ce57d1e4ef26fdb5d/cvxpy-1.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:1cc56d04515201a3e2f58a16ea59812a0126dd1419f0726f8322e6dc227a9062", size = 1386030, upload-time = "2026-06-22T04:35:23.856Z" }, ] [[package]] @@ -1054,409 +700,104 @@ wheels = [ ] [[package]] -name = "dacite" -version = "1.6.0" +name = "dictdiffer" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/bf/3f0912b4cfd861cd0fb7278c2b8d5bfb0c613ec1b7922e25e4115287b73a/dacite-1.6.0.tar.gz", hash = "sha256:d48125ed0a0352d3de9f493bf980038088f45f3f9d7498f090b50a847daaa6df", size = 16538, upload-time = "2020-11-30T09:08:06.69Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/9d/11a073172d889e9e0d0ad270a1b468876c82d759af7864a8095dfc73f46d/dacite-1.6.0-py3-none-any.whl", hash = "sha256:4331535f7aabb505c732fa4c3c094313fc0a1d5ea19907bf4726a7819a68b93f", size = 12458, upload-time = "2020-11-30T09:08:05.142Z" }, + { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" }, ] [[package]] -name = "daff" -version = "1.4.2" +name = "diskcache" +version = "5.6.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/d0/c0a1374db3afad0f9dfe6c795e5df102af03d49ad5e6e8502fb09eb88110/daff-1.4.2.tar.gz", hash = "sha256:47f0391eda7e2b5011f7ccac006b9178accb465bcb94a2c9f284257fff5d2686", size = 148251, upload-time = "2025-05-04T19:24:11.521Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/fe/d54a874e8d7b88bc03c459f63a993305db50039b734fab751a0466dabfc1/daff-1.4.2-py3-none-any.whl", hash = "sha256:88981a21d065e4378b5c4bd40b975dbfdea9b7ff540071f3bb5e20cc8b3590b5", size = 144922, upload-time = "2025-05-04T19:24:09.999Z" }, + { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, ] [[package]] -name = "dagshub" -version = "0.7.0" +name = "distlib" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "appdirs" }, - { name = "boto3" }, - { name = "click" }, - { name = "dacite" }, - { name = "dagshub-annotation-converter" }, - { name = "dataclasses-json" }, - { name = "gitpython" }, - { name = "gql", extra = ["requests"] }, - { name = "httpx" }, - { name = "pandas" }, - { name = "pathvalidate" }, - { name = "python-dateutil" }, - { name = "pyyaml" }, - { name = "rich" }, - { name = "semver" }, - { name = "tenacity" }, - { name = "treelib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/62/76/819093099d568fcd709d2b5c0a1b25419065e4884cf181db944490c63cf7/dagshub-0.7.0.tar.gz", hash = "sha256:19948c6922991c456e62483dbaa5656115268233e9c8d5fa0341faa17ffdc1c0", size = 247095, upload-time = "2026-04-16T13:45:28.968Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/e9/835554e67bcee09bd2e8827779271a1e0c53462061b03c3daf2b2c0b169e/dagshub-0.7.0-py3-none-any.whl", hash = "sha256:c1c2aae451290ba978e395ed3d556576e40f4eb4c91597fa05307c4f557da42f", size = 273081, upload-time = "2026-04-16T13:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, ] [[package]] -name = "dagshub-annotation-converter" -version = "0.2.0" +name = "distro" +version = "1.9.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lxml" }, - { name = "pandas" }, - { name = "pillow" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/fa/54a706589048618c3cdda92d74d793192a28c6b3290e3af6dc33a0f31a9f/dagshub_annotation_converter-0.2.0.tar.gz", hash = "sha256:d2df6ee954c1b1581b3ba38dcf5ada74ca525e75fdb13a7a524a3aaa9d006247", size = 75601, upload-time = "2026-04-14T09:12:57.223Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/44/d8/ff1d5c0f31eba4331c4eee97dc21730eb36743946381f8cca62675179f50/dagshub_annotation_converter-0.2.0-py3-none-any.whl", hash = "sha256:29752760ae6831df67c196dfd899ec9d158b5d4e9d47084d6d859c788bf9d91c", size = 68190, upload-time = "2026-04-14T09:12:55.916Z" }, + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] [[package]] -name = "databricks-sdk" -version = "0.72.0" +name = "dpath" +version = "2.2.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/b7/898e54cc4e32daf53c7beaa22c3ce2d2889f9720d556e4696c291dc0483b/databricks_sdk-0.72.0.tar.gz", hash = "sha256:54596db8f1f75330ddf5ec5527a9c0a6a98c4c4190870ed31449582797388d3b", size = 800694, upload-time = "2025-11-04T13:30:36.305Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/73/e14f15cd92fd1ddcf411dc14934c5b3890366604150886a16455870190e9/databricks_sdk-0.72.0-py3-none-any.whl", hash = "sha256:86f6a6c8ad5cc84a66d394f2151bfd326a207e11d83eb943cb8a447c762b905c", size = 753845, upload-time = "2025-11-04T13:30:34.1Z" }, + { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, ] [[package]] -name = "dataclasses-json" -version = "0.6.7" +name = "duckdb" +version = "1.5.4" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "marshmallow" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } +sdist = { url = "https://files.pythonhosted.org/packages/31/29/9bad86ed7aa812d8c822a27c15c355b6d5423b991feeec86ed18027b6daa/duckdb-1.5.4.tar.gz", hash = "sha256:f9e32f1cdd106793d79d190186bed9e75289d51e68bd9174e47c04bffedeab6f", size = 18046634, upload-time = "2026-06-17T10:48:52.499Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, + { url = "https://files.pythonhosted.org/packages/56/bb/7921dabd50daef3969f14cd8a5a14c24eee337db7914a462f2defa8add92/duckdb-1.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3fb41d9cfccb7e44511eeeed263ae98143ca63bdb1ef84631ba637c314efa1b5", size = 32663142, upload-time = "2026-06-17T10:47:45.471Z" }, + { url = "https://files.pythonhosted.org/packages/a6/83/2137765eaba6a9aefe3bb9848ddaac7407fe3ba19b292f98b31f3b7ab27f/duckdb-1.5.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ba7b666bc9c78d6a930ee9f469024149f0c6a23fb7d2c3418aad6774339bec0", size = 17321485, upload-time = "2026-06-17T10:47:47.778Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b2/a02c1ee43fd7e8cf1fc2e3d377f3dcf9d4a3e58a4549557516e1866ff0da/duckdb-1.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d9e6817fcbc09d2605a2c8c041ac7824d738d917c35a4d427e977647e1d7944", size = 15470820, upload-time = "2026-06-17T10:47:49.977Z" }, + { url = "https://files.pythonhosted.org/packages/d8/48/a243d30223b024bc6057abe472b002cff01e97efefb4d2f0b0dcc5aece0b/duckdb-1.5.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02dd9f9a6124069213f13e3a474c208028c472fe1acdae12b38761f954fe4fc6", size = 19341849, upload-time = "2026-06-17T10:47:52.205Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/a5d48de4771e2403a8ef26a20dc7457b1c8f7e398ff0caf9c0cad8805f89/duckdb-1.5.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccc7f2694d02b4763fee61021d45e12f7bc5743993686563957df0cef799fbae", size = 21451698, upload-time = "2026-06-17T10:47:54.653Z" }, + { url = "https://files.pythonhosted.org/packages/79/b8/8244d7741b4afae67775cf0cb0d4eb9e923a83110907e4801e17fa078480/duckdb-1.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:4c430e788d99b50854209bf2833ba36a45df75e57f86efb477046cd408bbd077", size = 13132643, upload-time = "2026-06-17T10:47:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/e4/57/8169822a37f6dd7d561c567f9007e3cf04bf97bccb619afe90db849c0962/duckdb-1.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:e2dc8340cfb6006025a798c50f40126d6e945a1d2487be94667bb4166556ce7b", size = 13986386, upload-time = "2026-06-17T10:47:59.345Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f2/e2f4b477ae3a3b40e8b5f429832e48edb62ed9da99807cc4902e157e5646/duckdb-1.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:291a9e7502551170af989ff63139a7a49e99d68edbc5ef5017ac27541fe54c65", size = 32708876, upload-time = "2026-06-17T10:48:01.527Z" }, + { url = "https://files.pythonhosted.org/packages/2e/2b/b698d82a5e1e30b6a05748d72045f672994c6b22f4f0f8423523608b991f/duckdb-1.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83e8c089bbb756ca4471d8b05943b80a106058697cf00615e70423106bb783bc", size = 17346125, upload-time = "2026-06-17T10:48:04.035Z" }, + { url = "https://files.pythonhosted.org/packages/71/75/37e13f39268eaf34864453b3a039c4a1ff0b088d3eae45a4289b41c98c1b/duckdb-1.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ff96d2a342b200e1ec6f1f19986c77f4ac16a49b6112f71c5b763989203a9d60", size = 15488133, upload-time = "2026-06-17T10:48:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/cc/59/2d082af578f689231798245b54562c61416e49049b0bda81a06c56a4b53e/duckdb-1.5.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f935ef210ab00bc94bb1e3052697adaa36bb0ce7bdfeda8b0f34e2ff1643870", size = 19367895, upload-time = "2026-06-17T10:48:08.59Z" }, + { url = "https://files.pythonhosted.org/packages/52/2b/55c34d2863a76ca824ef8274691e84240b4ff1acde3d231709e82557c240/duckdb-1.5.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cda263d8c20addb8d4f95464787cbe0af1144f7ab7e21db3709fb826ee01725", size = 21486499, upload-time = "2026-06-17T10:48:10.963Z" }, + { url = "https://files.pythonhosted.org/packages/cf/30/ade5952b8182fac86fab43b95ebe3836e66381d0ad64eb1e54bd8207c988/duckdb-1.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:266c7c909558ce7377f57d082cee408aadebdd9111be017558ca54e44a031037", size = 13147934, upload-time = "2026-06-17T10:48:13.061Z" }, + { url = "https://files.pythonhosted.org/packages/f5/00/278f0f70e25b9911afe2fd227b9460f2e6d76177f0dcc03f7f1454afefa5/duckdb-1.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:f14e79a006341f29ee5a2692a24dac5114e77533d579c57ec39124adf0135033", size = 13965235, upload-time = "2026-06-17T10:48:15.782Z" }, ] [[package]] -name = "dbt-adapters" -version = "1.24.2" +name = "dulwich" +version = "1.2.10" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "agate" }, - { name = "dbt-common" }, - { name = "dbt-protos" }, - { name = "mashumaro", extra = ["msgpack"] }, - { name = "protobuf" }, - { name = "pytz" }, - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/6c/65190ca22dea211471275355c69fd84925d2e1712993c5868352234cb084/dbt_adapters-1.24.2.tar.gz", hash = "sha256:fb27141acdfad1188bcb584e9b9b1976cb9ae718618fcac132660fdaeb9b11a4", size = 143779, upload-time = "2026-05-21T10:36:02.862Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/d9/e6272430ccf84739509f45006543a1babcecc55a70cbd296424901df8ed5/dulwich-1.2.10.tar.gz", hash = "sha256:106546960f592216315e56d32a92e72e67f1e86671335722f833f5ecd5b73314", size = 1310282, upload-time = "2026-07-07T00:52:05.986Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/d5/e1f3d6a1bfa472ea00c3caa76301b73ec0e4f87b32468111ae7d31c16efa/dbt_adapters-1.24.2-py3-none-any.whl", hash = "sha256:f6afcef287165984d6e785bf1661473fdfbb6d2e723eaebd00355df2985124da", size = 176555, upload-time = "2026-05-21T10:36:00.965Z" }, + { url = "https://files.pythonhosted.org/packages/55/11/8a2aa5f7220d9290e4c9ad4ac6d482d9b34762079331e8e02d835329f552/dulwich-1.2.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6d989605955bfbb678425a2e8710100ef35960340bfb55faaca902006c69163", size = 1378353, upload-time = "2026-07-07T00:51:10.307Z" }, + { url = "https://files.pythonhosted.org/packages/7a/66/859b121c1f18fa32833a239beadd67cf11df89974ff223b28d9ff53f4f2f/dulwich-1.2.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af4a750597d1a6e77bca74b9246bf84b62a9bfa7e48783af4ed0e9f04f0dbfbd", size = 1371855, upload-time = "2026-07-07T00:51:11.999Z" }, + { url = "https://files.pythonhosted.org/packages/85/66/1833afc2444fc70e6e79458169275687aef858ea1c6a8198e2b7f97da27d/dulwich-1.2.10-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6c715289e8c5667978ff28a1847736633766ca1c7082144ba0ee95e459be3f97", size = 1493934, upload-time = "2026-07-07T00:51:13.865Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1f/a200b9071a4d9dce6ce0033654d63ee389be2d4de432d1d7028dccdbf8d5/dulwich-1.2.10-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f21c7f2dc4de8353b59b82bc0245838f3f94ff9332b71f6394642e3bb012c71b", size = 1523369, upload-time = "2026-07-07T00:51:15.338Z" }, + { url = "https://files.pythonhosted.org/packages/11/1b/e36deccfa78f674e7c4d7482b2a710581a1161b65080dd6b1daac17c5b42/dulwich-1.2.10-cp311-cp311-win32.whl", hash = "sha256:ca1fbe5f4b2ee48154b24c68eeb689b848394a857f7cb3dea3de0b86c464454b", size = 1043413, upload-time = "2026-07-07T00:51:16.9Z" }, + { url = "https://files.pythonhosted.org/packages/fc/67/fb78d06d58486ea3bef2c87af17ea0b106d9863e89664cc834cea060af6e/dulwich-1.2.10-cp311-cp311-win_amd64.whl", hash = "sha256:9874964ac78713c640c42c9dc27a74dd2d4169d99821a57301d37e5673409d27", size = 1103445, upload-time = "2026-07-07T00:51:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/73/fb/af9504fed5f023194143bd059132729aad00f4608f4ed785dc1f94ff500d/dulwich-1.2.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4bf4671061aa98e7f4d420fe5238b2ca2cb2a00a280c8f549fa92d34569971a8", size = 1371347, upload-time = "2026-07-07T00:51:20.811Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3e/aff9e8b9d07f3afac9c618cd2e7529e4695375a64ebd07831462bb78c946/dulwich-1.2.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43315f1e95f7d5c1a6f2695f5534263b80678ccbfd9756101f07759a4e65850e", size = 1356141, upload-time = "2026-07-07T00:51:22.108Z" }, + { url = "https://files.pythonhosted.org/packages/89/8d/03c2692896241b94d04cbc8f4db662458864e13604e5295baf96a2b647db/dulwich-1.2.10-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:699edc94fddd0a2346e41e405fd7c537072d75cb9ddb39bd47f2e8fbd56d1e4e", size = 1437999, upload-time = "2026-07-07T00:51:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/75/b37e1c9548d7b9355971e462acf5da795c9539d8b330f4495ddf0244d99a/dulwich-1.2.10-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2dc7c46cfeb40022d2fca9d23c3ca1aba602ee99bfb6554278f05f2aa890a0e", size = 1463581, upload-time = "2026-07-07T00:51:25.06Z" }, + { url = "https://files.pythonhosted.org/packages/32/a0/2de9d8bfd975b79e51c40394d8fb627dbffc97a4c89778eb3314b304ea6e/dulwich-1.2.10-cp312-cp312-win32.whl", hash = "sha256:9c01a6b29cedb054a16fa04adf834a6531a39450888fc7ce2b3bf0fb6cba70c4", size = 1082677, upload-time = "2026-07-07T00:51:26.578Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/49e913c54d7723c5a5bb12a38552a6db5cb691631326882ab2208a5186fa/dulwich-1.2.10-cp312-cp312-win_amd64.whl", hash = "sha256:96981c6704def4fd0f91fb9be9f613cf99ba327e2a655b8ef987f5f5245420f3", size = 1053703, upload-time = "2026-07-07T00:51:28.023Z" }, + { url = "https://files.pythonhosted.org/packages/d1/f5/efdcbd50a4480083d8d049ea1c49095097802599f7ecd2dc92942af7f4c4/dulwich-1.2.10-py3-none-any.whl", hash = "sha256:b9f18afbe8e6f6a380b199df3d2da14312b4eac4ee34ee77c6f61c56bd2693ed", size = 711548, upload-time = "2026-07-07T00:52:04.539Z" }, ] [[package]] -name = "dbt-common" -version = "1.38.0" +name = "dvc" +version = "3.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "agate" }, - { name = "colorama" }, - { name = "dbt-protos" }, - { name = "deepdiff" }, - { name = "isodate" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "mashumaro", extra = ["msgpack"] }, - { name = "pathspec" }, - { name = "protobuf" }, - { name = "python-dateutil" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/27/1cb45b595e280d4e66b122c539655be280236af27a964114b8f163702674/dbt_common-1.38.0.tar.gz", hash = "sha256:05f9c682c1c09b6a0ce79fe18dd7244a4549c3740329692bc19be9e7498f516f", size = 87065, upload-time = "2026-05-04T20:28:19.274Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/d9/11763d94eb3257c11463781c18aef3e9613829b9f0e55609a494e596f574/dbt_common-1.38.0-py3-none-any.whl", hash = "sha256:1646e5384a8e317fa63890e60e41ca50dd4c5a89e02ad9a141c184d4b672b696", size = 88460, upload-time = "2026-05-04T20:28:17.762Z" }, -] - -[[package]] -name = "dbt-core" -version = "1.11.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "agate" }, - { name = "click" }, - { name = "daff" }, - { name = "dbt-adapters" }, - { name = "dbt-common" }, - { name = "dbt-extractor" }, - { name = "dbt-protos" }, - { name = "dbt-semantic-interfaces" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "mashumaro", extra = ["msgpack"] }, - { name = "networkx" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "pytz" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "snowplow-tracker" }, - { name = "sqlparse" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/af8f4ab1e3e979b5677e876444f3a9c1b1021c2d4e77143e80f421bdc1f7/dbt_core-1.11.11.tar.gz", hash = "sha256:e53cfccc8c9c13a082e518bf80cf3bb33c2f1fdc3cab48ec822ef93737eccb81", size = 973079, upload-time = "2026-05-20T11:10:01.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/d2/9e1eb5d3c8f375d4e5523b9123b2ae25ea21d85abb044db2eff2e3ee9c1d/dbt_core-1.11.11-py3-none-any.whl", hash = "sha256:7f9e11f3f5400e20f40f544440800ace23fa0755086f87b08e9446181f8720e1", size = 1061938, upload-time = "2026-05-20T11:09:59.856Z" }, -] - -[[package]] -name = "dbt-duckdb" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dbt-adapters" }, - { name = "dbt-common" }, - { name = "dbt-core" }, - { name = "duckdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/d3/0f9f6a4de94e0f95dcfabbba5e8ef7670de695483345edd9e5b36e93d024/dbt_duckdb-1.10.1.tar.gz", hash = "sha256:5d6df1589d4ba21fe20ac08454a32763d3263798c9f5914280eabd1dc285cbc0", size = 145907, upload-time = "2026-02-17T17:29:04.283Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/26/0ce2b1aeecdd1a18a9fac6f02d08f60c95944f036866f3fe37146298d4e4/dbt_duckdb-1.10.1-py3-none-any.whl", hash = "sha256:90658ecb367082786c5ea2ffbf9e35bb4116fa5ad1bc2f287c4dc1f3984bafa1", size = 85089, upload-time = "2026-02-17T17:29:03.117Z" }, -] - -[[package]] -name = "dbt-extractor" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/06/1f7b5d277af4bd7c3ab5065f79407c46a73950f0879fac69e51067c87649/dbt_extractor-0.6.0.tar.gz", hash = "sha256:d6cf08ec793b8bc2bd6e260ef818230ae68a4f71436fa489f08d7db1a52e2ffe", size = 270461, upload-time = "2025-04-07T16:46:30.532Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/dd/ec8f9e48e7dd5a52a69cca7907681d1779cf1cc8b02f2aa2acb6a2bf8bb4/dbt_extractor-0.6.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:4b6b1e70dde78cb904ca7a8958c2c803e77779b6ce108f4ea7ac479f5700db89", size = 790206, upload-time = "2025-04-07T16:46:05.352Z" }, - { url = "https://files.pythonhosted.org/packages/03/5f/233f326336aa21fbd9e7268f239a8464af145abd398a360d894c3286699d/dbt_extractor-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dcf14ed245de8df269815ff4c4f555fa72d2621f4fff37c023b8c99d0e421b4f", size = 404381, upload-time = "2025-04-07T16:46:07.471Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2a/e14c13b9a437780c5712525ce537915b531bba45481fc7102deb4492ff83/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af451633390ac19669d3bde6c79822e657d32f5d903b3388bb00d56333fd52d5", size = 435109, upload-time = "2025-04-07T16:46:09.443Z" }, - { url = "https://files.pythonhosted.org/packages/58/2e/1ef1cd2b36973bea0a6823a7b7cd1b3db29b61ddebb015ceaea88b9e9347/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05bcfab7ebd70296ceb31742e8333ba66a2c939de44e61a7088bebafa939aaf6", size = 434550, upload-time = "2025-04-07T16:46:10.916Z" }, - { url = "https://files.pythonhosted.org/packages/40/5a/468a2855181aaee5402efbf9ef757d074cd306eec22bbcd267cdd0edbe94/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:71b3f8897138cc6698d313b9a3d0450fd021937ff5463269ee18ed415541781b", size = 470137, upload-time = "2025-04-07T16:46:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/b2/18/611dceb2fa7ea668471f290f34fec55fa3283e3ee9d0475d964e6ffaff97/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:868af715a6328d7317ce6e4db238f850f660fef13fb36b7ab4cf9163ed5f54ff", size = 524331, upload-time = "2025-04-07T16:46:14.177Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ad/9dd410d4d95e336ae6b10c53c939bf1ff8e9991e1adb5ea4aefc4a87c445/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c1fd2b083a75e80b13e9874dc9699bfdfddf3baa9b6a8dea48de06d51a082733", size = 517959, upload-time = "2025-04-07T16:46:15.68Z" }, - { url = "https://files.pythonhosted.org/packages/a4/4f/6994cdfb51c5652fad0c8f9cf5b3ec1816cb10e99ed145eb27e6a9bcc16b/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:311f0d3a4994751c541a4fa303d205727ba90e90c85286c03d3d9284e2bf0bd4", size = 494850, upload-time = "2025-04-07T16:46:17.265Z" }, - { url = "https://files.pythonhosted.org/packages/df/5e/fad01e18d68ffd09c0f39cdedeed8fcaaea74a8b46d1a944472b5f95b72b/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aecfa43f7e6f139e76d47e4e1d7b189655ae19a8cf697686230bacb89a94ae74", size = 442739, upload-time = "2025-04-07T16:46:19.002Z" }, - { url = "https://files.pythonhosted.org/packages/9d/82/49068ee2b9f38aa34d0f3196bb7b71d11af86630d5ed5cb6626108c97cd6/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a5cb810edc60c0486f78cc29739ebda70c81b10a1686861e78addc9f91fcd7de", size = 618014, upload-time = "2025-04-07T16:46:21.571Z" }, - { url = "https://files.pythonhosted.org/packages/18/c6/cdaf1ac8959d571b5cb3587b8afef9e5fe60b99fe59aca94560808501d8b/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:080fd1edf123926ed97929c65a75874d0fea687ccd5d3ebbc9e81b339f099604", size = 697290, upload-time = "2025-04-07T16:46:23.089Z" }, - { url = "https://files.pythonhosted.org/packages/94/6d/46bdb9a809c66784fcc19b853311568cfd3041c075f0a578cb7116686841/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:1b9ed7b15df983a735f87773f6765db8458680c02fcebbf89df4e238503c0e08", size = 644443, upload-time = "2025-04-07T16:46:24.463Z" }, - { url = "https://files.pythonhosted.org/packages/3b/02/b111856273e414ac80ef58d2103c9b7c6a5b29b1ec248999d3d5873ada00/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:caeaba8d8c813f8e32d586c12615c0c7d6b99bee4f1be845312e80ef731de164", size = 613017, upload-time = "2025-04-07T16:46:25.913Z" }, - { url = "https://files.pythonhosted.org/packages/c4/de/d1492ab6beaf0a18aee17c7a9562592ac2981e962b4058262f5eb6dabfc5/dbt_extractor-0.6.0-cp39-abi3-win32.whl", hash = "sha256:369dcc3499f160256756585783f1308868076d5a65d0a051348d22da8b90e67d", size = 252721, upload-time = "2025-04-07T16:46:27.295Z" }, - { url = "https://files.pythonhosted.org/packages/60/36/f5b1c4159fa911607f3a49fcbc535e4783870fd887bc0a1b3ad42587cb73/dbt_extractor-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:a79a570fdcb672505ac2bdc12360a2a7aec622ef604d8c607225854ff862518c", size = 277146, upload-time = "2025-04-07T16:46:28.991Z" }, -] - -[[package]] -name = "dbt-protos" -version = "1.0.491" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7c/f5/865d7125d40d1ae2fa17dff80811773dfd8f088b8f20eaf007262b8ec63f/dbt_protos-1.0.491.tar.gz", hash = "sha256:039b696cb8141419926163602f62ff2bbf68c134d8556a1a4474639900281d3b", size = 142067, upload-time = "2026-05-07T19:43:56.628Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/7a/07c4d33e20306d845bcdf2a41937dde209cc060ab02b31d85f4dc1a0da6b/dbt_protos-1.0.491-py3-none-any.whl", hash = "sha256:42141d099731fd00de8a12477d4cd58098532b811f92fa8c48f70f0b9422abf0", size = 207510, upload-time = "2026-05-07T19:43:54.928Z" }, -] - -[[package]] -name = "dbt-semantic-interfaces" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "importlib-metadata" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "more-itertools" }, - { name = "pydantic" }, - { name = "python-dateutil" }, - { name = "pyyaml" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b0/91/c702d8fb143541fda10f5eb7a7a89f34bda38ee043ecb3e3653363d0c5a0/dbt_semantic_interfaces-0.9.0.tar.gz", hash = "sha256:5c921257dce8bb51c9ffb5479f2bdd959e16ebfb98ee833de6daa70788c47271", size = 93865, upload-time = "2025-07-09T20:06:30.454Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/82/41708b2b69d5fead88dea5ca0d863d6291da83ca6f1bd19246842d397e2b/dbt_semantic_interfaces-0.9.0-py3-none-any.whl", hash = "sha256:1b54c06ba89190a47a7f0563360930a0cce869e55b484ca09d261ade0e319155", size = 147008, upload-time = "2025-07-09T20:06:32.466Z" }, -] - -[[package]] -name = "debugpy" -version = "1.8.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/b7/cd8080344452e4874aae67c40d8940e2b4d47b01601a8fd9f44786c757c7/debugpy-1.8.20.tar.gz", hash = "sha256:55bc8701714969f1ab89a6d5f2f3d40c36f91b2cbe2f65d98bf8196f6a6a2c33", size = 1645207, upload-time = "2026-01-29T23:03:28.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/56/c3baf5cbe4dd77427fd9aef99fcdade259ad128feeb8a786c246adb838e5/debugpy-1.8.20-cp311-cp311-macosx_15_0_universal2.whl", hash = "sha256:eada6042ad88fa1571b74bd5402ee8b86eded7a8f7b827849761700aff171f1b", size = 2208318, upload-time = "2026-01-29T23:03:36.481Z" }, - { url = "https://files.pythonhosted.org/packages/9a/7d/4fa79a57a8e69fe0d9763e98d1110320f9ecd7f1f362572e3aafd7417c9d/debugpy-1.8.20-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:7de0b7dfeedc504421032afba845ae2a7bcc32ddfb07dae2c3ca5442f821c344", size = 3171493, upload-time = "2026-01-29T23:03:37.775Z" }, - { url = "https://files.pythonhosted.org/packages/7d/f2/1e8f8affe51e12a26f3a8a8a4277d6e60aa89d0a66512f63b1e799d424a4/debugpy-1.8.20-cp311-cp311-win32.whl", hash = "sha256:773e839380cf459caf73cc533ea45ec2737a5cc184cf1b3b796cd4fd98504fec", size = 5209240, upload-time = "2026-01-29T23:03:39.109Z" }, - { url = "https://files.pythonhosted.org/packages/d5/92/1cb532e88560cbee973396254b21bece8c5d7c2ece958a67afa08c9f10dc/debugpy-1.8.20-cp311-cp311-win_amd64.whl", hash = "sha256:1f7650546e0eded1902d0f6af28f787fa1f1dbdbc97ddabaf1cd963a405930cb", size = 5233481, upload-time = "2026-01-29T23:03:40.659Z" }, - { url = "https://files.pythonhosted.org/packages/14/57/7f34f4736bfb6e00f2e4c96351b07805d83c9a7b33d28580ae01374430f7/debugpy-1.8.20-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:4ae3135e2089905a916909ef31922b2d733d756f66d87345b3e5e52b7a55f13d", size = 2550686, upload-time = "2026-01-29T23:03:42.023Z" }, - { url = "https://files.pythonhosted.org/packages/ab/78/b193a3975ca34458f6f0e24aaf5c3e3da72f5401f6054c0dfd004b41726f/debugpy-1.8.20-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:88f47850a4284b88bd2bfee1f26132147d5d504e4e86c22485dfa44b97e19b4b", size = 4310588, upload-time = "2026-01-29T23:03:43.314Z" }, - { url = "https://files.pythonhosted.org/packages/c1/55/f14deb95eaf4f30f07ef4b90a8590fc05d9e04df85ee379712f6fb6736d7/debugpy-1.8.20-cp312-cp312-win32.whl", hash = "sha256:4057ac68f892064e5f98209ab582abfee3b543fb55d2e87610ddc133a954d390", size = 5331372, upload-time = "2026-01-29T23:03:45.526Z" }, - { url = "https://files.pythonhosted.org/packages/a1/39/2bef246368bd42f9bd7cba99844542b74b84dacbdbea0833e610f384fee8/debugpy-1.8.20-cp312-cp312-win_amd64.whl", hash = "sha256:a1a8f851e7cf171330679ef6997e9c579ef6dd33c9098458bd9986a0f4ca52e3", size = 5372835, upload-time = "2026-01-29T23:03:47.245Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl", hash = "sha256:5be9bed9ae3be00665a06acaa48f8329d2b9632f15fd09f6a9a8c8d9907e54d7", size = 5337658, upload-time = "2026-01-29T23:04:17.404Z" }, -] - -[[package]] -name = "decorator" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, -] - -[[package]] -name = "deepdiff" -version = "8.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "orderly-set" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/50/767448e792d41bfb6094ee317a355c1cb221dca24b2e178e2203bbea2a77/deepdiff-8.6.2.tar.gz", hash = "sha256:186dcbd181e4d76cef11ab05f802d0056c5d6083c5a6748c1473e9d7481e183e", size = 634860, upload-time = "2026-03-18T17:16:33.785Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/5f/c52bd1255db763d0cdcb7084d2e90c42119cb229302c56bdf1d0aa78abd2/deepdiff-8.6.2-py3-none-any.whl", hash = "sha256:4d22034a866c3928303a9332c279362f714192d9305bac17c498720d095fd1b4", size = 91979, upload-time = "2026-03-18T17:16:32.171Z" }, -] - -[[package]] -name = "defusedxml" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, -] - -[[package]] -name = "dictdiffer" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" }, -] - -[[package]] -name = "dill" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, -] - -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - -[[package]] -name = "distlib" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - -[[package]] -name = "dpath" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/ce/e1fd64d36e4a5717bd5e6b2ad188f5eaa2e902fde871ea73a79875793fc9/dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e", size = 28266, upload-time = "2024-06-12T22:08:03.686Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/d1/8952806fbf9583004ab479d8f58a9496c3d35f6b6009ddd458bdd9978eaf/dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576", size = 17618, upload-time = "2024-06-12T22:08:01.881Z" }, -] - -[[package]] -name = "duckdb" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/00/d579dcb2a536b6ea3a2563cdad6844f77d81a9b2d4b22a858097f2468acf/duckdb-1.5.3.tar.gz", hash = "sha256:df39428eb130faa35ae96fd35245bdeae6ecf43936250b116b5fead568eb9f16", size = 18026640, upload-time = "2026-05-20T11:55:31.901Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/fc/a8a89c6c73f31c2b58c6abbc2f543e0b736042dd5ef7cc1784c24ec31428/duckdb-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:341a2672e2551ba51c95c1898f0ade983e76675e79038ccb16342c3d6cfb82d7", size = 32583465, upload-time = "2026-05-20T11:54:13.132Z" }, - { url = "https://files.pythonhosted.org/packages/63/f1/3423a2f523dd034e505d4a5dd8e210ae577212e152598dc13b6a5e736e1b/duckdb-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c9e8fa408705081160ede7ead238d16e73a36b8561b700f2bf2d650ae48e7b92", size = 17278520, upload-time = "2026-05-20T11:54:16.368Z" }, - { url = "https://files.pythonhosted.org/packages/e1/1a/7bf5ba1b7ea520557e6b2dbee1c85abab016bdac0c1779d9d0ef76c87300/duckdb-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:70a18f932cf6d87bd0e554613657a515c1443a1724aacfc7ec5137dd28698b03", size = 15424794, upload-time = "2026-05-20T11:54:19.891Z" }, - { url = "https://files.pythonhosted.org/packages/ad/16/ce4b1e386e45fab0268edbf1b85bace20e9437589e9edb2bd5f9a226fa44/duckdb-1.5.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e80eb4d0fb59869cb2c7d7ef494c07fb92014fe8e77d96c170cd1ebc1488a708", size = 19306666, upload-time = "2026-05-20T11:54:22.77Z" }, - { url = "https://files.pythonhosted.org/packages/99/1f/651f8453f26931e8061b7e27b3090f868868185814ecb9216d0bd71ec8ef/duckdb-1.5.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3248b49cd835ea322574bc6aac0ae7a83be85547f49d4f5f5777cb380ee6627f", size = 21418306, upload-time = "2026-05-20T11:54:25.616Z" }, - { url = "https://files.pythonhosted.org/packages/bc/64/e1ffebf010b1631a6fef8d1508f46d4eab3e97c18729af986bb796fa8452/duckdb-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:f4eff89c12c3a362efa012262e57b7b4ab904a7f79bad9178fe365510077abe8", size = 13101423, upload-time = "2026-05-20T11:54:28.107Z" }, - { url = "https://files.pythonhosted.org/packages/e7/42/b1d4e34f9658cc0e13d7aae581ab82643f50a548d5aee8767f0c587cc3a4/duckdb-1.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:75d13308c9da3ee431d1e72b8ab720aa74a1b3e9159d4124cb62435924496334", size = 13951740, upload-time = "2026-05-20T11:54:30.886Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c4/2e34929b16c8d544ef664fad8f7f3a2a9db05746aae1e7c8c4ee3a8b23e4/duckdb-1.5.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ff11a457258148337ef9a392148a8cdbd1069b6c27c21958816c7b67fe6c542d", size = 32626494, upload-time = "2026-05-20T11:54:33.738Z" }, - { url = "https://files.pythonhosted.org/packages/3a/53/3af681793d03771365ae3e2215331151c196a3ac8193f613344840694671/duckdb-1.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fd25f533cb1b6b2c84cc767a9a9bab7769bb1aa44571a2a0bfc91ac3e4a38ac", size = 17301121, upload-time = "2026-05-20T11:54:36.928Z" }, - { url = "https://files.pythonhosted.org/packages/15/e2/c80af1eac2ab5d35fc2c372ef0a84668842e549fbbf7799277b3fccf3e39/duckdb-1.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10960400ed60cdf0fe05bab2086fa8eb733889cb0ceca18d07ff9a00c0e0be7b", size = 15449283, upload-time = "2026-05-20T11:54:39.777Z" }, - { url = "https://files.pythonhosted.org/packages/2d/9a/c63af233c9f761bf5178a5210437e1bc6bcb30fa8a9073de6398cfb12c03/duckdb-1.5.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5f18e7561403054433706c187589e86629a7af09a7efc23a06a8b308e6acc68", size = 19332762, upload-time = "2026-05-20T11:54:42.51Z" }, - { url = "https://files.pythonhosted.org/packages/21/cc/2d77af4fff86012f334ef82e6d54a995a86c8745e58074f1218ed7d25171/duckdb-1.5.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fb7516255a8764545e30f7efacea408cc847764a3027b3b0b3e7d1a7bebbc5c", size = 21453290, upload-time = "2026-05-20T11:54:45.272Z" }, - { url = "https://files.pythonhosted.org/packages/8d/5e/9bc4817a98feb4dab83e56f2245cd3a30d00ee646d4dec7926464e2b3f28/duckdb-1.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:8001eccbc28be244dfd04d708526f34ddd6460b47a8aeb5d0e39d6f7f9e3fe15", size = 13118308, upload-time = "2026-05-20T11:54:48.058Z" }, - { url = "https://files.pythonhosted.org/packages/81/35/e3f32e4e53e2450ddb1db8312a17d1ce455d60cc4941b6ad2cfc908794b0/duckdb-1.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:6d2835e39bb6af73891f73c0f8d4324f98afe00d0b00c6d34b2a582c2256cbb0", size = 13927187, upload-time = "2026-05-20T11:54:50.584Z" }, -] - -[[package]] -name = "dulwich" -version = "1.2.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/85/ceb8ecff5cdeee4ceeebb86b599476dee559041dacc6c2c50cc0d4711549/dulwich-1.2.5.tar.gz", hash = "sha256:0395b2c8924c3424bafe2d9c1edd5348cc4b21ce9c1d6655bf01f9a5c47164c8", size = 1253230, upload-time = "2026-05-28T22:27:55.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/4a/654ae1671610fdf6b65a64586ad67ddd8550d4d08a632b2a4b9614754b6d/dulwich-1.2.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:556593fd11637f80f6018bee1916b1a84f5b420423b470ebb3f1a782ad6ef081", size = 1399277, upload-time = "2026-05-28T22:27:00.801Z" }, - { url = "https://files.pythonhosted.org/packages/85/d8/06ee3bc8eded4bd7adf8adf0c9ea5f19bf96f7e5e626bfaf7311cde4208a/dulwich-1.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a70477c991e96cfe8fdd7c866e7251faf71b38bfeb51d6f27554c9cce1caabf3", size = 1382310, upload-time = "2026-05-28T22:27:02.216Z" }, - { url = "https://files.pythonhosted.org/packages/07/17/a03adf50b9095f9f5d863393f21d585dea39bdc4fdf60788ff3a9407a512/dulwich-1.2.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9008ef25cabd379cda4fa86000fc38ca14b72afe17db798a8c85c0b2b7ce4d1e", size = 1470993, upload-time = "2026-05-28T22:27:04.075Z" }, - { url = "https://files.pythonhosted.org/packages/60/58/1dc352d2a5e80befe4338af7208febb44bcfd7496b0dde5ac6dacb07b031/dulwich-1.2.5-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a5549f4afc973e0a15ea6b0244d57f848d3f3ee13dac557eb311024aebebf128", size = 1497820, upload-time = "2026-05-28T22:27:05.549Z" }, - { url = "https://files.pythonhosted.org/packages/c1/a8/e058959a87e7df7753b112ef66a43ccbc57338c1bbdc23a0edf3833396df/dulwich-1.2.5-cp311-cp311-win32.whl", hash = "sha256:5108acead814d1de8b6262d6d8fb90af7e82f5a4d83788b6b48e39d01800a92f", size = 1066549, upload-time = "2026-05-28T22:27:06.832Z" }, - { url = "https://files.pythonhosted.org/packages/33/91/ff0b444f686718635348986bd73dfce42e947912417893de35de399b878b/dulwich-1.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:5e067b7feceb7034bc99e7c7143a704f1d97d4be7027d9a0aa5a83c0657ff091", size = 1079481, upload-time = "2026-05-28T22:27:08.33Z" }, - { url = "https://files.pythonhosted.org/packages/19/22/4f75770bbe5521cac61c4820ef46d4fbf8c2175d3519ba3d0378d4ba798e/dulwich-1.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:701a9ecf7a8a44f5e2459e46befa93530cf36a8b1ae3140aefc007db1d7d0207", size = 1396522, upload-time = "2026-05-28T22:27:09.997Z" }, - { url = "https://files.pythonhosted.org/packages/e5/b1/c07c347681c0cf6acd4b189bf6e8d6207c71a1347b7a1e865eb40faa46b9/dulwich-1.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f90d68bfa97c4ca71de7507984365aefe27b6d248cb28dc99644d0f3ae8c60b", size = 1334826, upload-time = "2026-05-28T22:27:11.582Z" }, - { url = "https://files.pythonhosted.org/packages/13/80/6818eb7ce492e18ab2efa92ab901d173b4b0b159e5681c1424f329600c40/dulwich-1.2.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00b54a1d56ddbacdd8eadd6d4787a51b3a05fefa30eadbf9165fd283a00b90ed", size = 1416616, upload-time = "2026-05-28T22:27:13.195Z" }, - { url = "https://files.pythonhosted.org/packages/14/a7/9790e60d19870f6554f7583722bb324c1355784316f20aeda1c0b5b1491a/dulwich-1.2.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d8f7ea8f47e38e5b0de3fab97e07e9c9161ffddc90b3964512cab2b7749df4e6", size = 1441354, upload-time = "2026-05-28T22:27:14.683Z" }, - { url = "https://files.pythonhosted.org/packages/91/44/0ea8a69c24aa1254ff5996d682eae2eab287d471b937dcdb26d9ea9720b4/dulwich-1.2.5-cp312-cp312-win32.whl", hash = "sha256:8929134acf4ff967203df7600b38535f9b5b590462067a7e30dbce01acb97af9", size = 1017058, upload-time = "2026-05-28T22:27:16.121Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/2fcddda7faec3bae52db7c64bfcb5dc756f597f33fae90e8d4e4b4d3b39b/dulwich-1.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:9693d2c9e226b2ea855c1dc3a87e2f4d972f7523fc0f7924e5997e9f4c23d97f", size = 1031731, upload-time = "2026-05-28T22:27:17.633Z" }, - { url = "https://files.pythonhosted.org/packages/07/4b/4a18a59ad230581cd0ef460e96001f90762e566dc2dfdba22aa358eb5a0e/dulwich-1.2.5-py3-none-any.whl", hash = "sha256:1679b376433a0fc7f36586afda1d4ed7427afa7a79d4bf17e5014474eea69fa4", size = 686745, upload-time = "2026-05-28T22:27:53.695Z" }, -] - -[[package]] -name = "dvc" -version = "3.67.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "celery" }, + { name = "attrs" }, + { name = "celery" }, { name = "colorama" }, { name = "configobj" }, { name = "distro" }, @@ -1584,16 +925,16 @@ wheels = [ [[package]] name = "dvc-studio-client" -version = "0.22.0" +version = "0.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dulwich" }, { name = "requests" }, { name = "voluptuous" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f4/52/f00bc978bfa313929221df1b6a1d82256b1c2727c55594dbbf9520f0adfd/dvc_studio_client-0.22.0.tar.gz", hash = "sha256:45d554a0386dd18bdfe17968e93f9b075563c888088b51bfa58713f64ed58ac8", size = 29432, upload-time = "2025-07-28T16:23:52.699Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/ee/c3c2901383633a2bd622a931948649a001a9836c803bb63580adeaa28299/dvc_studio_client-0.23.0.tar.gz", hash = "sha256:cd5582fcf9dc281a00cceb56b6dd6d7503052b0661285dac3315bbb5d3add240", size = 30229, upload-time = "2026-06-01T07:01:21.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/8b/42cb2c96555cf63b5c57c3b21f3901bb30a9ae963ecba86a8265b61eee7d/dvc_studio_client-0.22.0-py3-none-any.whl", hash = "sha256:99cb8874a1e5fc05de126a36a82b421f7af5c36d23c22024284733fc4d98029b", size = 16432, upload-time = "2025-07-28T16:23:51.256Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ff/d2abc593708901408c60a22dd430bc608bfda3496170e20aa77133948be9/dvc_studio_client-0.23.0-py3-none-any.whl", hash = "sha256:d0826e2232c0c8f3ea3dcd1348bf2a745aebd522ff4e8a37433c0790e96adba5", size = 16785, upload-time = "2026-06-01T07:01:19.914Z" }, ] [[package]] @@ -1621,93 +962,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/a8/365059bbcd4572cbc41de17fd5b682be5868b218c3c5479071865cab9078/entrypoints-0.4-py3-none-any.whl", hash = "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f", size = 5294, upload-time = "2022-02-02T21:30:26.024Z" }, ] -[[package]] -name = "executing" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, -] - -[[package]] -name = "fairlearn" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "narwhals" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "scikit-learn" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/07/f8b79188e1d148634f0aa88e2c503c9174c8ae26c5575747e832f989a837/fairlearn-0.14.0.tar.gz", hash = "sha256:dc9d12fc6d3f4c94f1bf32eb11032b73363c4a62c986485bcc87febe2e583ebd", size = 105832, upload-time = "2026-06-07T16:14:20.225Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/bd/776182bf82b0a3223773a43b0e7b9510afe6c3d32ed8dd5a27b30a89530b/fairlearn-0.14.0-py3-none-any.whl", hash = "sha256:2ffe37b71f5b6a8e140b5db75a7990da64fd04b46e7acfc81654f92497ed2ece", size = 135513, upload-time = "2026-06-07T16:14:18.888Z" }, -] - -[[package]] -name = "fastapi" -version = "0.136.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/45/c130091c2dfa061bbfe3150f2a5091ef1adf149f2a8d2ae769ecaf6e99a2/fastapi-0.136.1.tar.gz", hash = "sha256:7af665ad7acfa0a3baf8983d393b6b471b9da10ede59c60045f49fbc89a0fa7f", size = 397448, upload-time = "2026-04-23T16:49:44.046Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/ff/2e4eca3ade2c22fe1dea7043b8ee9dabe47753349eb1b56a202de8af6349/fastapi-0.136.1-py3-none-any.whl", hash = "sha256:a6e9d7eeada96c93a4d69cb03836b44fa34e2854accb7244a1ece36cd4781c3f", size = 117683, upload-time = "2026-04-23T16:49:42.437Z" }, -] - -[[package]] -name = "fastjsonschema" -version = "2.21.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, -] - [[package]] name = "filelock" -version = "3.29.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, -] - -[[package]] -name = "flask" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blinker" }, - { name = "click" }, - { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "markupsafe" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, -] - -[[package]] -name = "flask-cors" -version = "6.0.2" +version = "3.30.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flask" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/74/0fc0fa68d62f21daef41017dafab19ef4b36551521260987eb3a5394c7ba/flask_cors-6.0.2.tar.gz", hash = "sha256:6e118f3698249ae33e429760db98ce032a8bf9913638d085ca0f4c5534ad2423", size = 13472, upload-time = "2025-12-12T20:31:42.861Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2b/8b6480a70a647035334a604d0931926de4b5cd1f57835d45ad5eed2b1a1e/filelock-3.30.0.tar.gz", hash = "sha256:1774e682dbe443bd60f9609162fc596e2c80dc84ffc2957068953406d0520090", size = 174927, upload-time = "2026-07-16T03:53:58.152Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/af/72ad54402e599152de6d067324c46fe6a4f531c7c65baf7e96c63db55eaf/flask_cors-6.0.2-py3-none-any.whl", hash = "sha256:e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a", size = 13257, upload-time = "2025-12-12T20:31:41.3Z" }, + { url = "https://files.pythonhosted.org/packages/52/af/9b01bcf5c91e81899bb890b87bd9077732a9b3365c098e67fe77958c39ed/filelock-3.30.0-py3-none-any.whl", hash = "sha256:40632998f0772e64183bb819f086a1b9def6be1090cf1dcb9d45f46806ef279b", size = 93131, upload-time = "2026-07-16T03:53:56.727Z" }, ] [[package]] @@ -1734,36 +995,27 @@ wheels = [ [[package]] name = "fonttools" -version = "4.62.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/08/7012b00a9a5874311b639c3920270c36ee0c445b69d9989a85e5c92ebcb0/fonttools-4.62.1.tar.gz", hash = "sha256:e54c75fd6041f1122476776880f7c3c3295ffa31962dc6ebe2543c00dca58b5d", size = 3580737, upload-time = "2026-03-13T13:54:25.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/23ff32561ec8d45a4d48578b4d241369d9270dc50926c017570e60893701/fonttools-4.62.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:40975849bac44fb0b9253d77420c6d8b523ac4dcdcefeff6e4d706838a5b80f7", size = 2871039, upload-time = "2026-03-13T13:52:33.127Z" }, - { url = "https://files.pythonhosted.org/packages/24/7f/66d3f8a9338a9b67fe6e1739f47e1cd5cee78bd3bc1206ef9b0b982289a5/fonttools-4.62.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9dde91633f77fa576879a0c76b1d89de373cae751a98ddf0109d54e173b40f14", size = 2416346, upload-time = "2026-03-13T13:52:35.676Z" }, - { url = "https://files.pythonhosted.org/packages/aa/53/5276ceba7bff95da7793a07c5284e1da901cf00341ce5e2f3273056c0cca/fonttools-4.62.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6acb4109f8bee00fec985c8c7afb02299e35e9c94b57287f3ea542f28bd0b0a7", size = 5100897, upload-time = "2026-03-13T13:52:38.102Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a1/40a5c4d8e28b0851d53a8eeeb46fbd73c325a2a9a165f290a5ed90e6c597/fonttools-4.62.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c5c25671ce8805e0d080e2ffdeca7f1e86778c5cbfbeae86d7f866d8830517b", size = 5071078, upload-time = "2026-03-13T13:52:41.305Z" }, - { url = "https://files.pythonhosted.org/packages/e3/be/d378fca4c65ea1956fee6d90ace6e861776809cbbc5af22388a090c3c092/fonttools-4.62.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a5d8825e1140f04e6c99bb7d37a9e31c172f3bc208afbe02175339e699c710e1", size = 5076908, upload-time = "2026-03-13T13:52:44.122Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d9/ae6a1d0693a4185a84605679c8a1f719a55df87b9c6e8e817bfdd9ef5936/fonttools-4.62.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:268abb1cb221e66c014acc234e872b7870d8b5d4657a83a8f4205094c32d2416", size = 5202275, upload-time = "2026-03-13T13:52:46.591Z" }, - { url = "https://files.pythonhosted.org/packages/54/6c/af95d9c4efb15cabff22642b608342f2bd67137eea6107202d91b5b03184/fonttools-4.62.1-cp311-cp311-win32.whl", hash = "sha256:942b03094d7edbb99bdf1ae7e9090898cad7bf9030b3d21f33d7072dbcb51a53", size = 2293075, upload-time = "2026-03-13T13:52:48.711Z" }, - { url = "https://files.pythonhosted.org/packages/d3/97/bf54c5b3f2be34e1f143e6db838dfdc54f2ffa3e68c738934c82f3b2a08d/fonttools-4.62.1-cp311-cp311-win_amd64.whl", hash = "sha256:e8514f4924375f77084e81467e63238b095abda5107620f49421c368a6017ed2", size = 2344593, upload-time = "2026-03-13T13:52:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/47/d4/dbacced3953544b9a93088cc10ef2b596d348c983d5c67a404fa41ec51ba/fonttools-4.62.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:90365821debbd7db678809c7491ca4acd1e0779b9624cdc6ddaf1f31992bf974", size = 2870219, upload-time = "2026-03-13T13:52:53.664Z" }, - { url = "https://files.pythonhosted.org/packages/66/9e/a769c8e99b81e5a87ab7e5e7236684de4e96246aae17274e5347d11ebd78/fonttools-4.62.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12859ff0b47dd20f110804c3e0d0970f7b832f561630cd879969011541a464a9", size = 2414891, upload-time = "2026-03-13T13:52:56.493Z" }, - { url = "https://files.pythonhosted.org/packages/69/64/f19a9e3911968c37e1e620e14dfc5778299e1474f72f4e57c5ec771d9489/fonttools-4.62.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c125ffa00c3d9003cdaaf7f2c79e6e535628093e14b5de1dccb08859b680936", size = 5033197, upload-time = "2026-03-13T13:52:59.179Z" }, - { url = "https://files.pythonhosted.org/packages/9b/8a/99c8b3c3888c5c474c08dbfd7c8899786de9604b727fcefb055b42c84bba/fonttools-4.62.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:149f7d84afca659d1a97e39a4778794a2f83bf344c5ee5134e09995086cc2392", size = 4988768, upload-time = "2026-03-13T13:53:02.761Z" }, - { url = "https://files.pythonhosted.org/packages/d1/c6/0f904540d3e6ab463c1243a0d803504826a11604c72dd58c2949796a1762/fonttools-4.62.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0aa72c43a601cfa9273bb1ae0518f1acadc01ee181a6fc60cd758d7fdadffc04", size = 4971512, upload-time = "2026-03-13T13:53:05.678Z" }, - { url = "https://files.pythonhosted.org/packages/29/0b/5cbef6588dc9bd6b5c9ad6a4d5a8ca384d0cea089da31711bbeb4f9654a6/fonttools-4.62.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:19177c8d96c7c36359266e571c5173bcee9157b59cfc8cb0153c5673dc5a3a7d", size = 5122723, upload-time = "2026-03-13T13:53:08.662Z" }, - { url = "https://files.pythonhosted.org/packages/4a/47/b3a5342d381595ef439adec67848bed561ab7fdb1019fa522e82101b7d9c/fonttools-4.62.1-cp312-cp312-win32.whl", hash = "sha256:a24decd24d60744ee8b4679d38e88b8303d86772053afc29b19d23bb8207803c", size = 2281278, upload-time = "2026-03-13T13:53:10.998Z" }, - { url = "https://files.pythonhosted.org/packages/28/b1/0c2ab56a16f409c6c8a68816e6af707827ad5d629634691ff60a52879792/fonttools-4.62.1-cp312-cp312-win_amd64.whl", hash = "sha256:9e7863e10b3de72376280b515d35b14f5eeed639d1aa7824f4cf06779ec65e42", size = 2331414, upload-time = "2026-03-13T13:53:13.992Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ba/56147c165442cc5ba7e82ecf301c9a68353cede498185869e6e02b4c264f/fonttools-4.62.1-py3-none-any.whl", hash = "sha256:7487782e2113861f4ddcc07c3436450659e3caa5e470b27dc2177cade2d8e7fd", size = 1152647, upload-time = "2026-03-13T13:54:22.735Z" }, -] - -[[package]] -name = "fqdn" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/3e/a80a8c077fd798951169626cde3e239adeba7dab75deb3555716415bd9b0/fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", size = 6015, upload-time = "2021-03-11T07:16:29.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, +version = "4.63.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/2b/a7f1545bdf5da69c4bda0cea2a5781f0ad2a6623e0277267672db43c5fe6/fonttools-4.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b8ae05d9eacf6081414d759c0a352769ac28ce31280d6bb8e77b03f9e3c449f", size = 2881793, upload-time = "2026-05-14T12:02:56.645Z" }, + { url = "https://files.pythonhosted.org/packages/49/50/965308c703f085f225db2886813b27e015b8b3438c350b22dd65b52c2a2c/fonttools-4.63.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79cdc9f567aec74a72918fd060283911406750cbc9fd28c1316023deb6ce31a9", size = 2428130, upload-time = "2026-05-14T12:02:58.891Z" }, + { url = "https://files.pythonhosted.org/packages/d8/38/6937fbd7f2dc3a6b48725851bc2c15ec949b9af14d9bbcb5fe83cdf9bdf9/fonttools-4.63.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c14b4fd138c4bafcca294765c547914e1aa431ae1ca94ab99d8db08c958bd3b", size = 5111952, upload-time = "2026-05-14T12:03:01.263Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/a81f20050a3115b57d62c8e781446949512eac36690dc384ccea65ff4cc1/fonttools-4.63.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76ac49f929aecaf82d83250b8347e099d7aecba0f4726c1d9b6df3b8bb5fe18", size = 5082308, upload-time = "2026-05-14T12:03:03.211Z" }, + { url = "https://files.pythonhosted.org/packages/67/00/cdd9d4944ca6ae280d01e69cc37bde3bf663630b837a6fc6d2cd65d80e0e/fonttools-4.63.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dcf076a4474fe0d7367e5bbf5b052c7284fa1feca729c04176ce513521afd8a0", size = 5087932, upload-time = "2026-05-14T12:03:05.147Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f1/0aa0dbea778c75adbef223c42019fd47d22262b905974d62d829545d485f/fonttools-4.63.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7dd683fef0663e9f0f45cf541d788d24caa3ec9db50796b588e1757d8b3bc007", size = 5213271, upload-time = "2026-05-14T12:03:07.238Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/253e4056e1f0e67b9390125a154b73b5eb73ad521bece95c004858fdeec2/fonttools-4.63.0-cp311-cp311-win32.whl", hash = "sha256:afefc1ed0a59785a7fb06ea7e1678e849c193e1e387db783579bc7b3056fcfcb", size = 2304473, upload-time = "2026-05-14T12:03:09.271Z" }, + { url = "https://files.pythonhosted.org/packages/08/60/defa5e69641db890a63be281f41345f4c33b157824eaf0b9fad3e08b0dcb/fonttools-4.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:063e08bd17bd5a90127a14123de0d6a952dbc847695fd98b63c043d58057f90c", size = 2356389, upload-time = "2026-05-14T12:03:11.53Z" }, + { url = "https://files.pythonhosted.org/packages/08/ef/b3c6b9b5be2f82416d73fe2ed2e96e2793cd80e7510bd6a17ca79cdd88ec/fonttools-4.63.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:37dd23e621e3b0aef1baa70a303b80aaf38449632cfc8fd2a55fb285bbccfc02", size = 2881131, upload-time = "2026-05-14T12:03:13.386Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/c815bea63117fa63e4e1c01f8a1110d2112fa003f838e6467094ec2432ce/fonttools-4.63.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a9faff9e0c1f76f9fd55899d2ce785832efebab37eb8ae13995853aef178bef0", size = 2426704, upload-time = "2026-05-14T12:03:15.801Z" }, + { url = "https://files.pythonhosted.org/packages/44/04/0b91d8e916e92ad1fac9e4624760baf0fd5ff2ead614c2f68fb21373f03f/fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af", size = 5044298, upload-time = "2026-05-14T12:03:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8", size = 4999800, upload-time = "2026-05-14T12:03:20.161Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6d/67fe16c48d7ce050979b33f47e0d28a318f02da030602e944c34f7a16ef3/fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b", size = 4982666, upload-time = "2026-05-14T12:03:22.87Z" }, + { url = "https://files.pythonhosted.org/packages/f2/00/3bbab338c07c71fa56269953845e92c951a61457bbbb0f1022551ea266d9/fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78", size = 5133598, upload-time = "2026-05-14T12:03:25.168Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/aa27c7f98db5b064883dadcc5283947e81e034de42e22a33675878d98b54/fonttools-4.63.0-cp312-cp312-win32.whl", hash = "sha256:af2fd1664d00a397d75f806985ddb36282091c2131a73a6485c23b4a34722263", size = 2292575, upload-time = "2026-05-14T12:03:27.496Z" }, + { url = "https://files.pythonhosted.org/packages/87/36/cccb9bc2a6ab63d1b2980374f0dca72ce95ae267c9b4cfe77455bb70d0d4/fonttools-4.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:59ac449f8cca9b4ffa08d2e7bbadad87ce710d69d1eda5c3c1ce579baa987272", size = 2343211, upload-time = "2026-05-14T12:03:30.057Z" }, + { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, ] [[package]] @@ -1809,11 +1061,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.4.0" +version = "2026.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] [package.optional-dependencies] @@ -1847,265 +1099,141 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.52" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/fd/df0bafa4eb5ea2f51e1adee9f7a94c8e62c5d180e65117045dfca3439c8a/gitpython-3.1.52.tar.gz", hash = "sha256:de0a8ad86274c6e75ae8b37dd055ba68f19818c813108642263227b20775b48e", size = 223726, upload-time = "2026-07-16T03:15:59.599Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, + { url = "https://files.pythonhosted.org/packages/8d/90/04dff7c1e176bb1c3011ef1647393d368790da710d8dde1cdcfad301f45a/gitpython-3.1.52-py3-none-any.whl", hash = "sha256:79a36ee1f83523214a3f72d56cf1c4e490d577dc61af77e43dfe5862bd9da01a", size = 215366, upload-time = "2026-07-16T03:15:58.239Z" }, ] [[package]] -name = "google-auth" -version = "2.52.0" +name = "grandalf" +version = "0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, + { name = "pyparsing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d4/f8/80d2493cbedece1c623dc3e3cb1883300871af0dcdae254409522985ac23/google_auth-2.52.0.tar.gz", hash = "sha256:01f30e1a9e3638698d89464f5e603ce29d18e1c0e63ec31ac570aba4e164aaf5", size = 335027, upload-time = "2026-05-07T19:45:24.033Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/0e/4ac934b416857969f9135dec17ac80660634327e003a870835dd1f382659/grandalf-0.8.tar.gz", hash = "sha256:2813f7aab87f0d20f334a3162ccfbcbf085977134a17a5b516940a93a77ea974", size = 38128, upload-time = "2023-01-26T07:37:06.668Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/fc/2cdc74252746f547f81ff3f02d4d4234a3f411b5de5b61af97e633a060b9/google_auth-2.52.0-py3-none-any.whl", hash = "sha256:aee92803ba0ff93a70a3b8a35c7b4797837751cd6380b63ff38372b98f3ed627", size = 245614, upload-time = "2026-05-07T19:45:21.914Z" }, + { url = "https://files.pythonhosted.org/packages/61/30/44c7eb0a952478dbb5f2f67df806686d6a7e4b19f6204e091c4f49dc7c69/grandalf-0.8-py3-none-any.whl", hash = "sha256:793ca254442f4a79252ea9ff1ab998e852c1e071b863593e5383afee906b4185", size = 41802, upload-time = "2023-01-10T15:16:19.753Z" }, ] [[package]] -name = "gql" -version = "4.0.0" +name = "graphviz" +version = "0.21" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "backoff" }, - { name = "graphql-core" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/9f/cf224a88ed71eb223b7aa0b9ff0aa10d7ecc9a4acdca2279eb046c26d5dc/gql-4.0.0.tar.gz", hash = "sha256:f22980844eb6a7c0266ffc70f111b9c7e7c7c13da38c3b439afc7eab3d7c9c8e", size = 215644, upload-time = "2025-08-17T14:32:35.397Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/94/30bbd09e8d45339fa77a48f5778d74d47e9242c11b3cd1093b3d994770a5/gql-4.0.0-py3-none-any.whl", hash = "sha256:f3beed7c531218eb24d97cb7df031b4a84fdb462f4a2beb86e2633d395937479", size = 89900, upload-time = "2025-08-17T14:32:34.029Z" }, -] - -[package.optional-dependencies] -requests = [ - { name = "requests" }, - { name = "requests-toolbelt" }, + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, ] [[package]] -name = "grandalf" -version = "0.8" +name = "gto" +version = "1.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyparsing" }, + { name = "click" }, + { name = "entrypoints" }, + { name = "funcy" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "rich" }, + { name = "ruamel-yaml" }, + { name = "scmrepo" }, + { name = "semver" }, + { name = "tabulate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/0e/4ac934b416857969f9135dec17ac80660634327e003a870835dd1f382659/grandalf-0.8.tar.gz", hash = "sha256:2813f7aab87f0d20f334a3162ccfbcbf085977134a17a5b516940a93a77ea974", size = 38128, upload-time = "2023-01-26T07:37:06.668Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/9c/cf74e391f76ff2afd7aee9bdd0f1b191c0522ba8190439bfa3b2672f222a/gto-1.10.1.tar.gz", hash = "sha256:e2a008650e7be42b7e3dd65e2eef360d466553e3b41d8f37dae38328d84fd224", size = 60033, upload-time = "2026-07-09T22:08:26.369Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/30/44c7eb0a952478dbb5f2f67df806686d6a7e4b19f6204e091c4f49dc7c69/grandalf-0.8-py3-none-any.whl", hash = "sha256:793ca254442f4a79252ea9ff1ab998e852c1e071b863593e5383afee906b4185", size = 41802, upload-time = "2023-01-10T15:16:19.753Z" }, + { url = "https://files.pythonhosted.org/packages/88/af/23fba84f3e03fcf58bd73e1d28ef9db7b46c5b48df95fae4b5f658db7993/gto-1.10.1-py3-none-any.whl", hash = "sha256:f357f7dee2cf91552408545e17f53a4ea54d0d16e676e35b92edf1291414d06d", size = 45713, upload-time = "2026-07-09T22:08:24.848Z" }, ] [[package]] -name = "graphene" -version = "3.4.3" +name = "highspy" +version = "1.15.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "graphql-core" }, - { name = "graphql-relay" }, - { name = "python-dateutil" }, - { name = "typing-extensions" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cc/f6/bf62ff950c317ed03e77f3f6ddd7e34aaa98fe89d79ebd660c55343d8054/graphene-3.4.3.tar.gz", hash = "sha256:2a3786948ce75fe7e078443d37f609cbe5bb36ad8d6b828740ad3b95ed1a0aaa", size = 44739, upload-time = "2024-11-09T20:44:25.757Z" } +sdist = { url = "https://files.pythonhosted.org/packages/87/02/c6b658f79911fee921721da728b9ab8f5e19ff06121fff36f90f77127f4d/highspy-1.15.1.tar.gz", hash = "sha256:20ed2fbf1cb64bf3044ee6632364b7e2653d93e6901e2b19fd3d5df10702e8c5", size = 1703256, upload-time = "2026-07-02T12:03:25.009Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/e0/61d8e98007182e6b2aca7cf65904721fb2e4bce0192272ab9cb6f69d8812/graphene-3.4.3-py2.py3-none-any.whl", hash = "sha256:820db6289754c181007a150db1f7fff544b94142b556d12e3ebc777a7bf36c71", size = 114894, upload-time = "2024-11-09T20:44:23.851Z" }, + { url = "https://files.pythonhosted.org/packages/5d/69/74e9614a6e49f5e07fbb91a5cebad79bd31903888ca6b4f1c75dcf01df3b/highspy-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:45eb9f022f9083ef2e56d66f972d5fd40e6634f4497194b1f3f215ca0e8ea958", size = 4871151, upload-time = "2026-07-02T12:01:42.702Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5d/815b8f0488fda02e765b85e7a1ed2bf26b7d705fb1240ca842c4493d9414/highspy-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4b4c7e7af8d7927ed77836e9b869cbae55d6a74b85bb90d04776440b5e14c32b", size = 4468478, upload-time = "2026-07-02T12:01:44.995Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ea/def24ab38ff3ea983eb197dcbf01ec9ad82e2aa3380b72f01792486865e1/highspy-1.15.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:070c1ce9238b9e8b4c273253647ab0dbafc1839c195a52c7ef1eeb7ef6976f05", size = 4634480, upload-time = "2026-07-02T12:01:46.822Z" }, + { url = "https://files.pythonhosted.org/packages/ed/bb/0588b8137df2a0ae55a9a8ad40eff40561b60d6f76dbffd5ab4f4f6095f0/highspy-1.15.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a24329c328942b37a6a318ecf163d07dd387974f071b98b4498725eaea80f06f", size = 5035498, upload-time = "2026-07-02T12:01:48.717Z" }, + { url = "https://files.pythonhosted.org/packages/f2/50/5196e807cec6847b2bee8ceefb080b8175ea81aaa673249678398cf3181c/highspy-1.15.1-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:138506088c7f6106cbb58d1cd0ef14793dfb47477fd83a7ae0db5b104d1cf969", size = 5857714, upload-time = "2026-07-02T12:01:51.04Z" }, + { url = "https://files.pythonhosted.org/packages/32/25/80c96a11bcf3bdf4ffaa7b1e26629b6da340703b2bbc4e0349d40d7a8aaa/highspy-1.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00e1c13912501e96893136a1805b56b74cb4868fa04c1c2eacc5c0454304e08e", size = 6191044, upload-time = "2026-07-02T12:01:52.856Z" }, + { url = "https://files.pythonhosted.org/packages/11/10/27a7b87dbf56ab88d223fb9f1b6917c9df259c09980460ff56731c2406ef/highspy-1.15.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b5be1c777d0b57b6dc26e1d9754923e642a17c6313bcdf5186473644b214f0b", size = 7234613, upload-time = "2026-07-02T12:01:54.936Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1e/82159a50b8a17daf2e94453a8e0af3e381943223727fd9db6d4bebc1533a/highspy-1.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5de2dddc554442f3572bb4a36116278bee79568fbd726a697251d2606b79a5a1", size = 6632799, upload-time = "2026-07-02T12:01:57.967Z" }, + { url = "https://files.pythonhosted.org/packages/3c/69/59d4303d38d6c48cc6f79c12b11f756b381e8e5eed5c0733d8c82317240f/highspy-1.15.1-cp311-cp311-win32.whl", hash = "sha256:605d3204e41a465f9ce2f254571a90e8781605451a5e6a548f6b4be8988afb4f", size = 2304839, upload-time = "2026-07-02T12:01:59.752Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/6714276be39f1f0f7c2795fb466c39640bbcc541e151342f05f61d2f24a5/highspy-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:4715fcfbcff50fdbcc288499116f7e5722a9f9d2647087d54317febb94ec2b32", size = 2709129, upload-time = "2026-07-02T12:02:01.722Z" }, + { url = "https://files.pythonhosted.org/packages/de/59/b79a7b1711ddfcca36674ddb41759e98eb1797f4a94513e7dd215e32e94d/highspy-1.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a781dc8432568ea990fcdcc8d6e4365e67aa4848ca1f99275db096645b27cae3", size = 4878738, upload-time = "2026-07-02T12:02:03.82Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e4/ae08124f71187628471a177e6db1ed2c1c45e9dceadc45f7111dfd7c2254/highspy-1.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9499d631edeb9642fc08dee59ca6c5815be1764c13a336c58ab7ba063011aa24", size = 4473938, upload-time = "2026-07-02T12:02:05.754Z" }, + { url = "https://files.pythonhosted.org/packages/ff/7f/185b8c9579a9e4ef88eda45d1fdaf8d23a3a640f73c403a7b29fc0f0c4be/highspy-1.15.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef048fa722cdeb80062d271b8ba211cd6650ab73419762d80da7642bbd4a8420", size = 4636755, upload-time = "2026-07-02T12:02:07.996Z" }, + { url = "https://files.pythonhosted.org/packages/82/6b/18bec60d8585df860b8d33d310e99e7893eaabe3c8e9ebfa7e387ba9d2a4/highspy-1.15.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9730647160a6481426729f46d9989a0507d05f3cf96f9fb180f4ab9891bea67b", size = 5034168, upload-time = "2026-07-02T12:02:09.89Z" }, + { url = "https://files.pythonhosted.org/packages/d4/51/e43f06e64e994ccb41a336ff78802c0dae63aed46c17acd52167b5ca3d76/highspy-1.15.1-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6a6a2f21ee31a9205a928fbbc3f8c054893c1aec34f6a7c56588317e2800e673", size = 5861937, upload-time = "2026-07-02T12:02:11.801Z" }, + { url = "https://files.pythonhosted.org/packages/d4/2a/5501a23cac55926e4b0554352b4285734b417dbec385c593f2ae405ea637/highspy-1.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9a6760962b3e813814dc5e88301890d7cce975de5ce97cc3aed589cfdd461811", size = 6192004, upload-time = "2026-07-02T12:02:14.544Z" }, + { url = "https://files.pythonhosted.org/packages/94/08/fb7d30ea0e6c83fb943b16bf31951ba13a5be01a638ec13962a477009b91/highspy-1.15.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:787c92d5ff274256ba8848ab174cfc65d5af696f51bffe87423c85b2ea25c3fe", size = 7233098, upload-time = "2026-07-02T12:02:16.609Z" }, + { url = "https://files.pythonhosted.org/packages/23/77/9a07df7181834cfb61dafa5594e5eedc78369797c6487806bd3221d20667/highspy-1.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dd9ee8e139e7260ec1306a48e30f1bd7937d9cfb8cb201d25da10e1099e5129b", size = 6636717, upload-time = "2026-07-02T12:02:18.72Z" }, + { url = "https://files.pythonhosted.org/packages/9a/25/5083d8e3d5cf5ff5edf5bcc03e3f693630ab59142c9d0a0bcbb2d315c50e/highspy-1.15.1-cp312-cp312-win32.whl", hash = "sha256:01c6585e83938ecf4139248b074b2ee736816d63716a20dc608b1d2fc9637b66", size = 2306753, upload-time = "2026-07-02T12:02:20.837Z" }, + { url = "https://files.pythonhosted.org/packages/d4/01/05521ca6b38e34e68d707888c378d3bcac34e62715b739e7c0c9b9887993/highspy-1.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:8c548165270608a40147a7ea6d985fd62a65fabf0f075b3c0c59ea910b724223", size = 2711114, upload-time = "2026-07-02T12:02:22.621Z" }, ] [[package]] -name = "graphql-core" -version = "3.2.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/c5/36aa96205c3ecbb3d34c7c24189e4553c7ca2ebc7e1dd07432339b980272/graphql_core-3.2.8.tar.gz", hash = "sha256:015457da5d996c924ddf57a43f4e959b0b94fb695b85ed4c29446e508ed65cf3", size = 513181, upload-time = "2026-03-05T19:55:37.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/41/cb887d9afc5dabd78feefe6ccbaf83ff423c206a7a1b7aeeac05120b2125/graphql_core-3.2.8-py3-none-any.whl", hash = "sha256:cbee07bee1b3ed5e531723685369039f32ff815ef60166686e0162f540f1520c", size = 207349, upload-time = "2026-03-05T19:55:35.911Z" }, -] - -[[package]] -name = "graphql-relay" -version = "3.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "graphql-core" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d1/13/98fbf8d67552f102488ffc16c6f559ce71ea15f6294728d33928ab5ff14d/graphql-relay-3.2.0.tar.gz", hash = "sha256:1ff1c51298356e481a0be009ccdff249832ce53f30559c1338f22a0e0d17250c", size = 50027, upload-time = "2022-04-16T11:03:45.447Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/16/a4cf06adbc711bd364a73ce043b0b08d8fa5aae3df11b6ee4248bcdad2e0/graphql_relay-3.2.0-py3-none-any.whl", hash = "sha256:c9b22bd28b170ba1fe674c74384a8ff30a76c8e26f88ac3aa1584dd3179953e5", size = 16940, upload-time = "2022-04-16T11:03:43.895Z" }, -] - -[[package]] -name = "graphviz" -version = "0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, -] - -[[package]] -name = "greenlet" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/dbf99fb14bfeb88c28f16729215478c0e265cacd6dc22270c8f31bb6892f/greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4", size = 196995, upload-time = "2026-04-27T13:37:15.544Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/0f/a91f143f356523ff682309732b175765a9bc2836fd7c081c2c67fedc1ad4/greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082", size = 284726, upload-time = "2026-04-27T12:20:51.402Z" }, - { url = "https://files.pythonhosted.org/packages/95/82/800646c7ffc5dbabd75ddd2f6b519bb898c0c9c969e5d0473bfe5d20bcce/greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3", size = 604264, upload-time = "2026-04-27T12:52:39.494Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/354867c0bba812fc33b15bc55aedafedd0aee3c7dd91dfca22444157dc0c/greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c", size = 616099, upload-time = "2026-04-27T12:59:39.623Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b0/815bece7399e01cadb69014219eebd0042339875c59a59b0820a46ece356/greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662", size = 615198, upload-time = "2026-04-27T12:25:25.928Z" }, - { url = "https://files.pythonhosted.org/packages/10/80/3b2c0a895d6698f6ddb31b07942ebfa982f3e30888bc5546a5b5990de8b2/greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b", size = 1574927, upload-time = "2026-04-27T12:53:25.81Z" }, - { url = "https://files.pythonhosted.org/packages/44/0e/f354af514a4c61454dbc68e44d47544a5a4d6317e30b77ddfa3a09f4c5f3/greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4", size = 1642683, upload-time = "2026-04-27T12:25:23.9Z" }, - { url = "https://files.pythonhosted.org/packages/fa/6a/87f38255201e993a1915265ebb80cd7c2c78b04a45744995abbf6b259fd8/greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8", size = 238115, upload-time = "2026-04-27T12:21:48.845Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f8/450fe3c5938fa737ea4d22699772e6e34e8e24431a47bf4e8a1ceed4a98e/greenlet-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:6c18dfb59c70f5a94acd271c72e90128c3c776e41e5f07767908c8c1b74ad339", size = 235017, upload-time = "2026-04-27T12:22:26.768Z" }, - { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" }, - { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" }, - { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" }, - { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" }, - { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" }, - { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" }, - { url = "https://files.pythonhosted.org/packages/cb/cb/baa584cb00532126ffe12d9787db0a60c5a4f55c27bfe2666df5d4c30a32/greenlet-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:83ed9f27f1680b50e89f40f6df348a290ea234b249a4003d366663a12eab94f2", size = 235615, upload-time = "2026-04-27T12:21:38.57Z" }, -] - -[[package]] -name = "gto" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "entrypoints" }, - { name = "funcy" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "rich" }, - { name = "ruamel-yaml" }, - { name = "scmrepo" }, - { name = "semver" }, - { name = "tabulate" }, - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/06/d2ec91a6c1e6b1a55c419e8599df7ac3430323a1bb1e5c01a1f83f8ecb64/gto-1.9.0.tar.gz", hash = "sha256:3beb5c652a98585ad083dbb6879a580ffe926271661d9b7a50e428cd591005ea", size = 58999, upload-time = "2025-10-08T17:05:28.568Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/b3/6086ab9cfd4a27517a1269e8b7c48621beb79ccc0affd2485b9747976bfe/gto-1.9.0-py3-none-any.whl", hash = "sha256:e94371a67c25256f973722c5891e551ca3cd8cc25864dcf468f2b16e6bcca6b8", size = 45038, upload-time = "2025-10-08T17:05:26.947Z" }, -] - -[[package]] -name = "gunicorn" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/f4/e78fa054248fab913e2eab0332c6c2cb07421fca1ce56d8fe43b6aef57a4/gunicorn-25.3.0.tar.gz", hash = "sha256:f74e1b2f9f76f6cd1ca01198968bd2dd65830edc24b6e8e4d78de8320e2fe889", size = 634883, upload-time = "2026-03-27T00:00:26.092Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/c8/8aaf447698c4d59aa853fd318eed300b5c9e44459f242ab8ead6c9c09792/gunicorn-25.3.0-py3-none-any.whl", hash = "sha256:cacea387dab08cd6776501621c295a904fe8e3b7aae9a1a3cbb26f4e7ed54660", size = 208403, upload-time = "2026-03-27T00:00:27.386Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "highspy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/66/e74b1a805f65c52666e3b54cfc1ba783e745c2c8a7abaae9e7ef2d9e7270/highspy-1.14.0.tar.gz", hash = "sha256:b09cb5e3179a25fc615b8b0941130b0f71e19372c119f3dd620d63b54cd3ca4c", size = 1654913, upload-time = "2026-04-06T15:53:31.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/4e/c813156aa513eb3344b333c1424373cebf1f5843868b2ba5c49c64beecde/highspy-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:69558127aabad8b5718a58009dc3d36618c3aa5aa5e733206c32ce396189a132", size = 2310870, upload-time = "2026-04-06T15:51:51.586Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6c/d4baa83e8745d729764bf960b51828e9c99c90b4f5cd99e65b59fbf2b6f9/highspy-1.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3699fc8072a70664d5bbaf1c9239f8e2e8700c5090f57486f2ce4567f9b2b6aa", size = 2116701, upload-time = "2026-04-06T15:51:53.175Z" }, - { url = "https://files.pythonhosted.org/packages/cd/64/9dbafa1f3f9ec9293c4038d64b4a49a7a577e1ffcc5c48cf861849d2cff0/highspy-1.14.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4dd237b94494b14693edbebd05a0068fa02ae36b2629e6353be42bc2b491c1f0", size = 2404995, upload-time = "2026-04-06T15:51:54.664Z" }, - { url = "https://files.pythonhosted.org/packages/94/d6/d73cfcce4d3863d9839174a42fd976d84bb7781c132bf0bccdc74e83d9a7/highspy-1.14.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:526c54ff6e0384abcc495cda08fb2f98156544031afb9ac8bb02f20968d743ba", size = 2629054, upload-time = "2026-04-06T15:51:56.157Z" }, - { url = "https://files.pythonhosted.org/packages/71/c2/5ec46d5381815b849f25f4327af187d70325aa693bccdad960228c98ebb1/highspy-1.14.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:60265c69fb9d199526b8190f3bd42bc241896a335c299570d1dfc9d6e97f8895", size = 2787400, upload-time = "2026-04-06T15:51:57.682Z" }, - { url = "https://files.pythonhosted.org/packages/25/f0/1e89d849701388886d39ffda26f64bdc835d63f26aa4b5d865067d56fae1/highspy-1.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:895b3450ece50c1cdcd348aa5c6efd6ee823598f3a21c23677d57a06e3c6f28a", size = 3465733, upload-time = "2026-04-06T15:51:59.9Z" }, - { url = "https://files.pythonhosted.org/packages/fd/31/b3477f7ca17526167e5eff9d194ba8ea3eca0f04e98a248b26cc6612aa8f/highspy-1.14.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b50202f02c7db95163b0ad92160167541dad263b19327164dcf0f827cf9f69b3", size = 4044165, upload-time = "2026-04-06T15:52:01.482Z" }, - { url = "https://files.pythonhosted.org/packages/5d/85/67bead3e385de8f433572809b3cab6d7694c1a606edac2fbf94309b746d4/highspy-1.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e12591c1d495ebb5f6d8dc50e99ed48f338dfb59741929cd68c2c2b40d97a58f", size = 3707245, upload-time = "2026-04-06T15:52:03.05Z" }, - { url = "https://files.pythonhosted.org/packages/08/1c/0f61f66855a39e22f1f8e7d1ab3632b2a27cc1f42ee628fd9c317a5b4616/highspy-1.14.0-cp311-cp311-win32.whl", hash = "sha256:9ba82456280ef72cde8e45ecf6bdb2a244c56d80f7e44bb2e5ef7a9abf21f4d9", size = 1953948, upload-time = "2026-04-06T15:52:04.549Z" }, - { url = "https://files.pythonhosted.org/packages/f8/bd/4eaa775022d55519101a51a2b1ad5c46cfa8c725a400033943a79138001f/highspy-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:e726092e35237dccdd8093f8c91be195a5826e48aab349d6e7856e32c0e87b41", size = 2320210, upload-time = "2026-04-06T15:52:06.417Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3d/83ee11de10ff6499efdb6edbb4586b472a4e9f982c0f6c5d3faa670bf1c3/highspy-1.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a8d0339887d65f5ef20c59be529af33115197d47f775ee21fca911a87d30f92", size = 2311831, upload-time = "2026-04-06T15:52:08.042Z" }, - { url = "https://files.pythonhosted.org/packages/b6/74/51cfca0c382886e302c4e6b9a50f9b160d214a88b4bc5937f5f8e2452dd9/highspy-1.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f61406a287b19680ece9a8c0bc3926e31d26ad2b4df8ba49f22666ce762fb7d", size = 2122237, upload-time = "2026-04-06T15:52:09.587Z" }, - { url = "https://files.pythonhosted.org/packages/78/69/a60f9dc033712f564089700441fb08c7f89db32bccab7926ef95db6b2306/highspy-1.14.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74ee022cc8cc0e3a576f9b2309287974ea80db68adf8c9f1c5698dc725ec0497", size = 2409165, upload-time = "2026-04-06T15:52:11.067Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c5/efa6d74704aa0bc5ffce9975553f6d13f4527e6fad8f79e7cacadfedd3d3/highspy-1.14.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb7c564ee426355671edf6ad17b2ece5aaa74f21b326ed5e7a8ba5bdc880f207", size = 2632243, upload-time = "2026-04-06T15:52:12.529Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1f/a701fee9ca318e6d175d719f8916d090dd7c8100c28bc591adac9fc2db35/highspy-1.14.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:44d80756dde11941336933d777ae85b913c303fe40c3b7eb55fae0fe62bbea08", size = 2792226, upload-time = "2026-04-06T15:52:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a4/b1db0018292e46d75d7aa7889f220d0fa0f2243a628e57790de80f1fc22f/highspy-1.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ba00a35b6c4b96eb2d20d75238c6046ecf82ac85a603706e5f588d28c3b3abf", size = 3465843, upload-time = "2026-04-06T15:52:15.692Z" }, - { url = "https://files.pythonhosted.org/packages/6c/41/64c4b290a5237e14fbc7e4812d110aecf457273bbade8b27ca9b9ea86c5b/highspy-1.14.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e991b5ac8af64d686f2a56fb59391b4bb81a50d647fe1e31a1b0ff34d9d2bb51", size = 4042641, upload-time = "2026-04-06T15:52:17.573Z" }, - { url = "https://files.pythonhosted.org/packages/30/3a/cff37994f2fd313467749bc9939e5baab4aef0210c79547471d6bbda5e81/highspy-1.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c10554bd0a37d4be126f24cbb7f5aca5a4252dd6e572b51b2433ad6d94ce7a9d", size = 3712946, upload-time = "2026-04-06T15:52:19.466Z" }, - { url = "https://files.pythonhosted.org/packages/6d/60/f328af00a9f05838e766aa7808dc733bb74a4fa79adcf5fdd665cfb8d7ed/highspy-1.14.0-cp312-cp312-win32.whl", hash = "sha256:7290540f0352192e43bdc790a59a82cee1f8029bd8d6b9ca20b54b651256bab4", size = 1955124, upload-time = "2026-04-06T15:52:20.965Z" }, - { url = "https://files.pythonhosted.org/packages/69/ea/0b47c49b6df4474c603b6a232278d5d6e6afddcb9da5044e06dfec579222/highspy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0568d0fb514dc82776c3be1041988fe428c9df2be0ce98c1bff6382c0d2a5ba", size = 2323321, upload-time = "2026-04-06T15:52:22.418Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "huey" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/29/3428d52eb8e85025e264a291641a9f9d6407cc1e51d1b630f6ac5815999a/huey-2.6.0.tar.gz", hash = "sha256:8d11f8688999d65266af1425b831f6e3773e99415027177b8734b0ffd5e251f6", size = 221068, upload-time = "2026-01-06T03:01:02.055Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/34/fae9ac8f1c3a552fd3f7ff652b94c78d219dedc5fce0c0a4232457760a00/huey-2.6.0-py3-none-any.whl", hash = "sha256:1b9df9d370b49c6d5721ba8a01ac9a787cf86b3bdc584e4679de27b920395c3f", size = 76951, upload-time = "2026-01-06T03:01:00.808Z" }, -] - -[[package]] -name = "hydra-core" -version = "1.3.2" +name = "hydra-core" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "antlr4-python3-runtime" }, { name = "omegaconf" }, { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/8e/07e42bc434a847154083b315779b0a81d567154504624e181caf2c71cd98/hydra-core-1.3.2.tar.gz", hash = "sha256:8a878ed67216997c3e9d88a8e72e7b4767e81af37afb4ea3334b269a4390a824", size = 3263494, upload-time = "2023-02-23T18:33:43.03Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/dd/220f0e91743136725352497e98540772a01fc7c3ab96ff16c3c74424e984/hydra_core-1.3.4.tar.gz", hash = "sha256:ad0f7b05a0242255a8984d5a4ed2f6847f7b783ed727368a2c0155ec52d6c34c", size = 3263348, upload-time = "2026-07-04T16:25:38.891Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b", size = 154547, upload-time = "2023-02-23T18:33:40.801Z" }, + { url = "https://files.pythonhosted.org/packages/ed/cd/a568610bafe991fdd3f628fb606316b3b2be52ded019284e895d9beb3a1e/hydra_core-1.3.4-py3-none-any.whl", hash = "sha256:e58683692904a09f1fdfffa1a9b86bfd94e215b59f1ee17e7cd7d92738090d33", size = 155478, upload-time = "2026-07-04T16:25:37.291Z" }, ] [[package]] name = "hypothesis" -version = "6.155.2" +version = "6.156.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f5/04/64032a1dccd2233615c8a3f701bbb563558575ed017496a24b6d81762c91/hypothesis-6.155.2.tar.gz", hash = "sha256:ae36880287c9c5defe9f199d3d2b67d9947a4da2a46e6c57373cbdf2345b20e1", size = 477765, upload-time = "2026-06-05T16:32:23.63Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/6e/e735f27ac1a530a4cd0a31cd970ec495a3a11830fdc5d281cc292593b330/hypothesis-6.155.2-py3-none-any.whl", hash = "sha256:c85ce6dcd630a90ce501f1d1dd1bc84b97f5649ca8a27e134c8cbf5aa480b1a5", size = 544213, upload-time = "2026-06-05T16:32:21.15Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/20/83/8dbe89bdb8c6f25a7a52e7898af6d82fe35dfef08e5c702f6e33231ce6c6/hypothesis-6.156.6.tar.gz", hash = "sha256:96de02faefa3ce079873541da96f42595583bb001e8e4219294ed7d4501cc4cc", size = 476304, upload-time = "2026-07-10T20:56:49.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/dc/0c2a851f06c91d5ac9ef0f3b9615efc1ed650411d2eee23b6334f491c85e/hypothesis-6.156.6-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:caf6a93d011c10972da111c38ceb34ced20feaa8581e2b350c0655b022e27875", size = 747998, upload-time = "2026-07-10T20:56:16.311Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/59203ca978ab51595d12d6bc7e7a63300d7373431ab42ca3f1742e45db68/hypothesis-6.156.6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:07f2bc9df1aeba80e12029c1618e2ee54abc440068c305d7075ffd6b85251843", size = 743073, upload-time = "2026-07-10T20:55:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/68/d8/86a0023740434098d1b187a62bd5f99b198f098fb43e7fc58342283a8270/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baca17f4803ad4aa151732326f3990baf54c3127df44aa872ac5bdf8a98a9a6", size = 1070169, upload-time = "2026-07-10T20:55:49.47Z" }, + { url = "https://files.pythonhosted.org/packages/9b/82/673453915fd0c67673f35a4876ba88f48c621335f293f3537d77b27d4286/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8083806645f84243aade727f4978185caaa0b7190af4318673999ee15fdbf424", size = 1121760, upload-time = "2026-07-10T20:55:53.502Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c3/3a5557f52912f2fecc6ed59642dcf80dd8e89d0d9664502b68e23d66bf3d/hypothesis-6.156.6-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a922eedcd8618f9c2e17b79fa7b3f3f0b2df34e201958611cc3f0f46cca33c10", size = 1111440, upload-time = "2026-07-10T20:55:43.054Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/ae636d4ca7f996a1ccb4b3d5997d949f1718fba52b01559b3ab53b237b3f/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5291bd33c4704d274d7c214d5c200e77f372a06644f5cbbe96dcbe53cb2fbf10", size = 1244944, upload-time = "2026-07-10T20:55:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/1e/79/c425d22d734be0268ca60d120c6296299e4220a1783cb1a4cc76232807bb/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55f3ec50161b4a95bae63bff2b5166e45935b493013d3be30ede279bf6192318", size = 1288808, upload-time = "2026-07-10T20:56:06.249Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3a/cc9f479d22cbdd36ddfc55a978378eddadd183b09339ebdb81be33bb18e7/hypothesis-6.156.6-cp310-abi3-win32.whl", hash = "sha256:e96570ca5cdd9a5f2ff9e80a6fb2fd5420ebf33b833d7de5b09b6ebb26a3eb6c", size = 634868, upload-time = "2026-07-10T20:55:37.959Z" }, + { url = "https://files.pythonhosted.org/packages/d6/89/2008d287289841a936456cb13443ca89d88da6e4527d611d482e9544164d/hypothesis-6.156.6-cp310-abi3-win_amd64.whl", hash = "sha256:32710718c22fe8c5571464e898bb87d282837b02617d6ad68130abf7cb4843cb", size = 640382, upload-time = "2026-07-10T20:55:30.634Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/e4a0796190d8089e85f06731e21fdddd7e8edd3a4e562101527a048e21c4/hypothesis-6.156.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5baec7943a14d106e982121dd4f74cfc5ef45e37c17f94fe49338d3d1377f38c", size = 748988, upload-time = "2026-07-10T20:56:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/4a789b286cd2cced31992e1f683036b51dd6909b934ea007ffb43aa3a32f/hypothesis-6.156.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5f519905ddeb10e23b8ba2c254541a5b1a8f146fe0551be94d972f4a77226f4", size = 743754, upload-time = "2026-07-10T20:56:09.113Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f7/3dd36c1c03d24ae3ffc3c5b0eca8cc4ae90c07abc320f76509eceb37019a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c108655960b58ded3ca71b2dc5c69fb2ba7e9c723aeb6106facec3892d09087", size = 1070732, upload-time = "2026-07-10T20:56:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/57/51/befc4b816b471078034a875eb1ef69e0411ab84bcce582b4be173258785a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8d37bebb6924729bc0bbf5852689df568842948abe4d93dd0ad3377adf76fa", size = 1121988, upload-time = "2026-07-10T20:56:07.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/b7/a796f5e3e4b7cb911ff346008d49720296d1f4073490b8bc1cce6b3fbb07/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:74137bef6d502305c3648b2ed1a9bb4bc05fb1025e96b30a2c092204c40fe097", size = 1245596, upload-time = "2026-07-10T20:55:50.662Z" }, + { url = "https://files.pythonhosted.org/packages/37/65/849c4cba44a6f6cc888fd931124429b24180234ccc4883abab8cad5fcfcb/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5e0afdf79cceed20fcf0a9fb80d4064a9b2b53d4d4eecbac0e21208a13f5a31b", size = 1289172, upload-time = "2026-07-10T20:55:58.915Z" }, + { url = "https://files.pythonhosted.org/packages/13/03/7106a110df29eb631d66776e8aa8128f82f04a9dd2b6b22b612e6025e3a2/hypothesis-6.156.6-cp311-cp311-win_amd64.whl", hash = "sha256:84dc89caaf741a02f904ca7bd02b1af99650c75552868162290208aeecb70858", size = 640222, upload-time = "2026-07-10T20:56:10.396Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9f009005b9c796f4a40424484ac7e70847bc088456fd940a937f96bb4b6d/hypothesis-6.156.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a2a728b514fceb81e3f0464508911d5220fd74dadc3270f859427a686b60c4cf", size = 748844, upload-time = "2026-07-10T20:56:38.036Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/4d852bb8a9c73a68b18eca9b5b085285282122166e158f4d2a477639bfee/hypothesis-6.156.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7489b9a8f9df8227edd6c7cd8b9ccfab2483bab24da6a474c175973ca2294f58", size = 741936, upload-time = "2026-07-10T20:55:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/b9968070ae042f9bf3149bb6ba6399d5f28f452e0fb7f638cafc69ff0b9a/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42760873d6db1069d6edbaa355a61b9078a9950259efcfc72fc695741d7db7cd", size = 1069749, upload-time = "2026-07-10T20:56:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/00/a9/753806f5292b40aeab1d269e408e3a7e85be3c0d88828fb78ab4a34d6626/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d", size = 1120983, upload-time = "2026-07-10T20:56:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/85/88/8386d064d680be27e936eba94f1448bc93ef6fa05473ee5034139f1c4284/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:08796b674c0b31a5dd4119b2173823390055921588d13eb77324e861b00fd7f8", size = 1243911, upload-time = "2026-07-10T20:55:54.799Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8c/7524c1e5279e7728eb47c99f2357cbc5f08ae92e9bce49bf50118b53f9c9/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4ca8cc26ea2d31d22cf7710e92951cfaa921f0f8aa1b6db33a5176335f583a4f", size = 1287806, upload-time = "2026-07-10T20:56:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b3/c347ad913e1c5f2988956fe17826c0400b4ce470b973e6c248e97b6a0acf/hypothesis-6.156.6-cp312-cp312-win_amd64.whl", hash = "sha256:c3363d3fb8015594636689572510bb6090602d8e8e838a5693c2d52d3b5b09d8", size = 637679, upload-time = "2026-07-10T20:55:39.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/b94f3a31665527b6181616b72990fcf8d6d5fa82b4187aab104ab5f548f0/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8893b4da90e06828846c1b100c3414a7729d047a020d854c0899ae9339df0e70", size = 749575, upload-time = "2026-07-10T20:55:29.371Z" }, + { url = "https://files.pythonhosted.org/packages/21/74/dcf695f79f526543ae5d0f8c1325508e9fe990a996c0e0853129a9a5d81d/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7fc0b7df9b28d028e4cc295b2ac8fbbbc22e090a23382c92fff5e37696be74f", size = 744351, upload-time = "2026-07-10T20:56:36.46Z" }, + { url = "https://files.pythonhosted.org/packages/11/d3/5bff4c55c6995a6c43f66ec8e5866b56e34f03837fd0be0e4922f3bab168/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ed3178526382d392d04ad699ad7a2e53845e521a09d40f1cbbc1e1ff63ba48", size = 1070916, upload-time = "2026-07-10T20:56:19.256Z" }, + { url = "https://files.pythonhosted.org/packages/ba/af/5ed42117a69221ea118caaff933d8212039a0ac0bc15afa915635f13984c/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ad94e28aabf4db0d479297d43b8a2a01e7caaa9bdfccfdac7a4a3717e05b993", size = 1122625, upload-time = "2026-07-10T20:56:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d3/02499badc6e3f3e980941021edf5fd780c895d8d08c9015e78516340ed83/hypothesis-6.156.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:983a5cfd955994bffc7eb02976241f7a1f3c2d94dbc3389430c45858fa5c1ae0", size = 640823, upload-time = "2026-07-10T20:55:45.461Z" }, ] [[package]] @@ -2119,11 +1247,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -2135,18 +1263,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/ce/f9018bf69ae91b273b6391a095e7c93fa5e1617f25b6ba81ad4b20c9df10/immutabledict-4.3.1-py3-none-any.whl", hash = "sha256:c9facdc0ff30fdb8e35bd16532026cac472a549e182c94fa201b51b25e4bf7bf", size = 5000, upload-time = "2026-02-15T10:32:33.672Z" }, ] -[[package]] -name = "importlib-metadata" -version = "8.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -2156,105 +1272,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "ipykernel" -version = "7.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "appnope", marker = "sys_platform == 'darwin'" }, - { name = "comm" }, - { name = "debugpy" }, - { name = "ipython" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "matplotlib-inline" }, - { name = "nest-asyncio" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyzmq" }, - { name = "tornado" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/8d/b68b728e2d06b9e0051019640a40a9eb7a88fcd82c2e1b5ce70bef5ff044/ipykernel-7.2.0.tar.gz", hash = "sha256:18ed160b6dee2cbb16e5f3575858bc19d8f1fe6046a9a680c708494ce31d909e", size = 176046, upload-time = "2026-02-06T16:43:27.403Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/b9/e73d5d9f405cba7706c539aa8b311b49d4c2f3d698d9c12f815231169c71/ipykernel-7.2.0-py3-none-any.whl", hash = "sha256:3bbd4420d2b3cc105cbdf3756bfc04500b1e52f090a90716851f3916c62e1661", size = 118788, upload-time = "2026-02-06T16:43:25.149Z" }, -] - -[[package]] -name = "ipython" -version = "9.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/86/3060e8029b7cc505cce9a0137431dda81d0a3fde93a8f0f50ee0bf37a795/ipython-9.13.0-py3-none-any.whl", hash = "sha256:57f9d4639e20818d328d287c7b549af3d05f12486ea8f2e7f73e52a36ec4d201", size = 627274, upload-time = "2026-04-24T12:24:53.038Z" }, -] - -[[package]] -name = "ipython-pygments-lexers" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, -] - -[[package]] -name = "ipywidgets" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "comm" }, - { name = "ipython" }, - { name = "jupyterlab-widgets" }, - { name = "traitlets" }, - { name = "widgetsnbextension" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, -] - -[[package]] -name = "isodate" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/7a/c0a56c7d56c7fa723988f122fa1f1ccf8c5c4ccc48efad0d214b49e5b1af/isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9", size = 28443, upload-time = "2021-12-13T20:28:31.525Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", size = 41722, upload-time = "2021-12-13T20:28:29.073Z" }, -] - -[[package]] -name = "isoduration" -version = "20.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "arrow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321, upload-time = "2020-11-01T10:59:58.02Z" }, -] - [[package]] name = "iterative-telemetry" version = "0.0.10" @@ -2270,27 +1287,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/82/7331bbf84f1ccce7a2dd09a580c7bad38417cf35c84dc0b81bce2cf779b9/iterative_telemetry-0.0.10-py3-none-any.whl", hash = "sha256:e58ffb60d22c3de8dad6a114697cc61f6c14911cae484bf90df394e0d6553603", size = 10644, upload-time = "2025-02-11T02:47:51.273Z" }, ] -[[package]] -name = "itsdangerous" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, -] - -[[package]] -name = "jedi" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parso" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, -] - [[package]] name = "jinja2" version = "3.1.6" @@ -2322,509 +1318,131 @@ wheels = [ ] [[package]] -name = "json5" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/4b/6f8906aaf67d501e259b0adab4d312945bb7211e8b8d4dcc77c92320edaa/json5-0.14.0.tar.gz", hash = "sha256:b3f492fad9f6cdbced8b7d40b28b9b1c9701c5f561bef0d33b81c2ff433fefcb", size = 52656, upload-time = "2026-03-27T22:50:48.108Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/42/cf027b4ac873b076189d935b135397675dac80cb29acb13e1ab86ad6c631/json5-0.14.0-py3-none-any.whl", hash = "sha256:56cf861bab076b1178eb8c92e1311d273a9b9acea2ccc82c276abf839ebaef3a", size = 36271, upload-time = "2026-03-27T22:50:47.073Z" }, -] - -[[package]] -name = "jsonpointer" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, -] - -[package.optional-dependencies] -format-nongpl = [ - { name = "fqdn" }, - { name = "idna" }, - { name = "isoduration" }, - { name = "jsonpointer" }, - { name = "rfc3339-validator" }, - { name = "rfc3986-validator" }, - { name = "rfc3987-syntax" }, - { name = "uri-template" }, - { name = "webcolors" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" +name = "kiwisolver" +version = "1.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, + { url = "https://files.pythonhosted.org/packages/12/dd/a495a9c104be1c476f0386e714252caf2b7eca883915422a64c50b88c6f5/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c", size = 122798, upload-time = "2026-03-09T13:12:58.963Z" }, + { url = "https://files.pythonhosted.org/packages/11/60/37b4047a2af0cf5ef6d8b4b26e91829ae6fc6a2d1f74524bcb0e7cd28a32/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb", size = 66216, upload-time = "2026-03-09T13:13:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/510dc933d87767584abfe03efa445889996c70c2990f6f87c3ebaa0a18c5/kiwisolver-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac", size = 63911, upload-time = "2026-03-09T13:13:01.671Z" }, + { url = "https://files.pythonhosted.org/packages/80/46/bddc13df6c2a40741e0cc7865bb1c9ed4796b6760bd04ce5fae3928ef917/kiwisolver-1.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27", size = 1438209, upload-time = "2026-03-09T13:13:03.385Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d6/76621246f5165e5372f02f5e6f3f48ea336a8f9e96e43997d45b240ed8cd/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398", size = 1248888, upload-time = "2026-03-09T13:13:05.231Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c1/31559ec6fb39a5b48035ce29bb63ade628f321785f38c384dee3e2c08bc1/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db", size = 1266304, upload-time = "2026-03-09T13:13:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ef/1cb8276f2d29cc6a41e0a042f27946ca347d3a4a75acf85d0a16aa6dcc82/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc", size = 1319650, upload-time = "2026-03-09T13:13:08.607Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e4/5ba3cecd7ce6236ae4a80f67e5d5531287337d0e1f076ca87a5abe4cd5d0/kiwisolver-1.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679", size = 970949, upload-time = "2026-03-09T13:13:10.299Z" }, + { url = "https://files.pythonhosted.org/packages/5a/69/dc61f7ae9a2f071f26004ced87f078235b5507ab6e5acd78f40365655034/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309", size = 2199125, upload-time = "2026-03-09T13:13:11.841Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7b/abbe0f1b5afa85f8d084b73e90e5f801c0939eba16ac2e49af7c61a6c28d/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2", size = 2293783, upload-time = "2026-03-09T13:13:14.399Z" }, + { url = "https://files.pythonhosted.org/packages/8a/80/5908ae149d96d81580d604c7f8aefd0e98f4fd728cf172f477e9f2a81744/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c", size = 1960726, upload-time = "2026-03-09T13:13:16.047Z" }, + { url = "https://files.pythonhosted.org/packages/84/08/a78cb776f8c085b7143142ce479859cfec086bd09ee638a317040b6ef420/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08", size = 2464738, upload-time = "2026-03-09T13:13:17.897Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e1/65584da5356ed6cb12c63791a10b208860ac40a83de165cb6a6751a686e3/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4", size = 2270718, upload-time = "2026-03-09T13:13:19.421Z" }, + { url = "https://files.pythonhosted.org/packages/be/6c/28f17390b62b8f2f520e2915095b3c94d88681ecf0041e75389d9667f202/kiwisolver-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b", size = 73480, upload-time = "2026-03-09T13:13:20.818Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0e/2ee5debc4f77a625778fec5501ff3e8036fe361b7ee28ae402a485bb9694/kiwisolver-1.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac", size = 64930, upload-time = "2026-03-09T13:13:21.997Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b2/818b74ebea34dabe6d0c51cb1c572e046730e64844da6ed646d5298c40ce/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9", size = 123158, upload-time = "2026-03-09T13:13:23.127Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d9/405320f8077e8e1c5c4bd6adc45e1e6edf6d727b6da7f2e2533cf58bff71/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588", size = 66388, upload-time = "2026-03-09T13:13:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/99/9f/795fedf35634f746151ca8839d05681ceb6287fbed6cc1c9bf235f7887c2/kiwisolver-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819", size = 64068, upload-time = "2026-03-09T13:13:25.878Z" }, + { url = "https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f", size = 1477934, upload-time = "2026-03-09T13:13:27.166Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf", size = 1278537, upload-time = "2026-03-09T13:13:28.707Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d", size = 1296685, upload-time = "2026-03-09T13:13:30.528Z" }, + { url = "https://files.pythonhosted.org/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083", size = 1346024, upload-time = "2026-03-09T13:13:32.818Z" }, + { url = "https://files.pythonhosted.org/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6", size = 987241, upload-time = "2026-03-09T13:13:34.435Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1", size = 2227742, upload-time = "2026-03-09T13:13:36.4Z" }, + { url = "https://files.pythonhosted.org/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0", size = 2323966, upload-time = "2026-03-09T13:13:38.204Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15", size = 1977417, upload-time = "2026-03-09T13:13:39.966Z" }, + { url = "https://files.pythonhosted.org/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314", size = 2491238, upload-time = "2026-03-09T13:13:41.698Z" }, + { url = "https://files.pythonhosted.org/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9", size = 2294947, upload-time = "2026-03-09T13:13:43.343Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0348374369ca588f8fe9c338fae49fa4e16eeb10ffb3d012f23a54578a9e/kiwisolver-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384", size = 73569, upload-time = "2026-03-09T13:13:45.792Z" }, + { url = "https://files.pythonhosted.org/packages/28/26/192b26196e2316e2bd29deef67e37cdf9870d9af8e085e521afff0fed526/kiwisolver-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7", size = 64997, upload-time = "2026-03-09T13:13:46.878Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fa/2910df836372d8761bb6eff7d8bdcb1613b5c2e03f260efe7abe34d388a7/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797", size = 130262, upload-time = "2026-03-09T13:15:35.629Z" }, + { url = "https://files.pythonhosted.org/packages/0f/41/c5f71f9f00aabcc71fee8b7475e3f64747282580c2fe748961ba29b18385/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203", size = 138036, upload-time = "2026-03-09T13:15:36.894Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7", size = 194295, upload-time = "2026-03-09T13:15:38.22Z" }, + { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" }, + { url = "https://files.pythonhosted.org/packages/e9/eb/5fcbbbf9a0e2c3a35effb88831a483345326bbc3a030a3b5b69aee647f84/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232", size = 59532, upload-time = "2026-03-09T13:15:47.047Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9b/e17104555bb4db148fd52327feea1e96be4b88e8e008b029002c281a21ab/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a", size = 57420, upload-time = "2026-03-09T13:15:48.199Z" }, + { url = "https://files.pythonhosted.org/packages/48/44/2b5b95b7aa39fb2d8d9d956e0f3d5d45aef2ae1d942d4c3ffac2f9cfed1a/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737", size = 79892, upload-time = "2026-03-09T13:15:49.694Z" }, + { url = "https://files.pythonhosted.org/packages/52/7d/7157f9bba6b455cfb4632ed411e199fc8b8977642c2b12082e1bd9e6d173/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16", size = 77603, upload-time = "2026-03-09T13:15:50.945Z" }, + { url = "https://files.pythonhosted.org/packages/0a/dd/8050c947d435c8d4bc94e3252f4d8bb8a76cfb424f043a8680be637a57f1/kiwisolver-1.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1", size = 73558, upload-time = "2026-03-09T13:15:52.112Z" }, ] [[package]] -name = "jupyter" -version = "1.1.1" +name = "kombu" +version = "5.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel" }, - { name = "ipywidgets" }, - { name = "jupyter-console" }, - { name = "jupyterlab" }, - { name = "nbconvert" }, - { name = "notebook" }, + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/f3/af28ea964ab8bc1e472dba2e82627d36d470c51f5cd38c37502eeffaa25e/jupyter-1.1.1.tar.gz", hash = "sha256:d55467bceabdea49d7e3624af7e33d59c37fff53ed3a350e1ac957bed731de7a", size = 5714959, upload-time = "2024-08-30T07:15:48.299Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/64/285f20a31679bf547b75602702f7800e74dbabae36ef324f716c02804753/jupyter-1.1.1-py2.py3-none-any.whl", hash = "sha256:7a59533c22af65439b24bbe60373a4e95af8f16ac65a6c00820ad378e3f7cc83", size = 2657, upload-time = "2024-08-30T07:15:47.045Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, ] [[package]] -name = "jupyter-builder" -version = "1.0.2" +name = "librt" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-core" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/45/d0df8b43c10a61529c0f4a8af5e19ebe108f0c3af8f57e0fc358969907af/jupyter_builder-1.0.2.tar.gz", hash = "sha256:6155d78a5325010532a6419ffcba89eac643fd1aa56ea83115e661924d6f6aab", size = 968638, upload-time = "2026-06-12T02:33:25.767Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/b6/c418e0b3256f67c04933566b80bfce947350682db92c4b786a8653db32d6/jupyter_builder-1.0.2-py3-none-any.whl", hash = "sha256:b024f65d36e1d530542db597b00dd513261aa59842e0d0fbbb1015a9f1935e9c", size = 910789, upload-time = "2026-06-12T02:33:23.317Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/dc/2f/3908645ddddab7120b46295e541ead308109fa48dbec7d67d7a778870d60/librt-0.13.0.tar.gz", hash = "sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781", size = 211402, upload-time = "2026-07-08T12:26:29.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/25/a6498964cfeec270c468cffdc118f69c29b412593610d55fa1327ca51ff4/librt-0.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1b5a7bbff495baedbd9b916c367d66854008f8f3b575908ded477c499dc60082", size = 148029, upload-time = "2026-07-08T12:24:45.961Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/dc86d1bffd8e0c2818bace29d9f7783cfbb8e0673bf3673b5bbd5bbe0420/librt-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34bc7938b9fdf14fe32a406c19c71faf894c5cee7e7474bd0be2f17200b82d14", size = 153036, upload-time = "2026-07-08T12:24:47.257Z" }, + { url = "https://files.pythonhosted.org/packages/29/3f/b923826660f02f286186cd9303d52bb05ced0a13708edc104dc8480920e3/librt-0.13.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f40e56b61b41be5f7dec938cfeffd660668cf4b5e72c78e7bd671d66b7bc2c79", size = 493062, upload-time = "2026-07-08T12:24:48.483Z" }, + { url = "https://files.pythonhosted.org/packages/88/87/6c0980a9c9b1302cb68d108906697b89eceb55889bb1dcf77c109aa56ca5/librt-0.13.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:9c5d02b89de5acd0379a51ec44a89476fb03df6145442e1c8ecd6bee2f91b176", size = 485510, upload-time = "2026-07-08T12:24:49.727Z" }, + { url = "https://files.pythonhosted.org/packages/32/81/795ae3b9df5dd94079fb807e38191855e023e8c6249014ae6bc3f0d9a490/librt-0.13.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7db9a3ff32ef5f7d1703d93831a3316cdf0b537de6a1cc03cc8fdd09b9194e89", size = 515909, upload-time = "2026-07-08T12:24:51.135Z" }, + { url = "https://files.pythonhosted.org/packages/20/e5/182de15abce8907108a6fdb41487de65beb5099b74dc5841b19b099168db/librt-0.13.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3dbb2a31882456cadc7053378e81ad7ed7693db4ac9f98ab5f81ef034aa8ec9f", size = 508620, upload-time = "2026-07-08T12:24:52.358Z" }, + { url = "https://files.pythonhosted.org/packages/32/03/33978d32db76e1f66377e8f78e42a2ca3c162143331677d1f50bbad36cfb/librt-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c6014e3c80f9c1fe268ef8b0e0ef113bac672cc032f2f93866e7ddad4f3e663d", size = 530363, upload-time = "2026-07-08T12:24:53.503Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f5/b291fbd2d00f7d8287bcbf67b5aa0c6afed4bc26cef23e079629c47a2c04/librt-0.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:091b60a4d2174fc1ec5c34cdc0b72efb6224753d76b7da61ebeab7a191aec8bd", size = 534209, upload-time = "2026-07-08T12:24:55.138Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/6f41f17939d191bc21609f220da8509316bc62797f078545fe83be522e78/librt-0.13.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:66cb1138f384a191a6d75f986064841fcfdc0cea98f7bd9c9ab9b38049917588", size = 514254, upload-time = "2026-07-08T12:24:56.276Z" }, + { url = "https://files.pythonhosted.org/packages/af/c2/2e4befa5410a7443019c14abccc94ff619797171f6b72013635fb87f31d7/librt-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:17221a7569f8f292aa0014226e48aa25b8c2b08da18088cd230953d0ea0f9cd1", size = 557611, upload-time = "2026-07-08T12:24:57.561Z" }, + { url = "https://files.pythonhosted.org/packages/ab/54/8b69f81448417adbc040a2185f4e2eece1e1994b7dcfaeed4662b30f98a5/librt-0.13.0-cp311-cp311-win32.whl", hash = "sha256:fc67741da44c6eaa90e01eafb586bbba9b51eb5b6ed381ee6f5ae72eb3316d21", size = 104906, upload-time = "2026-07-08T12:24:58.806Z" }, + { url = "https://files.pythonhosted.org/packages/76/5a/f4aaf37b50f2fde12c8c663b83fdd499cdc24f957f19543d7414bfcc9e25/librt-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:cc99dfb62b23c9207c33d0be8a2e2af7a42e21e6ea388b380a0c948c7b88953b", size = 125852, upload-time = "2026-07-08T12:25:00.065Z" }, + { url = "https://files.pythonhosted.org/packages/f2/99/bf1820e6feeabc2f218c24450ec0c995d6a91e8ba0fd3caf042c9e8adb2a/librt-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:40ccd13c252d3fe473ffc8a57be7565abc8b64cf1b108344c859d5164f7f3e0c", size = 111832, upload-time = "2026-07-08T12:25:01.148Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/b2933ddae222dac338476abb872641169a5cfed2c2bb5444a5b07b32b0c3/librt-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0", size = 150990, upload-time = "2026-07-08T12:25:02.42Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/db98f744ca50e6efc9c95c70ee49b77aefac31f6a3fc7c83754a42d6a74f/librt-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5", size = 155238, upload-time = "2026-07-08T12:25:03.681Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/a197e7bc72baf2c61ce7fdc6906a5054dc05bd8da0819aa894e4857bf87e/librt-0.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9", size = 503073, upload-time = "2026-07-08T12:25:05.049Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e7/7887712e27da7c1ab80fcabb1de6eb24243964f6557cae530d4b70706dbd/librt-0.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b", size = 496528, upload-time = "2026-07-08T12:25:06.26Z" }, + { url = "https://files.pythonhosted.org/packages/94/f0/f2283385bb6b950b26a1410f4ce51ec27231e0b3a4b925c46366d218b198/librt-0.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03", size = 531786, upload-time = "2026-07-08T12:25:07.658Z" }, + { url = "https://files.pythonhosted.org/packages/36/11/69ac3b54766ffba5fd7e5acebfb048d66dbe1f9f2d14516c2b3edc59cf87/librt-0.13.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e", size = 524393, upload-time = "2026-07-08T12:25:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/61/5f/d72f95fd444a926a3c14b4e24979474116988dd57a45be242077c45d3c22/librt-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd", size = 543026, upload-time = "2026-07-08T12:25:10.459Z" }, + { url = "https://files.pythonhosted.org/packages/c4/08/dcd9993ad192737a004ba263d549f8ea605b326b952e7d6205c7d4170b76/librt-0.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348", size = 546829, upload-time = "2026-07-08T12:25:11.716Z" }, + { url = "https://files.pythonhosted.org/packages/96/d5/6d9bb2f54e4109a956b7128836529653eb9d740f784bc47ed10a02c1000e/librt-0.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7", size = 535700, upload-time = "2026-07-08T12:25:13.144Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f2/10946922503858a359492fa27f13e86228bde702116a740ac7b3cd185f24/librt-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82", size = 573566, upload-time = "2026-07-08T12:25:14.336Z" }, + { url = "https://files.pythonhosted.org/packages/48/a8/94f00e3c99479a18088af3685ea016c42f3c7d5d1964d8dbb40c08d7f1aa/librt-0.13.0-cp312-cp312-win32.whl", hash = "sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3", size = 106099, upload-time = "2026-07-08T12:25:16.159Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7b/2da9c74c1ed25a89cc4e1c8e007ea2eb4a0f1fafa3e70d757fe3242c5c5c/librt-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa", size = 126934, upload-time = "2026-07-08T12:25:17.275Z" }, + { url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" }, ] [[package]] -name = "jupyter-client" -version = "8.8.0" +name = "loguru" +version = "0.7.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core" }, - { name = "python-dateutil" }, - { name = "pyzmq" }, - { name = "tornado" }, - { name = "traitlets" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/05/e4/ba649102a3bc3fbca54e7239fb924fd434c766f855693d86de0b1f2bec81/jupyter_client-8.8.0.tar.gz", hash = "sha256:d556811419a4f2d96c869af34e854e3f059b7cc2d6d01a9cd9c85c267691be3e", size = 348020, upload-time = "2026-01-08T13:55:47.938Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/0b/ceb7694d864abc0a047649aec263878acb9f792e1fec3e676f22dc9015e3/jupyter_client-8.8.0-py3-none-any.whl", hash = "sha256:f93a5b99c5e23a507b773d3a1136bd6e16c67883ccdbd9a829b0bbdb98cd7d7a", size = 107371, upload-time = "2026-01-08T13:55:45.562Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, ] [[package]] -name = "jupyter-console" -version = "6.6.3" +name = "markdown-it-py" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel" }, - { name = "ipython" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "pyzmq" }, - { name = "traitlets" }, + { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/2d/e2fd31e2fc41c14e2bcb6c976ab732597e907523f6b2420305f9fc7fdbdb/jupyter_console-6.6.3.tar.gz", hash = "sha256:566a4bf31c87adbfadf22cdf846e3069b59a71ed5da71d6ba4d8aaad14a53539", size = 34363, upload-time = "2023-03-06T14:13:31.02Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/77/71d78d58f15c22db16328a476426f7ac4a60d3a5a7ba3b9627ee2f7903d4/jupyter_console-6.6.3-py3-none-any.whl", hash = "sha256:309d33409fcc92ffdad25f0bcdf9a4a9daa61b6f341177570fdac03de5352485", size = 24510, upload-time = "2023-03-06T14:13:28.229Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] [[package]] -name = "jupyter-core" -version = "5.9.1" +name = "markupsafe" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, -] - -[[package]] -name = "jupyter-events" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonschema", extra = ["format-nongpl"] }, - { name = "packaging" }, - { name = "python-json-logger" }, - { name = "pyyaml" }, - { name = "referencing" }, - { name = "rfc3339-validator" }, - { name = "rfc3986-validator" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/f8/475c4241b2b75af0deaae453ed003c6c851766dbc44d332d8baf245dc931/jupyter_events-0.12.1.tar.gz", hash = "sha256:faff25f77218335752f35f23c5fe6e4a392a7bd99a5939ccb9b8fbf594636cf3", size = 62854, upload-time = "2026-04-20T23:17:50.66Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/6c/6fcde0c8f616ed360ffd3587f7db9e225a7e62b583a04494d2f069cf64ea/jupyter_events-0.12.1-py3-none-any.whl", hash = "sha256:c366585253f537a627da52fa7ca7410c5b5301fe893f511e7b077c2d93ec8bcf", size = 19512, upload-time = "2026-04-20T23:17:48.927Z" }, -] - -[[package]] -name = "jupyter-lsp" -version = "2.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-server" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/e8/9d61dcbd1dce8ef418f06befd4ac084b4720429c26b0b1222bc218685eff/jupyter_lsp-2.3.1-py3-none-any.whl", hash = "sha256:71b954d834e85ff3096400554f2eefaf7fe37053036f9a782b0f7c5e42dadb81", size = 77513, upload-time = "2026-04-02T08:10:01.753Z" }, -] - -[[package]] -name = "jupyter-server" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "argon2-cffi" }, - { name = "jinja2" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "jupyter-events" }, - { name = "jupyter-server-terminals" }, - { name = "nbconvert" }, - { name = "nbformat" }, - { name = "overrides", marker = "python_full_version < '3.12'" }, - { name = "packaging" }, - { name = "prometheus-client" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, - { name = "pyzmq" }, - { name = "send2trash" }, - { name = "terminado" }, - { name = "tornado" }, - { name = "traitlets" }, - { name = "websocket-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6b/dc/db3a582633170186f8c8b31298d7eb26ad0eb031a1f53476c258b64eed05/jupyter_server-2.20.0.tar.gz", hash = "sha256:b5778ba337d8015a3dc2b80803ecdd5ac18d3797fddf61a50ea5fb472b4ebe14", size = 756523, upload-time = "2026-06-17T12:09:09.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/71/8c002223e873a870f5c41dc69b0a7c922301123e4a31d5d01ecb700aef77/jupyter_server-2.20.0-py3-none-any.whl", hash = "sha256:c3b67c93c471e947c18b5026f04f21614218adb706df8f48227d3ee8e0a7cdcc", size = 393143, upload-time = "2026-06-17T12:09:07.234Z" }, -] - -[[package]] -name = "jupyter-server-terminals" -version = "0.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywinpty", marker = "os_name == 'nt'" }, - { name = "terminado" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/2d/6674563f71c6320841fc300911a55143925112a72a883e2ca71fba4c618d/jupyter_server_terminals-0.5.4-py3-none-any.whl", hash = "sha256:55be353fc74a80bc7f3b20e6be50a55a61cd525626f578dcb66a5708e2007d14", size = 13704, upload-time = "2026-01-14T16:53:18.738Z" }, -] - -[[package]] -name = "jupyterlab" -version = "4.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-lru" }, - { name = "httpx" }, - { name = "ipykernel" }, - { name = "jinja2" }, - { name = "jupyter-builder" }, - { name = "jupyter-core" }, - { name = "jupyter-lsp" }, - { name = "jupyter-server" }, - { name = "jupyterlab-server" }, - { name = "notebook-shim" }, - { name = "packaging" }, - { name = "tornado" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/2a/d6af53bfd45a43a5bfe7e40ba47ee7a8921a807daf4bb708e3a295bbb54d/jupyterlab-4.6.1.tar.gz", hash = "sha256:75315982ed28427edaa62bb85eadb5105e4043a757643c910efd787fe6ed0837", size = 28179125, upload-time = "2026-06-29T12:48:45.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/81/90ac6cc31d248e83a0d1eab343a5e6e68bc783d3f74fbe61640f42a61da4/jupyterlab-4.6.1-py3-none-any.whl", hash = "sha256:85a58546c831f3dce6cf919468c26874c9065e99c42279fb4abb8e1b552a98bb", size = 17164660, upload-time = "2026-06-29T12:48:41.21Z" }, -] - -[[package]] -name = "jupyterlab-pygments" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/51/9187be60d989df97f5f0aba133fa54e7300f17616e065d1ada7d7646b6d6/jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d", size = 512900, upload-time = "2023-11-23T09:26:37.44Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/dd/ead9d8ea85bf202d90cc513b533f9c363121c7792674f78e0d8a854b63b4/jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780", size = 15884, upload-time = "2023-11-23T09:26:34.325Z" }, -] - -[[package]] -name = "jupyterlab-server" -version = "2.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "babel" }, - { name = "jinja2" }, - { name = "json5" }, - { name = "jsonschema" }, - { name = "jupyter-server" }, - { name = "packaging" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/07/a000fe835f76b7e1143242ab1122e6362ef1c03f23f83a045c38859c2ae0/jupyterlab_server-2.28.0-py3-none-any.whl", hash = "sha256:e4355b148fdcf34d312bbbc80f22467d6d20460e8b8736bf235577dd18506968", size = 59830, upload-time = "2025-10-22T13:59:16.767Z" }, -] - -[[package]] -name = "jupyterlab-widgets" -version = "3.0.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/2d/ef58fed122b268c69c0aa099da20bc67657cdfb2e222688d5731bd5b971d/jupyterlab_widgets-3.0.16.tar.gz", hash = "sha256:423da05071d55cf27a9e602216d35a3a65a3e41cdf9c5d3b643b814ce38c19e0", size = 897423, upload-time = "2025-11-01T21:11:29.724Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl", hash = "sha256:45fa36d9c6422cf2559198e4db481aa243c7a32d9926b500781c830c80f7ecf8", size = 914926, upload-time = "2025-11-01T21:11:28.008Z" }, -] - -[[package]] -name = "jupytext" -version = "1.19.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "mdit-py-plugins" }, - { name = "nbformat" }, - { name = "packaging" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/2d/15624c3d9440d85a280ff13d2d23afd989802f25470ac59932f4fef6f0c6/jupytext-1.19.3.tar.gz", hash = "sha256:713c3ed4441afe0f31474d28ea2e6b61a268c04c40fd78e5ccfd7f7ac9e9f766", size = 4305350, upload-time = "2026-05-17T09:09:29.294Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/ec/d9be3bd1db141e76b2f525c265f70e66edd30a51a3307d8edf0ef1909c54/jupytext-1.19.3-py3-none-any.whl", hash = "sha256:acf75492f80895ad8e664fd8db1708b617008dd0e71c341a1abc3d0d07310ed0", size = 170579, upload-time = "2026-05-17T09:09:27.478Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/dd/a495a9c104be1c476f0386e714252caf2b7eca883915422a64c50b88c6f5/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c", size = 122798, upload-time = "2026-03-09T13:12:58.963Z" }, - { url = "https://files.pythonhosted.org/packages/11/60/37b4047a2af0cf5ef6d8b4b26e91829ae6fc6a2d1f74524bcb0e7cd28a32/kiwisolver-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb", size = 66216, upload-time = "2026-03-09T13:13:00.155Z" }, - { url = "https://files.pythonhosted.org/packages/0a/aa/510dc933d87767584abfe03efa445889996c70c2990f6f87c3ebaa0a18c5/kiwisolver-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac", size = 63911, upload-time = "2026-03-09T13:13:01.671Z" }, - { url = "https://files.pythonhosted.org/packages/80/46/bddc13df6c2a40741e0cc7865bb1c9ed4796b6760bd04ce5fae3928ef917/kiwisolver-1.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27", size = 1438209, upload-time = "2026-03-09T13:13:03.385Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d6/76621246f5165e5372f02f5e6f3f48ea336a8f9e96e43997d45b240ed8cd/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398", size = 1248888, upload-time = "2026-03-09T13:13:05.231Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c1/31559ec6fb39a5b48035ce29bb63ade628f321785f38c384dee3e2c08bc1/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db", size = 1266304, upload-time = "2026-03-09T13:13:06.743Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ef/1cb8276f2d29cc6a41e0a042f27946ca347d3a4a75acf85d0a16aa6dcc82/kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc", size = 1319650, upload-time = "2026-03-09T13:13:08.607Z" }, - { url = "https://files.pythonhosted.org/packages/4c/e4/5ba3cecd7ce6236ae4a80f67e5d5531287337d0e1f076ca87a5abe4cd5d0/kiwisolver-1.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679", size = 970949, upload-time = "2026-03-09T13:13:10.299Z" }, - { url = "https://files.pythonhosted.org/packages/5a/69/dc61f7ae9a2f071f26004ced87f078235b5507ab6e5acd78f40365655034/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309", size = 2199125, upload-time = "2026-03-09T13:13:11.841Z" }, - { url = "https://files.pythonhosted.org/packages/e5/7b/abbe0f1b5afa85f8d084b73e90e5f801c0939eba16ac2e49af7c61a6c28d/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2", size = 2293783, upload-time = "2026-03-09T13:13:14.399Z" }, - { url = "https://files.pythonhosted.org/packages/8a/80/5908ae149d96d81580d604c7f8aefd0e98f4fd728cf172f477e9f2a81744/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c", size = 1960726, upload-time = "2026-03-09T13:13:16.047Z" }, - { url = "https://files.pythonhosted.org/packages/84/08/a78cb776f8c085b7143142ce479859cfec086bd09ee638a317040b6ef420/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08", size = 2464738, upload-time = "2026-03-09T13:13:17.897Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e1/65584da5356ed6cb12c63791a10b208860ac40a83de165cb6a6751a686e3/kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4", size = 2270718, upload-time = "2026-03-09T13:13:19.421Z" }, - { url = "https://files.pythonhosted.org/packages/be/6c/28f17390b62b8f2f520e2915095b3c94d88681ecf0041e75389d9667f202/kiwisolver-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b", size = 73480, upload-time = "2026-03-09T13:13:20.818Z" }, - { url = "https://files.pythonhosted.org/packages/d8/0e/2ee5debc4f77a625778fec5501ff3e8036fe361b7ee28ae402a485bb9694/kiwisolver-1.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac", size = 64930, upload-time = "2026-03-09T13:13:21.997Z" }, - { url = "https://files.pythonhosted.org/packages/4d/b2/818b74ebea34dabe6d0c51cb1c572e046730e64844da6ed646d5298c40ce/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9", size = 123158, upload-time = "2026-03-09T13:13:23.127Z" }, - { url = "https://files.pythonhosted.org/packages/bf/d9/405320f8077e8e1c5c4bd6adc45e1e6edf6d727b6da7f2e2533cf58bff71/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588", size = 66388, upload-time = "2026-03-09T13:13:24.765Z" }, - { url = "https://files.pythonhosted.org/packages/99/9f/795fedf35634f746151ca8839d05681ceb6287fbed6cc1c9bf235f7887c2/kiwisolver-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819", size = 64068, upload-time = "2026-03-09T13:13:25.878Z" }, - { url = "https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f", size = 1477934, upload-time = "2026-03-09T13:13:27.166Z" }, - { url = "https://files.pythonhosted.org/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf", size = 1278537, upload-time = "2026-03-09T13:13:28.707Z" }, - { url = "https://files.pythonhosted.org/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d", size = 1296685, upload-time = "2026-03-09T13:13:30.528Z" }, - { url = "https://files.pythonhosted.org/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083", size = 1346024, upload-time = "2026-03-09T13:13:32.818Z" }, - { url = "https://files.pythonhosted.org/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6", size = 987241, upload-time = "2026-03-09T13:13:34.435Z" }, - { url = "https://files.pythonhosted.org/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1", size = 2227742, upload-time = "2026-03-09T13:13:36.4Z" }, - { url = "https://files.pythonhosted.org/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0", size = 2323966, upload-time = "2026-03-09T13:13:38.204Z" }, - { url = "https://files.pythonhosted.org/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15", size = 1977417, upload-time = "2026-03-09T13:13:39.966Z" }, - { url = "https://files.pythonhosted.org/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314", size = 2491238, upload-time = "2026-03-09T13:13:41.698Z" }, - { url = "https://files.pythonhosted.org/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9", size = 2294947, upload-time = "2026-03-09T13:13:43.343Z" }, - { url = "https://files.pythonhosted.org/packages/ad/cf/0348374369ca588f8fe9c338fae49fa4e16eeb10ffb3d012f23a54578a9e/kiwisolver-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384", size = 73569, upload-time = "2026-03-09T13:13:45.792Z" }, - { url = "https://files.pythonhosted.org/packages/28/26/192b26196e2316e2bd29deef67e37cdf9870d9af8e085e521afff0fed526/kiwisolver-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7", size = 64997, upload-time = "2026-03-09T13:13:46.878Z" }, - { url = "https://files.pythonhosted.org/packages/1c/fa/2910df836372d8761bb6eff7d8bdcb1613b5c2e03f260efe7abe34d388a7/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797", size = 130262, upload-time = "2026-03-09T13:15:35.629Z" }, - { url = "https://files.pythonhosted.org/packages/0f/41/c5f71f9f00aabcc71fee8b7475e3f64747282580c2fe748961ba29b18385/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203", size = 138036, upload-time = "2026-03-09T13:15:36.894Z" }, - { url = "https://files.pythonhosted.org/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7", size = 194295, upload-time = "2026-03-09T13:15:38.22Z" }, - { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/e9/eb/5fcbbbf9a0e2c3a35effb88831a483345326bbc3a030a3b5b69aee647f84/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232", size = 59532, upload-time = "2026-03-09T13:15:47.047Z" }, - { url = "https://files.pythonhosted.org/packages/c3/9b/e17104555bb4db148fd52327feea1e96be4b88e8e008b029002c281a21ab/kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a", size = 57420, upload-time = "2026-03-09T13:15:48.199Z" }, - { url = "https://files.pythonhosted.org/packages/48/44/2b5b95b7aa39fb2d8d9d956e0f3d5d45aef2ae1d942d4c3ffac2f9cfed1a/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737", size = 79892, upload-time = "2026-03-09T13:15:49.694Z" }, - { url = "https://files.pythonhosted.org/packages/52/7d/7157f9bba6b455cfb4632ed411e199fc8b8977642c2b12082e1bd9e6d173/kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16", size = 77603, upload-time = "2026-03-09T13:15:50.945Z" }, - { url = "https://files.pythonhosted.org/packages/0a/dd/8050c947d435c8d4bc94e3252f4d8bb8a76cfb424f043a8680be637a57f1/kiwisolver-1.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1", size = 73558, upload-time = "2026-03-09T13:15:52.112Z" }, -] - -[[package]] -name = "kombu" -version = "5.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "packaging" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, -] - -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - -[[package]] -name = "leather" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/09/849cf129d7eae1e42f873f2dbd60323267c738390b686a7384fb3fb289ad/leather-0.4.1.tar.gz", hash = "sha256:67119c2aee93be821f077193bd8534e296c05b38bd174d9c5a80c4aa31d1a4d3", size = 44072, upload-time = "2025-12-15T19:01:42.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/d4/c4dcb02ed11f8884e169b3350fc40aa4c08edf8bed77a8f0f267542e6452/leather-0.4.1-py3-none-any.whl", hash = "sha256:ec61cba1ca3ccb96ed90e38b116fc58757d97d352171006b3288c47ce3fbd183", size = 30340, upload-time = "2025-12-15T19:01:40.823Z" }, -] - -[[package]] -name = "librt" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/87/2bf31fe17587b29e3f93ec31421e2b1e1c3e349b8bf6c7c313dbad1d5340/librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", size = 141092, upload-time = "2026-05-10T18:15:34.795Z" }, - { url = "https://files.pythonhosted.org/packages/cf/08/5c5bf772920b7ebac6e32bc91a643e0ab3870199c0b542356d3baa83970a/librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", size = 142035, upload-time = "2026-05-10T18:15:36.242Z" }, - { url = "https://files.pythonhosted.org/packages/06/20/662a03d254e5b000d838e8b345d83303ddb768c080fd488e40634c0fa66b/librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", size = 475022, upload-time = "2026-05-10T18:15:37.56Z" }, - { url = "https://files.pythonhosted.org/packages/de/f3/aa81523e45184c6ec23dc7f63263362ec55f80a09d424c012359ecbe7e35/librt-0.11.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b", size = 467273, upload-time = "2026-05-10T18:15:39.182Z" }, - { url = "https://files.pythonhosted.org/packages/6b/6f/59c74b560ca8853834d5501d589c8a2519f4184f273a085ffd0f37a1cc47/librt-0.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89", size = 497083, upload-time = "2026-05-10T18:15:40.634Z" }, - { url = "https://files.pythonhosted.org/packages/fe/7b/5aa4d2c9600a719401160bf7055417df0b2a47439b9d88286ce45e56b65f/librt-0.11.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc", size = 489139, upload-time = "2026-05-10T18:15:41.934Z" }, - { url = "https://files.pythonhosted.org/packages/d6/31/9143803d7da6856a69153785768c4936864430eec0fd9461c3ea527d9922/librt-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5", size = 508442, upload-time = "2026-05-10T18:15:43.206Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5a/bce08184488426bda4ccc2c4964ac048c8f68ae89bd7120082eef4233cfd/librt-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7", size = 514230, upload-time = "2026-05-10T18:15:44.761Z" }, - { url = "https://files.pythonhosted.org/packages/89/8c/bb5e213d254b7505a0e658da199d8ab719086632ce09eef311ab27976523/librt-0.11.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d", size = 494231, upload-time = "2026-05-10T18:15:46.308Z" }, - { url = "https://files.pythonhosted.org/packages/9d/fb/541cdad5b1ab1300398c74c4c9a497b88e5074c21b1244c8f49731d3a284/librt-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412", size = 537585, upload-time = "2026-05-10T18:15:47.629Z" }, - { url = "https://files.pythonhosted.org/packages/8f/f2/464bb69295c320cb06bddb4f14a4ec67934ee14b2bffb12b19fb7ab287ba/librt-0.11.0-cp311-cp311-win32.whl", hash = "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d", size = 100509, upload-time = "2026-05-10T18:15:49.157Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e7/a17ee1788f9e4fbf548c19f4afa07c92089b9e24fef6cb2410863781ef4c/librt-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73", size = 118628, upload-time = "2026-05-10T18:15:50.345Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c7/6c766214f9f9903bcfcfbef97d807af8d8f5aa3502d247858ab17582d212/librt-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c", size = 103122, upload-time = "2026-05-10T18:15:52.068Z" }, - { url = "https://files.pythonhosted.org/packages/8b/d0/07c77e067f0838949b43bd89232c29d72efebb9d2801a9750184eb706b71/librt-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46", size = 144147, upload-time = "2026-05-10T18:15:53.227Z" }, - { url = "https://files.pythonhosted.org/packages/7a/24/8493538fa4f62f982686398a5b8f68008138a75086abdea19ade64bf4255/librt-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3", size = 143614, upload-time = "2026-05-10T18:15:54.657Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1e/f8bad050810d9171f34a1648ed910e56814c2ba61639f2bd53c6377ae24b/librt-0.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67", size = 485538, upload-time = "2026-05-10T18:15:56.117Z" }, - { url = "https://files.pythonhosted.org/packages/c0/fe/3594ebfbaf03084ba4b120c9ba5c3183fd938a48725e9bbe6ff0a5159ad8/librt-0.11.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a", size = 479623, upload-time = "2026-05-10T18:15:57.544Z" }, - { url = "https://files.pythonhosted.org/packages/b0/da/5d1876984b3746c85dbd219dbfcb73c85f54ee263fd32e5b2a632ec14571/librt-0.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a", size = 513082, upload-time = "2026-05-10T18:15:58.805Z" }, - { url = "https://files.pythonhosted.org/packages/19/6e/55bdf5d5ca00c3e18430690bf2c953d8d3ffd3c337418173d33dec985dc9/librt-0.11.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f", size = 508105, upload-time = "2026-05-10T18:16:00.2Z" }, - { url = "https://files.pythonhosted.org/packages/07/10/f1f23a7c595ee90ece4d35c851e5d104b1311a887ed1b4ac4c35bbd13da8/librt-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b", size = 522268, upload-time = "2026-05-10T18:16:01.708Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/5720f5697a7f54b78b3aefbe20df3a48cedcff1276618c4aa481177942ed/librt-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766", size = 527348, upload-time = "2026-05-10T18:16:03.496Z" }, - { url = "https://files.pythonhosted.org/packages/50/db/b4a47c6f91db4ff76348a0b3dd0cc65e090a078b765a810a62ff9434c3d3/librt-0.11.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d", size = 516294, upload-time = "2026-05-10T18:16:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/9e/58/9384b2f4eb1ed1d273d40948a7c5c4b2360213b402ef3be4641c06299f9c/librt-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8", size = 553608, upload-time = "2026-05-10T18:16:06.839Z" }, - { url = "https://files.pythonhosted.org/packages/21/7b/5aa8848a7c6a9278c79375146da1812e695754ceec5f005e6043461a7315/librt-0.11.0-cp312-cp312-win32.whl", hash = "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a", size = 101879, upload-time = "2026-05-10T18:16:08.103Z" }, - { url = "https://files.pythonhosted.org/packages/37/33/8a745436944947575b584231750a41417de1a38cf6a2e9251d1065651c09/librt-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9", size = 119831, upload-time = "2026-05-10T18:16:09.174Z" }, - { url = "https://files.pythonhosted.org/packages/59/67/a6739ac96e28b7855808bdb0370e250606104a859750d209e5a0716fe7ab/librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", size = 103470, upload-time = "2026-05-10T18:16:10.369Z" }, -] - -[[package]] -name = "loguru" -version = "0.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "win32-setctime", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, -] - -[[package]] -name = "lxml" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/28/30/9abc9e34c657c33834eaf6cd02124c61bdf5944d802aa48e69be8da3585d/lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13", size = 4197006, upload-time = "2026-04-18T04:32:51.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/5d/3bccad330292946f97962df9d5f2d3ae129cce6e212732a781e856b91e07/lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9", size = 8526232, upload-time = "2026-04-18T04:27:40.389Z" }, - { url = "https://files.pythonhosted.org/packages/a7/51/adc8826570a112f83bb4ddb3a2ab510bbc2ccd62c1b9fe1f34fae2d90b57/lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50", size = 4595448, upload-time = "2026-04-18T04:27:44.208Z" }, - { url = "https://files.pythonhosted.org/packages/54/84/5a9ec07cbe1d2334a6465f863b949a520d2699a755738986dcd3b6b89e3f/lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5", size = 4923771, upload-time = "2026-04-18T04:32:17.402Z" }, - { url = "https://files.pythonhosted.org/packages/a7/23/851cfa33b6b38adb628e45ad51fb27105fa34b2b3ba9d1d4aa7a9428dfe0/lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e", size = 5068101, upload-time = "2026-04-18T04:32:21.437Z" }, - { url = "https://files.pythonhosted.org/packages/b0/38/41bf99c2023c6b79916ba057d83e9db21d642f473cac210201222882d38b/lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512", size = 5002573, upload-time = "2026-04-18T04:32:25.373Z" }, - { url = "https://files.pythonhosted.org/packages/c2/20/053aa10bdc39747e1e923ce2d45413075e84f70a136045bb09e5eaca41d3/lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c", size = 5202816, upload-time = "2026-04-18T04:32:29.393Z" }, - { url = "https://files.pythonhosted.org/packages/9a/da/bc710fad8bf04b93baee752c192eaa2210cd3a84f969d0be7830fea55802/lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5", size = 5329999, upload-time = "2026-04-18T04:32:34.019Z" }, - { url = "https://files.pythonhosted.org/packages/b3/cb/bf035dedbdf7fab49411aa52e4236f3445e98d38647d85419e6c0d2806b9/lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289", size = 4659643, upload-time = "2026-04-18T04:32:37.932Z" }, - { url = "https://files.pythonhosted.org/packages/5c/4f/22be31f33727a5e4c7b01b0a874503026e50329b259d3587e0b923cf964b/lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a", size = 5265963, upload-time = "2026-04-18T04:32:41.881Z" }, - { url = "https://files.pythonhosted.org/packages/c8/2b/d44d0e5c79226017f4ab8c87a802ebe4f89f97e6585a8e4166dffcdd7b6e/lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3", size = 5045444, upload-time = "2026-04-18T04:32:44.512Z" }, - { url = "https://files.pythonhosted.org/packages/d3/c3/3f034fec1594c331a6dbf9491238fdcc9d66f68cc529e109ec75b97197e1/lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9", size = 4712703, upload-time = "2026-04-18T04:32:47.16Z" }, - { url = "https://files.pythonhosted.org/packages/12/16/0b83fccc158218aca75a7aa33e97441df737950734246b9fffa39301603d/lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11", size = 5252745, upload-time = "2026-04-18T04:32:50.427Z" }, - { url = "https://files.pythonhosted.org/packages/dd/ee/12e6c1b39a77666c02eaa77f94a870aaf63c4ac3a497b2d52319448b01c6/lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4", size = 5226822, upload-time = "2026-04-18T04:32:53.437Z" }, - { url = "https://files.pythonhosted.org/packages/34/20/c7852904858b4723af01d2fc14b5d38ff57cb92f01934a127ebd9a9e51aa/lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3", size = 3594026, upload-time = "2026-04-18T04:27:31.903Z" }, - { url = "https://files.pythonhosted.org/packages/02/05/d60c732b56da5085175c07c74b2df4e6d181b0c9a61e1691474f06ef4b39/lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7", size = 4025114, upload-time = "2026-04-18T04:27:34.077Z" }, - { url = "https://files.pythonhosted.org/packages/c2/df/c84dcc175fd690823436d15b41cb920cd5ba5e14cd8bfb00949d5903b320/lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39", size = 3667742, upload-time = "2026-04-18T04:27:38.45Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d4/9326838b59dc36dfae42eec9656b97520f9997eee1de47b8316aaeed169c/lxml-6.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d", size = 8570663, upload-time = "2026-04-18T04:27:48.253Z" }, - { url = "https://files.pythonhosted.org/packages/d8/a4/053745ce1f8303ccbb788b86c0db3a91b973675cefc42566a188637b7c40/lxml-6.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93", size = 4624024, upload-time = "2026-04-18T04:27:52.594Z" }, - { url = "https://files.pythonhosted.org/packages/90/97/a517944b20f8fd0932ad2109482bee4e29fe721416387a363306667941f6/lxml-6.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d", size = 4930895, upload-time = "2026-04-18T04:32:56.29Z" }, - { url = "https://files.pythonhosted.org/packages/94/7c/e08a970727d556caa040a44773c7b7e3ad0f0d73dedc863543e9a8b931f2/lxml-6.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a", size = 5093820, upload-time = "2026-04-18T04:32:58.94Z" }, - { url = "https://files.pythonhosted.org/packages/88/ee/2a5c2aa2c32016a226ca25d3e1056a8102ea6e1fe308bf50213586635400/lxml-6.1.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105", size = 5005790, upload-time = "2026-04-18T04:33:01.272Z" }, - { url = "https://files.pythonhosted.org/packages/e3/38/a0db9be8f38ad6043ab9429487c128dd1d30f07956ef43040402f8da49e8/lxml-6.1.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485", size = 5630827, upload-time = "2026-04-18T04:33:04.036Z" }, - { url = "https://files.pythonhosted.org/packages/31/ba/3c13d3fc24b7cacf675f808a3a1baabf43a30d0cd24c98f94548e9aa58eb/lxml-6.1.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814", size = 5240445, upload-time = "2026-04-18T04:33:06.87Z" }, - { url = "https://files.pythonhosted.org/packages/55/ba/eeef4ccba09b2212fe239f46c1692a98db1878e0872ae320756488878a94/lxml-6.1.0-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32", size = 5350121, upload-time = "2026-04-18T04:33:09.365Z" }, - { url = "https://files.pythonhosted.org/packages/7e/01/1da87c7b587c38d0cbe77a01aae3b9c1c49ed47d76918ef3db8fc151b1ca/lxml-6.1.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad", size = 4694949, upload-time = "2026-04-18T04:33:11.628Z" }, - { url = "https://files.pythonhosted.org/packages/a1/88/7db0fe66d5aaf128443ee1623dec3db1576f3e4c17751ec0ef5866468590/lxml-6.1.0-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54", size = 5243901, upload-time = "2026-04-18T04:33:13.95Z" }, - { url = "https://files.pythonhosted.org/packages/00/a8/1346726af7d1f6fca1f11223ba34001462b0a3660416986d37641708d57c/lxml-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d", size = 5048054, upload-time = "2026-04-18T04:33:16.965Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b7/85057012f035d1a0c87e02f8c723ca3c3e6e0728bcf4cb62080b21b1c1e3/lxml-6.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69", size = 4777324, upload-time = "2026-04-18T04:33:19.832Z" }, - { url = "https://files.pythonhosted.org/packages/75/6c/ad2f94a91073ef570f33718040e8e160d5fb93331cf1ab3ca1323f939e2d/lxml-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d", size = 5645702, upload-time = "2026-04-18T04:33:22.436Z" }, - { url = "https://files.pythonhosted.org/packages/3b/89/0bb6c0bd549c19004c60eea9dc554dd78fd647b72314ef25d460e0d208c6/lxml-6.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5", size = 5232901, upload-time = "2026-04-18T04:33:26.21Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d9/d609a11fb567da9399f525193e2b49847b5a409cdebe737f06a8b7126bdc/lxml-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d", size = 5261333, upload-time = "2026-04-18T04:33:28.984Z" }, - { url = "https://files.pythonhosted.org/packages/a6/3a/ac3f99ec8ac93089e7dd556f279e0d14c24de0a74a507e143a2e4b496e7c/lxml-6.1.0-cp312-cp312-win32.whl", hash = "sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f", size = 3596289, upload-time = "2026-04-18T04:27:42.819Z" }, - { url = "https://files.pythonhosted.org/packages/f2/a7/0a915557538593cb1bbeedcd40e13c7a261822c26fecbbdb71dad0c2f540/lxml-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366", size = 3997059, upload-time = "2026-04-18T04:27:46.764Z" }, - { url = "https://files.pythonhosted.org/packages/92/96/a5dc078cf0126fbfbc35611d77ecd5da80054b5893e28fb213a5613b9e1d/lxml-6.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819", size = 3659552, upload-time = "2026-04-18T04:27:51.133Z" }, - { url = "https://files.pythonhosted.org/packages/f2/88/55143966481409b1740a3ac669e611055f49efd68087a5ce41582325db3e/lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842", size = 3930134, upload-time = "2026-04-18T04:32:35.008Z" }, - { url = "https://files.pythonhosted.org/packages/b5/97/28b985c2983938d3cb696dd5501423afb90a8c3e869ef5d3c62569282c0f/lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c", size = 4210749, upload-time = "2026-04-18T04:36:03.626Z" }, - { url = "https://files.pythonhosted.org/packages/29/67/dfab2b7d58214921935ccea7ce9b3df9b7d46f305d12f0f532ac7cf6b804/lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de", size = 4318463, upload-time = "2026-04-18T04:36:06.309Z" }, - { url = "https://files.pythonhosted.org/packages/32/a2/4ac7eb32a4d997dd352c32c32399aae27b3f268d440e6f9cfa405b575d2f/lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635", size = 4251124, upload-time = "2026-04-18T04:36:09.056Z" }, - { url = "https://files.pythonhosted.org/packages/33/ef/d6abd850bb4822f9b720cfe36b547a558e694881010ff7d012191e8769c6/lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037", size = 4401758, upload-time = "2026-04-18T04:36:11.803Z" }, - { url = "https://files.pythonhosted.org/packages/40/44/3ee09a5b60cb44c4f2fbc1c9015cfd6ff5afc08f991cab295d3024dcbf2d/lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace", size = 3508860, upload-time = "2026-04-18T04:32:48.619Z" }, -] - -[[package]] -name = "mako" -version = "1.3.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/62/791b31e69ae182791ec67f04850f2f062716bbd205483d63a215f3e062d3/mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a", size = 400219, upload-time = "2026-04-28T19:01:08.512Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, -] - -[[package]] -name = "mapie" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "scikit-learn" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/35/ed/f1be8cd32f3bf00cfc4453b6313f7ae552f64fe19c48d60dc00a3b82b39e/mapie-1.4.0.tar.gz", hash = "sha256:d924b3aebae36e19127e861da348b631eb609afb8c28e235521a7977e894819e", size = 15269060, upload-time = "2026-04-30T15:34:26.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/e8/2c36847996dd64d5c78a56ffb903a6f74855145d8833b8dc4c26814db366/mapie-1.4.0-py3-none-any.whl", hash = "sha256:3a810b140eedb39f3fb072d78b058e97682e45e4c791ad2dbf8ffbd2671b98b9", size = 265371, upload-time = "2026-04-30T15:34:23.906Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, @@ -2850,93 +1468,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, ] -[[package]] -name = "marshmallow" -version = "3.26.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095, upload-time = "2025-12-22T06:53:53.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964, upload-time = "2025-12-22T06:53:51.801Z" }, -] - -[[package]] -name = "mashumaro" -version = "3.14" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/47/0a450b281bef2d7e97ec02c8e1168d821e283f58e02e6c403b2bb4d73c1c/mashumaro-3.14.tar.gz", hash = "sha256:5ef6f2b963892cbe9a4ceb3441dfbea37f8c3412523f25d42e9b3a7186555f1d", size = 166160, upload-time = "2024-10-23T21:48:40.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/35/8d63733a2c12149d0c7663c29bf626bdbeea5f0ff963afe58a42b4810981/mashumaro-3.14-py3-none-any.whl", hash = "sha256:c12a649599a8f7b1a0b35d18f12e678423c3066189f7bc7bd8dd431c5c8132c3", size = 92183, upload-time = "2024-10-23T21:48:38.334Z" }, -] - -[package.optional-dependencies] -msgpack = [ - { name = "msgpack" }, -] - [[package]] name = "matplotlib" -version = "3.10.9" +version = "3.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "contourpy" }, { name = "cycler" }, { name = "fonttools" }, { name = "kiwisolver" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "packaging" }, { name = "pillow" }, { name = "pyparsing" }, { name = "python-dateutil" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/1b/4be5be87d43d327a0cf4de1a56e86f7f84c89312452406cf122efe2839e6/matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358", size = 34811233, upload-time = "2026-04-24T00:14:13.539Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/8c/290f021104741fea63769c31494f5324c0cd249bf536a65a4350767b1f22/matplotlib-3.10.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:68cfdcede415f7c8f5577b03303dd94526cdb6d11036cecdc205e08733b2d2bb", size = 8306860, upload-time = "2026-04-24T00:12:01.207Z" }, - { url = "https://files.pythonhosted.org/packages/51/18/325cd32ece1120d1da51cc4e4294c6580190699490183fc2fe8cb6d61ec5/matplotlib-3.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfca0129678bd56379db26c52b5d77ed7de314c047492fbdc763aa7501710cfb", size = 8199254, upload-time = "2026-04-24T00:12:04.239Z" }, - { url = "https://files.pythonhosted.org/packages/79/db/e28c1b83e3680740aa78925f5fb2ae4d16207207419ad75ea9fe604f8676/matplotlib-3.10.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e436d155fa8a3399dc62683f8f5d0e2e50d25d0144a73edd73f82eec8f4abfb", size = 8777092, upload-time = "2026-04-24T00:12:06.793Z" }, - { url = "https://files.pythonhosted.org/packages/55/fa/3ce7adfe9ba101748f465211660d9c6374c876b671bdb8c2bb6d347e8b94/matplotlib-3.10.9-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56fc0bd271b00025c6edfdc7c2dcd247372c8e1544971d62e1dc7c17367e8bf9", size = 9595691, upload-time = "2026-04-24T00:12:09.706Z" }, - { url = "https://files.pythonhosted.org/packages/36/c4/6960a76686ed668f2c60f84e9799ba4c0d56abdb36b1577b60c1d061d1ec/matplotlib-3.10.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5a6104ed666402ba5106d7f36e0e0cdca4e8d7fa4d39708ca88019e2835a2eb", size = 9659771, upload-time = "2026-04-24T00:12:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/7e/0d/271aace3342157c64700c9ff4c59c7b392f3dbab393692e8db6fbe7ab96c/matplotlib-3.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:d730e984eddf56974c3e72b6129c7ca462ac38dc624338f4b0b23eb23ecba00f", size = 8205112, upload-time = "2026-04-24T00:12:15.773Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ee/cb57ad4754f3e7b9174ce6ce66d9205fb827067e48a9f58ac09d7e7d6b77/matplotlib-3.10.9-cp311-cp311-win_arm64.whl", hash = "sha256:51bf0ddbdc598e060d46c16b5590708f81a1624cefbaaf62f6a81bf9285b8c80", size = 8132310, upload-time = "2026-04-24T00:12:18.645Z" }, - { url = "https://files.pythonhosted.org/packages/35/c6/5581e26c72233ebb2a2a6fed2d24fb7c66b4700120b813f51b0555acf0b6/matplotlib-3.10.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0c3c28d9fbcc1fe7a03be236d73430cf6409c41fb2383a7ac52fe932b072cb1", size = 8319908, upload-time = "2026-04-24T00:12:21.323Z" }, - { url = "https://files.pythonhosted.org/packages/b7/18/4880dd762e40cd360c1bf06e890c5a97b997e91cb324602b1a19950ad5ce/matplotlib-3.10.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cb28c2bd769aa3e98322c6ab09854cbcc52ab69d2759d681bba3e327b2b320", size = 8216016, upload-time = "2026-04-24T00:12:23.4Z" }, - { url = "https://files.pythonhosted.org/packages/32/91/d024616abdba99e83120e07a20658976f6a343646710760c4a51df126029/matplotlib-3.10.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ae20801130378b82d647ff5047c07316295b68dc054ca6b3c13519d0ea624285", size = 8789336, upload-time = "2026-04-24T00:12:26.096Z" }, - { url = "https://files.pythonhosted.org/packages/5c/04/030a2f61ef2158f5e4c259487a92ac877732499fb33d871585d89e03c42d/matplotlib-3.10.9-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c63ebcd8b4b169eb2f5c200552ae6b8be8999a005b6b507ed76fb8d7d674fe2", size = 9604602, upload-time = "2026-04-24T00:12:29.052Z" }, - { url = "https://files.pythonhosted.org/packages/fc/c2/541e4d09d87bb6b5830fc28b4c887a9a8cf4e1c6cee698a8c05552ae2003/matplotlib-3.10.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d75d11c949914165976c621b2324f9ef162af7ebf4b057ddf95dd1dba7e5edcf", size = 9670966, upload-time = "2026-04-24T00:12:32.131Z" }, - { url = "https://files.pythonhosted.org/packages/04/a1/4571fc46e7702de8d0c2dc54ad1b2f8e29328dea3ee90831181f7353d93c/matplotlib-3.10.9-cp312-cp312-win_amd64.whl", hash = "sha256:d091f9d758b34aaaaa6331d13574bf01891d903b3dec59bfff458ef7551de5d6", size = 8217462, upload-time = "2026-04-24T00:12:35.226Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d0/2269edb12aa30c13c8bcc9382892e39943ce1d28aab4ec296e0381798e81/matplotlib-3.10.9-cp312-cp312-win_arm64.whl", hash = "sha256:10cc5ce06d10231c36f40e875f3c7e8050362a4ee8f0ee5d29a6b3277d57bb42", size = 8136688, upload-time = "2026-04-24T00:12:37.442Z" }, - { url = "https://files.pythonhosted.org/packages/63/e2/9f66ca6a651a52abfe0d4964ce01439ed34f3f1e119de10ff3a07f403043/matplotlib-3.10.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:42fb814efabe95c06c1994d8ab5a8385f43a249e23badd3ba931d4308e5bca20", size = 8304420, upload-time = "2026-04-24T00:14:04.57Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e8/467c03568218792906aa87b5e7bb379b605e056ed0c74fe00c051786d925/matplotlib-3.10.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba", size = 8197981, upload-time = "2026-04-24T00:14:07.233Z" }, - { url = "https://files.pythonhosted.org/packages/6f/87/afead29192170917537934c6aff4b008c805fff7b1ccea0c79120d96beda/matplotlib-3.10.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3fc0364dfbe1d07f6d15c5ebd0c5bf89e126916e5a8667dd4a7a6e84c36653d4", size = 8774002, upload-time = "2026-04-24T00:14:09.816Z" }, -] - -[[package]] -name = "matplotlib-inline" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, -] - -[[package]] -name = "mdit-py-plugins" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d8/3d/e0e8d9d1cee04f758120915e2b2a3a07eb41f8cf4654b4734788a522bcd1/mdit_py_plugins-0.6.0.tar.gz", hash = "sha256:2436f14a7295837ac9228a36feeabda867c4abc488c8d019ad5c0bda88eee040", size = 56025, upload-time = "2026-05-07T12:20:42.295Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/d6/48f5b9e44e2e760855d7b489b1317cd7620e82dcb73197961e5cc1391348/mdit_py_plugins-0.6.0-py3-none-any.whl", hash = "sha256:f7e7a25d8b616fee99cb1e330da73451d11a8061baf39bb9663ab9ce0e005b90", size = 66655, upload-time = "2026-05-07T12:20:41.226Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a2/78f662f1b18968531f67d3fcde1b7ea8496920bacd4f16ddb5b79d112e46/matplotlib-3.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f857524b442f0f36e641868ce2171aafa88cb0bc0644f4e1d8a5df9b32649fef", size = 9436261, upload-time = "2026-06-12T02:27:34.161Z" }, + { url = "https://files.pythonhosted.org/packages/5e/92/044f1de43901310202f4c79acf4f141be53b2ca8d8380e2fcefb3d523a75/matplotlib-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:57baa92fdc82948ed716eae6d2579d4d6f40965cd8d2f416755b4a72580a3233", size = 9264669, upload-time = "2026-06-12T02:27:37.413Z" }, + { url = "https://files.pythonhosted.org/packages/53/f4/f0b4f9ba7ec14a7af8151f3ad71ecfe3561e6ba38cfab1db3681ba4ca112/matplotlib-3.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630eee0e67d35cce2019a0e670719f4816e3b86aff0fa72729f6c69786fceb45", size = 10021076, upload-time = "2026-06-12T02:27:39.926Z" }, + { url = "https://files.pythonhosted.org/packages/d7/33/4d679c6dcd594a156542080ac907ddccf7b09ca11655c4b28eca8e9ee5da/matplotlib-3.11.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5106c444d0bf966eee2853548c03772af4ab7199118e086c62fbac8ccb07c055", size = 10828999, upload-time = "2026-06-12T02:27:42.433Z" }, + { url = "https://files.pythonhosted.org/packages/07/74/0a3683802037d8cd013144d77c247219b47f2aabace6fdde74faa12bacf7/matplotlib-3.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d7aea652b58e686444079be3376ef546bffa1eee9b9bb9c472b9fcf6cf410d3", size = 10913103, upload-time = "2026-06-12T02:27:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/970fcbf381e82ec66fdf5da8ea76e2e9240f61a24011ce9fd1d42c37ac2d/matplotlib-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:70a5b3e9a5dab708c0f039709ae7c68d5b4d254e291ef76492cdba230c8bb5e4", size = 9310945, upload-time = "2026-06-12T02:27:46.867Z" }, + { url = "https://files.pythonhosted.org/packages/14/4e/6e7cfed23611265ded53806852343b5c59339e506e84c474a9b5afc3b249/matplotlib-3.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:3d68266213e73823ac3be90615bab0cf31f88851e114cdb1dd25dacf3b01e1a7", size = 8999304, upload-time = "2026-06-12T02:27:48.798Z" }, + { url = "https://files.pythonhosted.org/packages/da/17/f5276b496c61477a6c4fc5e7401f4bfe1c2e5ef7c6cd67896f2ade3809cb/matplotlib-3.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06b5872e9cf11adc8f589ded3ce11bc3e1061ad498259664fabc1f6615beb918", size = 9449976, upload-time = "2026-06-12T02:27:50.989Z" }, + { url = "https://files.pythonhosted.org/packages/82/34/bdd77418adb2178a1d59f044bd67bfebb115896e91b840b8a197eb3f4f4e/matplotlib-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0515d495124be3124340e59f164d901ed4484e2246a5b74cfa483cac3b80bd97", size = 9279307, upload-time = "2026-06-12T02:27:53.247Z" }, + { url = "https://files.pythonhosted.org/packages/94/95/7f522393c88313336b20d70fc849555757b2e5febc22b83b3a3f0fd4bce9/matplotlib-3.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5f93a1d21981bfb802ded0d77a0caa92d4342a47d45754fac77e314a506344", size = 10031353, upload-time = "2026-06-12T02:27:55.215Z" }, + { url = "https://files.pythonhosted.org/packages/87/ce/8f25a0e3186aefd61913e7467d1b999465bcd0d0c03ac695c1b26ca559b7/matplotlib-3.11.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41635d7909d19e52e924a521dde6d8f670b0f53ab1d0e8c331fa831554f681d1", size = 10839232, upload-time = "2026-06-12T02:27:57.746Z" }, + { url = "https://files.pythonhosted.org/packages/85/c2/db15da2bbdf9e3ca66df7db8e2c33a1dfed67be24a24d2c878efaaff01d6/matplotlib-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94f5000f67ca9faa300863ea17f8bce9175cb67b88bec4bc7780502d53dd7c9e", size = 10923899, upload-time = "2026-06-12T02:28:00.223Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2f/a58a4443a4d052a4ea77557478336aefc26c7981f6408d37adba763aa758/matplotlib-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ac6f1ef39f3d0f9e2463303013094992cdbe0f85f43bc54155bc472b2042768e", size = 9329528, upload-time = "2026-06-12T02:28:02.27Z" }, + { url = "https://files.pythonhosted.org/packages/61/0f/4b669589d47733b97ab9df4b58d6fc1e68acb5ea42a928dc7cbdd6bf5871/matplotlib-3.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:9dd11fb612ce7bc60b1de5b4fc87ff959d22317b5de42aabf392f66f97af22eb", size = 9003413, upload-time = "2026-06-12T02:28:04.49Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c2/f5da6cd37ed6871f5c9b3c0507ddb69f14d6c36fac4541e4e0c60cb8cdfc/matplotlib-3.11.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81ae77077a1e16d37a5b61096ccb07c8d90a99b518fa8256b8f21578932f2f62", size = 9434094, upload-time = "2026-06-12T02:29:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/f8/07/56f66906e0f87a0c6d0d0acbd34dbc9432b1931d8f26ef618bd6f92932a9/matplotlib-3.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ddef37840695f5eef65f9f070fe2d2f510f584c2156203f9f622a5b0584efffd", size = 9262183, upload-time = "2026-06-12T02:29:11.283Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d8/c4ecab06b7ea36a570c4f3bd2d48d1799fd5d9174470e45c2194199431e7/matplotlib-3.11.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf662e5ac5707658cb931e19972c4bd99f7b4f8b7bf79d3c821d239fa6b71e64", size = 10015653, upload-time = "2026-06-12T02:29:13.251Z" }, ] [[package]] @@ -2948,144 +1514,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "mistune" -version = "3.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/84/620cc3f7e3adf6f5067e10f4dbae71295d8f9e16d5d3f9ef97c40f2f592c/mistune-3.2.1.tar.gz", hash = "sha256:7c8e5501d38bac1582e067e46c8343f17d57ea1aaa735823f3aba1fd59c88a28", size = 98003, upload-time = "2026-05-03T14:33:22.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl", hash = "sha256:78cdb0ba5e938053ccf63651b352508d2efa9411dc8810bfb05f2dc5140c0048", size = 53749, upload-time = "2026-05-03T14:33:20.551Z" }, -] - -[[package]] -name = "mlflow" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "alembic" }, - { name = "cryptography" }, - { name = "docker" }, - { name = "flask" }, - { name = "flask-cors" }, - { name = "graphene" }, - { name = "gunicorn", marker = "sys_platform != 'win32'" }, - { name = "huey" }, - { name = "matplotlib" }, - { name = "mlflow-skinny" }, - { name = "mlflow-tracing" }, - { name = "numpy" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "skops" }, - { name = "sqlalchemy" }, - { name = "waitress", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/69/d71afc475fa7e7b22bb27392247d2a3015c9da202dea44f150a54be4bd67/mlflow-3.13.0.tar.gz", hash = "sha256:a95198d592a8a15fad3db7f56b228acc9422c09f0daa7c6c976a9996ab73c3e2", size = 10086808, upload-time = "2026-06-01T05:55:09.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/1f/d44140128356f2f5db37f9fb4d9da31123d839d49f3fe00a079cd35bfe20/mlflow-3.13.0-py3-none-any.whl", hash = "sha256:7ca9cb2f623f300dabadaf5e985c85af77c5db3d7c36f56769d22c101b132f6c", size = 10788121, upload-time = "2026-06-01T05:55:06.86Z" }, -] - -[[package]] -name = "mlflow-skinny" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "click" }, - { name = "cloudpickle" }, - { name = "databricks-sdk" }, - { name = "fastapi" }, - { name = "gitpython" }, - { name = "importlib-metadata" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "sqlparse" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "uvicorn" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/13/840db21a4f46ebe6ba9837a38bc93d748e23b6b61986799c8040cd4bf728/mlflow_skinny-3.13.0.tar.gz", hash = "sha256:d2273bfa21f776359f7d6ab2267967e3a6732a5fb00996ad433d0e777dfa3b71", size = 2814837, upload-time = "2026-06-01T05:54:54.175Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/fd/f2739de1b6a09da981927aa90db87340cbe4b3cf6cd175fd5e6e4366208e/mlflow_skinny-3.13.0-py3-none-any.whl", hash = "sha256:ced3d9a580564fae093d14732df8531fb180574f6483d4c642b6083879eb86fc", size = 3365675, upload-time = "2026-06-01T05:54:52.166Z" }, -] - -[[package]] -name = "mlflow-tracing" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "databricks-sdk" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/b0/5912313e895e6ce02f1f67110164d147593d1b5379a4767a30a5b9e730c5/mlflow_tracing-3.13.0.tar.gz", hash = "sha256:42c435b0fdcab00f1865cab4a52f7a85a2a08d68a959f36bcf90a1c9fe65db0a", size = 1393079, upload-time = "2026-06-01T05:54:44.906Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/fd/2c53ebc2f7fbb34ed4a3913a71ff53962e6be35fa02f7cee1f52c77388cd/mlflow_tracing-3.13.0-py3-none-any.whl", hash = "sha256:2f8187ce2b1af7419be71d2d8ab5fec53d207d4b8d703cd15e5db64939098d72", size = 1664279, upload-time = "2026-06-01T05:54:42.911Z" }, -] - -[[package]] -name = "more-itertools" -version = "10.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "msgpack" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" }, - { url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" }, - { url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" }, - { url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" }, - { url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" }, - { url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" }, - { url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" }, - { url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" }, - { url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" }, - { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, - { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, - { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, - { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, - { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, - { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -3131,49 +1559,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] -[[package]] -name = "multiprocess" -version = "0.70.19" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/aa/714635c727dbfc251139226fa4eaf1b07f00dc12d9cd2eb25f931adaf873/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7", size = 144743, upload-time = "2026-01-19T06:47:24.562Z" }, - { url = "https://files.pythonhosted.org/packages/0f/e1/155f6abf5e6b5d9cef29b6d0167c180846157a4aca9b9bee1a217f67c959/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e", size = 144738, upload-time = "2026-01-19T06:47:26.636Z" }, - { url = "https://files.pythonhosted.org/packages/af/cb/f421c2869d75750a4f32301cc20c4b63fab6376e9a75c8e5e655bdeb3d9b/multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45", size = 144741, upload-time = "2026-01-19T06:47:27.985Z" }, - { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, - { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, - { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, - { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, -] - [[package]] name = "mypy" -version = "1.19.1" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "ast-serialize" }, { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/af/4e516a05d3ca2eb9283e9ec45b2c02225c1514dd6da49fd3c9eaa6639370/mypy-2.3.0.tar.gz", hash = "sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e", size = 3988104, upload-time = "2026-07-13T11:34:53.387Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/47/6b3ebabd5474d9cdc170d1342fbf9dddc1b0ec13ec90bf9004ee6f391c31/mypy-1.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d8dfc6ab58ca7dda47d9237349157500468e404b17213d44fc1cb77bce532288", size = 13028539, upload-time = "2025-12-15T05:03:44.129Z" }, - { url = "https://files.pythonhosted.org/packages/5c/a6/ac7c7a88a3c9c54334f53a941b765e6ec6c4ebd65d3fe8cdcfbe0d0fd7db/mypy-1.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e3f276d8493c3c97930e354b2595a44a21348b320d859fb4a2b9f66da9ed27ab", size = 12083163, upload-time = "2025-12-15T05:03:37.679Z" }, - { url = "https://files.pythonhosted.org/packages/67/af/3afa9cf880aa4a2c803798ac24f1d11ef72a0c8079689fac5cfd815e2830/mypy-1.19.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2abb24cf3f17864770d18d673c85235ba52456b36a06b6afc1e07c1fdcd3d0e6", size = 12687629, upload-time = "2025-12-15T05:02:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/2d/46/20f8a7114a56484ab268b0ab372461cb3a8f7deed31ea96b83a4e4cfcfca/mypy-1.19.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a009ffa5a621762d0c926a078c2d639104becab69e79538a494bcccb62cc0331", size = 13436933, upload-time = "2025-12-15T05:03:15.606Z" }, - { url = "https://files.pythonhosted.org/packages/5b/f8/33b291ea85050a21f15da910002460f1f445f8007adb29230f0adea279cb/mypy-1.19.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f7cee03c9a2e2ee26ec07479f38ea9c884e301d42c6d43a19d20fb014e3ba925", size = 13661754, upload-time = "2025-12-15T05:02:26.731Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a3/47cbd4e85bec4335a9cd80cf67dbc02be21b5d4c9c23ad6b95d6c5196bac/mypy-1.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:4b84a7a18f41e167f7995200a1d07a4a6810e89d29859df936f1c3923d263042", size = 10055772, upload-time = "2025-12-15T05:03:26.179Z" }, - { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, - { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, - { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, - { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, - { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, - { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b9/d75b3082b05f1b3028828aeb18e74ae5ab0a0936051bbf1f32f59f654747/mypy-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3419d00717afbc5265b50dd14b1278f29ea4884dd398ab67873489ac093fd329", size = 14838725, upload-time = "2026-07-13T11:32:44.655Z" }, + { url = "https://files.pythonhosted.org/packages/a9/50/79a65c6ea6e115bc73296038a4543b2d5c91f07912b918a2c616a2514bba/mypy-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfca8ee88544090f86b6dcce05ec55d66eb48a762412ac2507810ba4bd793b6f", size = 13911128, upload-time = "2026-07-13T11:32:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/90/48/e11ed7716c26953ca321f726e452e374dbf81a6f2b8b212ec02af29b6b8f/mypy-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75cbb4b9ef04a0c84a957f07abc4504fbf64b8dcc145675101f2d3a78a4b1d6a", size = 14146742, upload-time = "2026-07-13T11:33:03.313Z" }, + { url = "https://files.pythonhosted.org/packages/06/72/6807565b1c4861ef66f7fdd98b51c61556356eab80235717b46c53bb8627/mypy-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:982e3d53dd23d0a4cef67dd66791fdbede0cf38f9eb617bf47663554c51e1e36", size = 15081418, upload-time = "2026-07-13T11:31:13.899Z" }, + { url = "https://files.pythonhosted.org/packages/00/80/1ea14c5d80e589e415973db3e47c78c2219a305b808b2b506395342c1d79/mypy-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85c5385b93012ffa3b31479ab579aef5415f4f3a32c6cf1ae07a984d2a0ff461", size = 15328164, upload-time = "2026-07-13T11:31:35.723Z" }, + { url = "https://files.pythonhosted.org/packages/37/28/8223157404a3d51920078459c37f80fbdc590e1d8ea049dc5ce48643022a/mypy-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:13b1b16e2fa39f3b2e33fb1c468abc7a69369fa2e886b4b87b5afc81472325cd", size = 11136472, upload-time = "2026-07-13T11:27:37.018Z" }, + { url = "https://files.pythonhosted.org/packages/6f/cc/ea27e5959c5f258585a756b252031f3b313583d81b5064b2bebc41d3706b/mypy-2.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:b5cd2f027a972a4a5f2278a11fac9747f5f81a53a30b714d74950b6807e55568", size = 10135800, upload-time = "2026-07-13T11:30:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/0e7e592619e2133596a47cdd642534b0456545c218430bd3b9d8fefdd1b1/mypy-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5", size = 15026523, upload-time = "2026-07-13T11:34:49.206Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/1e1731df090a857df2807177a4626863e5ac0f0256513c35780efe53986f/mypy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c", size = 14032189, upload-time = "2026-07-13T11:33:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/cab921f4a806e171f34113e6181dd23c55358ccf6a80741269ef594a410e/mypy-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491", size = 14198696, upload-time = "2026-07-13T11:32:12.767Z" }, + { url = "https://files.pythonhosted.org/packages/66/80/e6d008bb19fe446e3662d85e0e2717bf9f2d611a2164fb29d6e067dbf46c/mypy-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7", size = 15286904, upload-time = "2026-07-13T11:34:27.594Z" }, + { url = "https://files.pythonhosted.org/packages/db/83/94397c9293608a364aa03e8084fb34ede4ae976a260384b9b52929308135/mypy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3", size = 15528342, upload-time = "2026-07-13T11:34:07.819Z" }, + { url = "https://files.pythonhosted.org/packages/cf/96/d8b37d819adec6cfccfb1fd3afc1735d94717ddeafb45536db9c6943e09b/mypy-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c", size = 11218346, upload-time = "2026-07-13T11:28:27.745Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cd/cd9f725b19b19e5b530a154cf9bcf9e94279c5d55b3c34fb42b3aa48ea1b/mypy-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595", size = 10204525, upload-time = "2026-07-13T11:31:02.552Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/fdc54fe583ba3cafbcedfb70eeeaf03849f75b1827a07096c7bd996f582d/mypy-2.3.0-py3-none-any.whl", hash = "sha256:6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88", size = 2753292, upload-time = "2026-07-13T11:33:18.48Z" }, ] [[package]] @@ -3187,87 +1600,11 @@ wheels = [ [[package]] name = "narwhals" -version = "2.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/0e/3ad61eb87088cc4932e0d851531fa82f845a6230b68b091a0e298cc7e537/narwhals-2.21.0.tar.gz", hash = "sha256:7c6e7f50528e62b7a967dd864d7e117d2955d38d4f730653ce46a9861358e2dc", size = 633083, upload-time = "2026-05-08T12:29:02.587Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl", hash = "sha256:1e6617d0fca68ae1fda29e5397c4eaacd3ffc9fffe6bcd6ded0c690475e853be", size = 451943, upload-time = "2026-05-08T12:29:01.058Z" }, -] - -[[package]] -name = "nbclient" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "nbformat" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, -] - -[[package]] -name = "nbconvert" -version = "7.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "bleach", extra = ["css"] }, - { name = "defusedxml" }, - { name = "jinja2" }, - { name = "jupyter-core" }, - { name = "jupyterlab-pygments" }, - { name = "markupsafe" }, - { name = "mistune" }, - { name = "nbclient" }, - { name = "nbformat" }, - { name = "packaging" }, - { name = "pandocfilters" }, - { name = "pygments" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl", hash = "sha256:aa85c087b435e7bf1ffd03319f658e285f2b89eccab33bc1ba7025495ab3e7c8", size = 261927, upload-time = "2026-04-08T00:44:12.845Z" }, -] - -[[package]] -name = "nbformat" -version = "5.10.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastjsonschema" }, - { name = "jsonschema" }, - { name = "jupyter-core" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, -] - -[[package]] -name = "nbstripout" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nbformat" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f9/6f/b52c4da26babeb521078c08c78c3187a59197098ffc7a70b0fe76851813a/nbstripout-0.9.1.tar.gz", hash = "sha256:313bbb4217c8e38998567e5d790b6bd6c3a17a8c39073b205b84dadfc5d756dc", size = 32356, upload-time = "2026-02-21T16:19:55.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/16/e777eadfa0c0305878c36fae1d5e6db474fbb15dae202b9ec378809dfb4d/nbstripout-0.9.1-py3-none-any.whl", hash = "sha256:ca027ee45742ee77e4f8e9080254f9a707f1161ba11367b82fdf4a29892c759e", size = 19136, upload-time = "2026-02-21T16:19:54.868Z" }, -] - -[[package]] -name = "nest-asyncio" -version = "1.6.0" +version = "2.24.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, + { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, ] [[package]] @@ -3285,287 +1622,86 @@ version = "1.10.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, -] - -[[package]] -name = "notebook" -version = "7.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-builder" }, - { name = "jupyter-server" }, - { name = "jupyterlab" }, - { name = "jupyterlab-server" }, - { name = "notebook-shim" }, - { name = "tornado" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/44/d5c65783f490298473bb1c05722e05ee2256231389559c2c5ae0a3e5d975/notebook-7.6.0.tar.gz", hash = "sha256:ea13e79e601bf273074895fdfb17dd3f2da916d3c045e0b9c47d18b16ab62481", size = 5497344, upload-time = "2026-06-18T16:18:55.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/d1/e617c40db57ff40e75f43a7d4d1c305e3a54c053ab5cb0534a6c314664f9/notebook-7.6.0-py3-none-any.whl", hash = "sha256:98aa2811b54ac191321d5dfce12ca700f8a511a33a26e4de2fa106a357c43d6a", size = 5544575, upload-time = "2026-06-18T16:18:52.551Z" }, -] - -[[package]] -name = "notebook-shim" -version = "0.2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-server" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, -] - -[[package]] -name = "numpy" -version = "2.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, - { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, - { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, - { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, - { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, - { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, - { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, - { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, - { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.0.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, - { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, - { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.29.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, - { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "omegaconf" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "antlr4-python3-runtime" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.41.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fa/fc/b7564cbef36601aef0d6c9bc01f7badb64be8e862c2e1c3c5c3b43b53e4f/opentelemetry_api-1.41.1.tar.gz", hash = "sha256:0ad1814d73b875f84494387dae86ce0b12c68556331ce6ce8fe789197c949621", size = 71416, upload-time = "2026-04-24T13:15:38.262Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/59/3e7118ed140f76b0982ba4321bdaed1997a0473f9720de2d10788a577033/opentelemetry_api-1.41.1-py3-none-any.whl", hash = "sha256:a22df900e75c76dc08440710e51f52f1aa6b451b429298896023e60db5b3139f", size = 69007, upload-time = "2026-04-24T13:15:15.662Z" }, + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] [[package]] -name = "opentelemetry-proto" -version = "1.41.1" +name = "numpy" +version = "2.4.6" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] -sdist = { url = "https://files.pythonhosted.org/packages/99/e8/633c6d8a9c8840338b105907e55c32d3da1983abab5e52f899f72a82c3d1/opentelemetry_proto-1.41.1.tar.gz", hash = "sha256:4b9d2eb631237ea43b80e16c073af438554e32bc7e9e3f8ca4a9582f900020e5", size = 45670, upload-time = "2026-04-24T13:15:49.768Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/1e/5cd77035e3e82070e2265a63a760f715aacd3cb16dddc7efee913f297fcc/opentelemetry_proto-1.41.1-py3-none-any.whl", hash = "sha256:0496713b804d127a4147e32849fbaf5683fac8ee98550e8e7679cd706c289720", size = 72076, upload-time = "2026-04-24T13:15:32.542Z" }, + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, ] [[package]] -name = "opentelemetry-sdk" -version = "1.41.1" +name = "numpy" +version = "2.5.1" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] -sdist = { url = "https://files.pythonhosted.org/packages/58/d0/54ee30dab82fb0acda23d144502771ff76ef8728459c83c3e89ef9fb1825/opentelemetry_sdk-1.41.1.tar.gz", hash = "sha256:724b615e1215b5aeacda0abb8a6a8922c9a1853068948bd0bd225a56d0c792e6", size = 230180, upload-time = "2026-04-24T13:15:50.991Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/e7/a1420b698aad018e1cf60fdbaaccbe49021fb415e2a0d81c242f4c518f54/opentelemetry_sdk-1.41.1-py3-none-any.whl", hash = "sha256:edee379c126c1bce952b0c812b48fe8ff35b30df0eecf17e98afa4d598b7d85d", size = 180213, upload-time = "2026-04-24T13:15:33.767Z" }, + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.62b1" +name = "omegaconf" +version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/de/911ac9e309052aca1b20b2d5549d3db45d1011e1a610e552c6ccdd1b64f8/opentelemetry_semantic_conventions-0.62b1.tar.gz", hash = "sha256:c5cc6e04a7f8c7cdd30be2ed81499fa4e75bfbd52c9cb70d40af1f9cd3619802", size = 145750, upload-time = "2026-04-24T13:15:52.236Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/3d/e4b57b8d9008c6ebe0d5eff901f91d5700cf7bdb8c8863df817463a7fd5e/omegaconf-2.3.1.tar.gz", hash = "sha256:e5e7de64aeebeddaf8e6d3f7a783b32ac2a01c0fbd9c878012caecb891a1f42a", size = 3298472, upload-time = "2026-06-11T05:05:12.885Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/a6/83dc2ab6fa397ee66fba04fe2e74bdf7be3b3870005359ceb7689103c058/opentelemetry_semantic_conventions-0.62b1-py3-none-any.whl", hash = "sha256:cf506938103d331fbb78eded0d9788095f7fd59016f2bda813c3324e5a74a93c", size = 231620, upload-time = "2026-04-24T13:15:35.454Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/152509871bf30df6fc38569f52a2db9b55dd41aae957adae50a053ac7778/omegaconf-2.3.1-py3-none-any.whl", hash = "sha256:3d701d14e9a8828f1edd28bb70b725908b34277cdd72cf7d6a83f94dadc6b6a0", size = 79502, upload-time = "2026-06-11T05:05:09.954Z" }, ] [[package]] @@ -3574,57 +1710,20 @@ version = "0.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "matplotlib" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "ortools" }, { name = "pandas" }, { name = "ropwr" }, { name = "scikit-learn" }, - { name = "scipy" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ba/5f/94090d11b7bce6ddec38f20c3acd7f283f6968b9dd36c1270238f94fa660/optbinning-0.21.0.tar.gz", hash = "sha256:d608934cbf6fad8be896634d75ebdbc02f887d75c71ef46abbf2daa3b5d5afd9", size = 176827, upload-time = "2025-10-26T22:00:21.94Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3d/d4/baa2d9d5cd97578a727f554296309cfe45a3e7089968e9c603a175d47a2c/optbinning-0.21.0-py3-none-any.whl", hash = "sha256:973f12219f0b06ae17d6ae0486afd2780d0187e41c46e265b6a98bf26003ce3c", size = 214766, upload-time = "2025-10-26T22:00:19.127Z" }, ] -[[package]] -name = "optuna" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alembic" }, - { name = "colorlog" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "sqlalchemy" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/aa/05f5e3f662cc96a4c478fc3446b8ed6359825a2b504ecb614a9ac84e4a4d/optuna-4.9.0.tar.gz", hash = "sha256:b322e5cbdf1655fb84c37646c4a7a1f391de1b47806bbe222e015825d0a82b87", size = 485834, upload-time = "2026-06-01T06:23:30.424Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/f3/e5fcd5d9b15771ed6dc10e3a7eeddc672e418f4f4c4653d216cc1d857e2d/optuna-4.9.0-py3-none-any.whl", hash = "sha256:f52f3be6148654850c92a5860d398fd88ec6b2c84ab68d9c3d07dcff02e7afee", size = 425553, upload-time = "2026-06-01T06:23:28.804Z" }, -] - -[[package]] -name = "optuna-integration" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "optuna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/15/378c53a5fce6924be331ba7a7e0b3227f1d2ec8a0c8461c3d2475e5ba2e8/optuna_integration-4.9.0.tar.gz", hash = "sha256:c650615ea52259f518fee245be4f63be55c7495ffc7361d05e5054496ec427b8", size = 89927, upload-time = "2026-06-01T06:11:53.237Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/5b/05b6bc0e2fa1dd09d96f75d6cddfeaaaafde6028f89bbfb45e006a82a89a/optuna_integration-4.9.0-py3-none-any.whl", hash = "sha256:578dccd6657c12e0457750512ce718d4e3dab8730fa36dcd0b502eacfaa1e795", size = 103447, upload-time = "2026-06-01T06:11:51.831Z" }, -] - -[[package]] -name = "orderly-set" -version = "5.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/88/39c83c35d5e97cc203e9e77a4f93bf87ec89cf6a22ac4818fdcc65d66584/orderly_set-5.5.0.tar.gz", hash = "sha256:e87185c8e4d8afa64e7f8160ee2c542a475b738bc891dc3f58102e654125e6ce", size = 27414, upload-time = "2025-07-10T20:10:55.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, -] - [[package]] name = "orjson" version = "3.11.9" @@ -3665,60 +1764,54 @@ wheels = [ [[package]] name = "ortools" -version = "9.10.4067" +version = "9.11.4210" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, { name = "immutabledict" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pandas" }, { name = "protobuf" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/94/41/781bca55edecd30ac0c6717709a5fc2194184d168b1d438b431f4689002c/ortools-9.10.4067-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:37e5344d0f2695aeec756962e434b715977b59aa489f88f96f87582748e7ae32", size = 21542305, upload-time = "2024-05-08T08:25:15.871Z" }, - { url = "https://files.pythonhosted.org/packages/40/7c/4b8ab56c76f1ba10afabbb8bb7f3e2ecf057c500bd9f8beab24c44cb9c0e/ortools-9.10.4067-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a385302d465ca4683e30e40ad04c276d7d5b3569324e35e965795480ff787d0", size = 19832205, upload-time = "2024-05-08T08:25:20.052Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c8/72e6e9cea27bd7a1ef575f1d5aa759c28f20953b18c74d25438f8f13593c/ortools-9.10.4067-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56b7cdcbdcabd613cd297c770a56f41ccbfea72ca812a84adf536112a35a648f", size = 25097631, upload-time = "2024-05-08T08:25:25.653Z" }, - { url = "https://files.pythonhosted.org/packages/66/0c/5400c91bc600b52af80f53741214914f6bf30723da06fd174630dd440ed0/ortools-9.10.4067-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c035610952c35cf247ed0ab07b7caa618a37839904196aa6d21d9d5172bb1685", size = 26673926, upload-time = "2024-05-08T08:25:29.489Z" }, - { url = "https://files.pythonhosted.org/packages/03/58/3f90ba5dc0cb6caaed614b687adc2cd31be37ff5a6132220ce2f9cb60be1/ortools-9.10.4067-cp311-cp311-win_amd64.whl", hash = "sha256:6bb53096193bd887b92f73de6534e265ae71ff738b47d38efbeb97d584a9dcf7", size = 130205869, upload-time = "2024-05-08T08:25:37.985Z" }, - { url = "https://files.pythonhosted.org/packages/e5/2d/f6e5b66f3418acc6ae15b218d6828cc2c699de7a060f9e8571d52e7ea139/ortools-9.10.4067-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:25ec7b23d7aca612ad98e5b2613f1a547e6ae8f39bcef04d93da49b3495f6312", size = 21536240, upload-time = "2024-05-08T08:25:44.341Z" }, - { url = "https://files.pythonhosted.org/packages/7b/49/ca1042902f145b97ea66b15d7106fbcdfe04ac20b03664349c4817201656/ortools-9.10.4067-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d21ffff6676ab3f0dc1b50993ddc569537232ef9d91ae94431be565d35e911c5", size = 19828880, upload-time = "2024-05-08T08:25:47.681Z" }, - { url = "https://files.pythonhosted.org/packages/b7/9c/cb70caf731cb14a1abbe8ad91ced2f6be9acd9aab6e904e69af3f2aa9420/ortools-9.10.4067-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:751399f1122e1cc8e1c1e32d5e4f13add77230aae828a1bdbc5b685942aaa6ec", size = 25080047, upload-time = "2024-05-08T08:25:51.521Z" }, - { url = "https://files.pythonhosted.org/packages/89/a7/a319eac65db090a33e23055e5fe7798f0ecb2d4da37a9649db5f39bb6c39/ortools-9.10.4067-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc8980266522e2c203e07fe7c83f286408f475998dd2748531455576a4fcdfd", size = 26654437, upload-time = "2024-05-08T08:25:55.41Z" }, - { url = "https://files.pythonhosted.org/packages/d8/09/0a4bdc34ad730dbeae1a42ebe0e059c50cdd5497b1e6c224e084b47757d8/ortools-9.10.4067-cp312-cp312-win_amd64.whl", hash = "sha256:ea445366009535bb230339f631bfe2977b56ebf2ff51e75f04d90a4781d6499c", size = 130215073, upload-time = "2024-05-08T08:26:03.779Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/294d84c3e5c5bb4ca6143f33bb08648186f66605a278642745fed334ef59/ortools-9.11.4210-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:0f902caa1576d737714f6a4fa165db62469bce82115e250409607197b3b6b434", size = 22334086, upload-time = "2024-09-16T06:29:41.538Z" }, + { url = "https://files.pythonhosted.org/packages/2e/50/a7ed37e96c803b6a24de6b746753e1bdbd804cdeb38892379daa05ffed97/ortools-9.11.4210-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6f3e2869396dc6d8ee2d11b65d6f88f6386bb3ad64212c0ad7a6d32ddcb48ca", size = 20660318, upload-time = "2024-09-16T06:29:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/ce/af/31337925edd389a9d42b52b81198deeaecebe5dbd608389dfad389a63536/ortools-9.11.4210-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56c5844ff927ce3c5428159cdd01b7fdbe243e8062bf1dfdb2e0eb305a55a30", size = 26433101, upload-time = "2024-09-16T06:27:14.339Z" }, + { url = "https://files.pythonhosted.org/packages/f6/3b/f4e1200886852c09ba4ef8664ae7ff2323ab23326e27515d4b1159230f7e/ortools-9.11.4210-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5a17a37aeaa7d149e2fe8c8dfd5f09630ae28ad734a109ad55536605f8059df", size = 28072628, upload-time = "2024-09-16T06:27:18.352Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c4/3d790fd19d7c2bb8fa6e3a9c9184aa4f1bdbec9662cfa7ed99b6325b8ecd/ortools-9.11.4210-cp311-cp311-win_amd64.whl", hash = "sha256:d9b858f0273e19f81555428d54d407428d0a70a8cb5df2c320935bb735f2c6bb", size = 142454491, upload-time = "2024-09-16T06:30:36.174Z" }, + { url = "https://files.pythonhosted.org/packages/21/36/f6292656614fcf979bef92ca2dc117b9207185a816317a5bb4ff4ec1bf4b/ortools-9.11.4210-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:079bea08c6341dcfe3fb9586eb6edec6ae80f4ed16ed366fd7a46ef4b5709009", size = 22328002, upload-time = "2024-09-16T06:29:48.923Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7e/5497823b8a5b8d26d976425f0721e3bf0871deb05c65d61323523cabb39c/ortools-9.11.4210-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f55124f9d1afa6434d0d6de07c6a4eb836f29b00b3413d27138634d5d79b606", size = 20654740, upload-time = "2024-09-16T06:29:52.758Z" }, + { url = "https://files.pythonhosted.org/packages/45/06/a7a622da3d7b7a0e6f01ffef4229b9627f64c1c920cafa71ba9607524801/ortools-9.11.4210-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f48e3d4053a169440608d881c1abd2a706db885d9b0af85bf45b444a1fec244", size = 26414535, upload-time = "2024-09-16T06:27:22.432Z" }, + { url = "https://files.pythonhosted.org/packages/3c/38/b2e915c4b3f3bf966d2d73e69ee4c72a09452822ea912af0a506a32e2a48/ortools-9.11.4210-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03ecd32e5d760d48e59832ef6bf724f8cac95e4e40db72a7fb912abf7adcf931", size = 28053736, upload-time = "2024-09-16T06:27:26.59Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d8/9a4589e4ab103e809f2de718ddb6718f2547cdd785d3d7efd4da958a9c29/ortools-9.11.4210-cp312-cp312-win_amd64.whl", hash = "sha256:bc1b6e4cc0a121ef888481a99194765e6df72d4d3da81f928543171a2bac8cbb", size = 142461842, upload-time = "2024-09-16T06:30:46.349Z" }, ] [[package]] name = "osqp" -version = "1.1.1" +version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "joblib" }, - { name = "numpy" }, - { name = "scipy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "setuptools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/09/fb28f57d6eba067fbb6c941c9136f8d2e41b3d4fd4ddac643cf734210085/osqp-1.1.1.tar.gz", hash = "sha256:1719e6a88f2ec2bd5dab06131331d1433152fb222372832727d9eb5604d7acf4", size = 57059, upload-time = "2026-02-11T18:15:45.329Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/7e/b821d6819cbae6c7a0d76b5db23836c8b3b6db7684332414a1ff9cf737eb/osqp-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:48a6f62df0ec55a5a3a445e4143f51a813931f1e48ac006b15b7e5c9899e2937", size = 319461, upload-time = "2026-02-11T18:07:22.32Z" }, - { url = "https://files.pythonhosted.org/packages/81/ac/1d5361ddfad9e23ca56a0c580024ca1c99535f11b16d94b3d4c9fbc543af/osqp-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0e569d36955e1a69129f391bb27b2240b3b69d0bcff28e5d19446013dda59836", size = 301952, upload-time = "2026-02-11T18:07:23.542Z" }, - { url = "https://files.pythonhosted.org/packages/83/93/5c245d507d061a1532cc2bf0ad8ad907b464851b31f36d8316771a5b3c84/osqp-1.1.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fd6d87d5aa17161c43b95e44ab53c76cef466b851cc4ed32da658596cb0a0a1", size = 321822, upload-time = "2026-02-11T18:07:24.78Z" }, - { url = "https://files.pythonhosted.org/packages/f2/f7/351c42d12b2632d1512ec6e67f42c1367599f0b224d0621c94e9c76d06ab/osqp-1.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a0481c1f19f70eea9e9883b176eb37b64cd52525920c9ed765acb02411998ae", size = 343182, upload-time = "2026-02-11T18:07:26.165Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d9/ebf4d06bb13ecad4edcb2c6b491c47a798b58b9b5e71ba315e01a8696e2d/osqp-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7524d22e91a8381ed30eecbfdf82935528f84b3d8a1b5ad1f8dd84dff3fc07e", size = 310097, upload-time = "2026-02-11T18:07:27.683Z" }, - { url = "https://files.pythonhosted.org/packages/07/49/0ac80ef771eeb985b12a0c7fad86279a59269bce192110dc499890013b9f/osqp-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ca4e41477852f725293c666ffa5f795413151c9a14155a7750dff25d3107b851", size = 321455, upload-time = "2026-02-11T18:07:29.258Z" }, - { url = "https://files.pythonhosted.org/packages/4c/7f/39437ab1c73ea432c6510741f954ccec87bda9a2b9b3233439384363cd92/osqp-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25cd4e8995d18b65c54d1163769797665b9ca5a8a0009f1c4adf4dafe30e33be", size = 302230, upload-time = "2026-02-11T18:07:30.365Z" }, - { url = "https://files.pythonhosted.org/packages/d3/1f/83d45a3a194e7f58eec3bdfc2879390c56b18351936d569f3f6a79e5ca39/osqp-1.1.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed006d74017578fe98a2afad77f4bbeb096f2d64aa00f50809bb394a7bbd98bf", size = 322626, upload-time = "2026-02-11T18:07:31.508Z" }, - { url = "https://files.pythonhosted.org/packages/03/13/45db581134dab5942e723622bf994f30a79fe00029b90127625f3464bb3f/osqp-1.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61aca4a356d1555d13c26166c282b9b7985c6c715baf093f839e338e6b49aca0", size = 345768, upload-time = "2026-02-11T18:07:32.938Z" }, - { url = "https://files.pythonhosted.org/packages/9d/c7/4beccbd5993b3d92956d450eb24edd4055b66475d9802166160bc8dfd1ad/osqp-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:cd4ac30fd125e12ef5b67836442ebd3bb90925828816e0253e96a203197f5dc7", size = 310738, upload-time = "2026-02-11T18:08:54.392Z" }, -] - -[[package]] -name = "overrides" -version = "7.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/36/86/b585f53236dec60aba864e050778b25045f857e17f6e5ea0ae95fe80edd2/overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a", size = 22812, upload-time = "2024-01-27T21:01:33.423Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/7f/b441062e4766851fdf1d066857134c4c3bcf7e0089e4a1d007b6114cecd5/osqp-1.1.3.tar.gz", hash = "sha256:48f53ef5ec89e6ce99ffa955bc6ea0cf2eec09ea3d40905f0c9fadc939609907", size = 57816, upload-time = "2026-06-12T16:59:27.208Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/ab/fc8290c6a4c722e5514d80f62b2dc4c4df1a68a41d1364e625c35990fcf3/overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49", size = 17832, upload-time = "2024-01-27T21:01:31.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/66/f25cb13b72eedd74bcff17421761c520eabdade1b7166689d5b76f68d2b6/osqp-1.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0a168153f6058075d017e713d20eb4214c5daeb957b497c3c31340d0e63e1288", size = 326215, upload-time = "2026-06-12T16:58:43.489Z" }, + { url = "https://files.pythonhosted.org/packages/32/bb/95a38971abb93501e1870dcaa20c04877d19c59cb98bf9919ed85af32805/osqp-1.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fefecf59a4d84a63ff6fb34e24987f12d61c740e78cf7fb7864f427a3d1c74e", size = 308213, upload-time = "2026-06-12T16:58:44.784Z" }, + { url = "https://files.pythonhosted.org/packages/14/49/b941c22530f24538dbeb3a2d5b56faab4f853100f47250f07aeab943e671/osqp-1.1.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2b12ea5bdcf3750703c1fc1ea5d678f3f0fdddc6513c9352439f8075ca07bf4", size = 326653, upload-time = "2026-06-12T16:58:45.965Z" }, + { url = "https://files.pythonhosted.org/packages/18/b6/d569a65c4c371ee0ec866a17cafb21af2f1201390ac3aaa9c70672459fea/osqp-1.1.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6e6b384c356681b93c8698304b8910bd4a198e95664ab533cf5a422c126cd40", size = 351147, upload-time = "2026-06-12T16:58:47.352Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fe/4133eedbd73931a38fabb2ccae2a62e7f532e3d9e97ef51e0fcb04e668db/osqp-1.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:fb3734923e7606d8a5c14e41e1e710352aa6696e9f3f94e770def85d4633abf4", size = 314758, upload-time = "2026-06-12T16:58:48.508Z" }, + { url = "https://files.pythonhosted.org/packages/49/d2/f21ddf1b41ecc14784186129125c4e9c19d8f6f7ade0f2e0efd4a44dded6/osqp-1.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2b59e8094fe29d928d568cad0156a42daa44257ce142fb7808400016a62dc28c", size = 328512, upload-time = "2026-06-12T16:58:49.796Z" }, + { url = "https://files.pythonhosted.org/packages/de/45/1f99a9f25dc9534323b8ef9e578dc93acb33e2b762d5be267e1380b2dda5/osqp-1.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5620abbdb259190da6f0e5421b9fdfd9386b46690210ce82aea6102cc85c67f", size = 308891, upload-time = "2026-06-12T16:58:51.07Z" }, + { url = "https://files.pythonhosted.org/packages/d5/25/176dbb33c3c3605367c0de8720945c6bdd8f9b0d411e0a94ef747085f034/osqp-1.1.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e44ad08234cfbd6d9f2a823118547e683b038676887e096533935b9fcc15fd3c", size = 328135, upload-time = "2026-06-12T16:58:52.185Z" }, + { url = "https://files.pythonhosted.org/packages/77/89/96e2e4de3b89e7363931c70eb58bded22656f5512bde18eb7a9272ce48d8/osqp-1.1.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6ceef7fb4f332892b6e0bbc17323d5c9e028c3f9db726b62a15d876d0f81cc06", size = 354077, upload-time = "2026-06-12T16:58:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/c4/4b/6a1e4f5aa28117bc0a78be5e9f4071b6a11c00a33a01bc673c3faf2a0a65/osqp-1.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:e7e9bd3939e47e726673218a7b04b9784ecbf9ef0b8bc69107d20cc659836ac1", size = 316733, upload-time = "2026-06-12T16:58:54.397Z" }, ] [[package]] @@ -3732,177 +1825,94 @@ wheels = [ [[package]] name = "pandas" -version = "2.3.3" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, - { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, - { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, - { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, - { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, - { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, - { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, - { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, - { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, - { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, - { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, -] - -[[package]] -name = "pandera" -version = "0.31.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "pydantic" }, - { name = "typeguard" }, - { name = "typing-extensions" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/e1/aaa14c989ffd30c7acb293fb986715517ca2b5b435ca291432535bb2b111/pandera-0.31.1.tar.gz", hash = "sha256:c75aa3868af15d4f9aa613acf1a7f436a518f81f1eb658ad630c1dbe1dab0f13", size = 729785, upload-time = "2026-04-15T03:18:59.967Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/2d/6ea7cad2c2f0625c4120bef5353ab7cf749141bf1d070011cebb72f68189/pandera-0.31.1-py3-none-any.whl", hash = "sha256:f9f1ff4852804e1a181a4cb968e732a492f4b6dbefe051a8c5500da43d5c326d", size = 386913, upload-time = "2026-04-15T03:18:58.358Z" }, -] - -[[package]] -name = "pandocfilters" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/6f/3dd4940bbe001c06a65f88e36bad298bc7a0de5036115639926b0c5c0458/pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e", size = 8454, upload-time = "2024-01-18T20:08:13.726Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/af/4fbc8cab944db5d21b7e2a5b8e9211a03a79852b1157e2c102fcc61ac440/pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc", size = 8663, upload-time = "2024-01-18T20:08:11.28Z" }, -] - -[[package]] -name = "parsedatetime" -version = "2.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a8/20/cb587f6672dbe585d101f590c3871d16e7aec5a576a1694997a3777312ac/parsedatetime-2.6.tar.gz", hash = "sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455", size = 60114, upload-time = "2020-05-31T23:50:57.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/a4/3dd804926a42537bf69fb3ebb9fd72a50ba84f807d95df5ae016606c976c/parsedatetime-2.6-py3-none-any.whl", hash = "sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b", size = 42548, upload-time = "2020-05-31T23:50:56.315Z" }, -] - -[[package]] -name = "parso" -version = "0.8.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, -] - -[[package]] -name = "pathos" -version = "0.3.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dill" }, - { name = "multiprocess" }, - { name = "pox" }, - { name = "ppft" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/37/0c730979d3890f8096a86af2683fac74edd4d15cb037391098dca70dcb1d/pathos-0.3.5.tar.gz", hash = "sha256:8fe041b8545c5d3880a038f866022bdebf935e5cf68f56ed3407cb7e65193a61", size = 166975, upload-time = "2026-01-20T00:06:57.848Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/44/be2146c650ee9ca4d9a770c995f5c92c1ea52292dcf618ce1a336d3146dd/pathos-0.3.5-py3-none-any.whl", hash = "sha256:c95b04103c40a16c08db69cd4b5c52624d55208beadf1348681edece809ec4f8", size = 82248, upload-time = "2026-01-20T00:06:56.291Z" }, + { url = "https://files.pythonhosted.org/packages/42/16/b5c76b838fd9bf6ce84d3a53346b8874ec05c5f0040d75ef2c320100cd2a/pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98", size = 10338495, upload-time = "2026-05-11T18:52:11.558Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b0/a4ffc4ae74d2d822200dcc46898987d8eb6032d1e2b219cae39da6f5cbcc/pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639", size = 9938250, upload-time = "2026-05-11T18:52:17.005Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/3323601a52caee42c019e370090ca4544b241437240ca04f786cce82b0cf/pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2", size = 10770558, upload-time = "2026-05-11T18:52:19.865Z" }, + { url = "https://files.pythonhosted.org/packages/32/f1/bbecd2f867b97abebe0f9b53d750f862251b40337e061b36676ded3d920f/pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27", size = 11274611, upload-time = "2026-05-11T18:52:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/7f/4f/eafabf2d5fae5adf143b4d18d3706c5efdc368a7c4eb1ee8a3eddabbd0f6/pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824", size = 11784670, upload-time = "2026-05-11T18:52:25.4Z" }, + { url = "https://files.pythonhosted.org/packages/49/44/1eb20389301b57b19cc099a1c2f662501f72f08a65f912d05822613c1532/pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938", size = 12353708, upload-time = "2026-05-11T18:52:28.139Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/c321f13b5ba1819fc8dca456c7fce578da2dcfecff1abbf0eaddf8406c0f/pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea", size = 9907609, upload-time = "2026-05-11T18:52:30.982Z" }, + { url = "https://files.pythonhosted.org/packages/53/85/1b7f563ebc6357c27233a02a96b589bcce1fa9c6eb89fb4f0e56421d277e/pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a", size = 9165596, upload-time = "2026-05-11T18:52:33.334Z" }, + { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, ] [[package]] name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - -[[package]] -name = "pathvalidate" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, -] - -[[package]] -name = "pexpect" -version = "4.9.0" +version = "1.1.1" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, - { url = "https://files.pythonhosted.org/packages/47/a1/d5ff69e747374c33a3b53b9f98cca7889fce1fd03d79cdc4e1bccc6c5a87/pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65", size = 4695873, upload-time = "2026-04-01T14:42:46.452Z" }, - { url = "https://files.pythonhosted.org/packages/df/21/e3fbdf54408a973c7f7f89a23b2cb97a7ef30c61ab4142af31eee6aebc88/pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7", size = 6280168, upload-time = "2026-04-01T14:42:49.228Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f1/00b7278c7dd52b17ad4329153748f87b6756ec195ff786c2bdf12518337d/pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e", size = 8088188, upload-time = "2026-04-01T14:42:51.735Z" }, - { url = "https://files.pythonhosted.org/packages/ad/cf/220a5994ef1b10e70e85748b75649d77d506499352be135a4989c957b701/pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705", size = 6394401, upload-time = "2026-04-01T14:42:54.343Z" }, - { url = "https://files.pythonhosted.org/packages/e9/bd/e51a61b1054f09437acfbc2ff9106c30d1eb76bc1453d428399946781253/pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176", size = 7079655, upload-time = "2026-04-01T14:42:56.954Z" }, - { url = "https://files.pythonhosted.org/packages/6b/3d/45132c57d5fb4b5744567c3817026480ac7fc3ce5d4c47902bc0e7f6f853/pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b", size = 6503105, upload-time = "2026-04-01T14:42:59.847Z" }, - { url = "https://files.pythonhosted.org/packages/7d/2e/9df2fc1e82097b1df3dce58dc43286aa01068e918c07574711fcc53e6fb4/pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909", size = 7203402, upload-time = "2026-04-01T14:43:02.664Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2e/2941e42858ebb67e50ae741473de81c2984e6eff7b397017623c676e2e8d/pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808", size = 6378149, upload-time = "2026-04-01T14:43:05.274Z" }, - { url = "https://files.pythonhosted.org/packages/69/42/836b6f3cd7f3e5fa10a1f1a5420447c17966044c8fbf589cc0452d5502db/pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60", size = 7082626, upload-time = "2026-04-01T14:43:08.557Z" }, - { url = "https://files.pythonhosted.org/packages/c2/88/549194b5d6f1f494b485e493edc6693c0a16f4ada488e5bd974ed1f42fad/pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe", size = 2463531, upload-time = "2026-04-01T14:43:10.743Z" }, - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, - { url = "https://files.pythonhosted.org/packages/4e/b7/2437044fb910f499610356d1352e3423753c98e34f915252aafecc64889f/pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f", size = 5273969, upload-time = "2026-04-01T14:45:55.538Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f4/8316e31de11b780f4ac08ef3654a75555e624a98db1056ecb2122d008d5a/pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d", size = 4659674, upload-time = "2026-04-01T14:45:58.093Z" }, - { url = "https://files.pythonhosted.org/packages/d4/37/664fca7201f8bb2aa1d20e2c3d5564a62e6ae5111741966c8319ca802361/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f", size = 5288479, upload-time = "2026-04-01T14:46:01.141Z" }, - { url = "https://files.pythonhosted.org/packages/49/62/5b0ed78fce87346be7a5cfcfaaad91f6a1f98c26f86bdbafa2066c647ef6/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e", size = 7032230, upload-time = "2026-04-01T14:46:03.874Z" }, - { url = "https://files.pythonhosted.org/packages/c3/28/ec0fc38107fc32536908034e990c47914c57cd7c5a3ece4d8d8f7ffd7e27/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0", size = 5355404, upload-time = "2026-04-01T14:46:06.33Z" }, - { url = "https://files.pythonhosted.org/packages/5e/8b/51b0eddcfa2180d60e41f06bd6d0a62202b20b59c68f5a132e615b75aecf/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1", size = 6002215, upload-time = "2026-04-01T14:46:08.83Z" }, - { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5b/a02d30018abd97ced9f5a6c63d28597694a00d066516b9c1c6de45859fc9/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6", size = 4785266, upload-time = "2026-07-01T11:53:49.079Z" }, + { url = "https://files.pythonhosted.org/packages/c8/98/766667a4be768150a202836acd9fad19c06824ca86c4286d3cf6b274964e/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd", size = 6263814, upload-time = "2026-07-01T11:53:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2d/ede717bc1144f63886c21fd349bb95860b0d1a21149ff16f2bb362b612b6/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd", size = 6934408, upload-time = "2026-07-01T11:53:53.487Z" }, + { url = "https://files.pythonhosted.org/packages/a3/48/9c58b685e69d49c31af6c8eb9012055fab7e665785165c84796e2c73ce72/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c", size = 6337160, upload-time = "2026-07-01T11:53:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/ff/fa/dc2a5c0ba6df93f67c31d34b808b7ce440b40cdbf96f0b81cde1d1e6fa93/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5", size = 7045172, upload-time = "2026-07-01T11:53:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/86/a5/444817a4d4c4c2417df00513086ca196f388d8f9ef40c2e4ccd1ad1af54b/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b", size = 6472232, upload-time = "2026-07-01T11:53:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a", size = 7233653, upload-time = "2026-07-01T11:54:02.066Z" }, + { url = "https://files.pythonhosted.org/packages/fd/16/00f91ab7760dc842f5aad55217e80fc4a7067a0604535249bc8a2d6d9870/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26", size = 2568195, upload-time = "2026-07-01T11:54:04.622Z" }, + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" }, + { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" }, + { url = "https://files.pythonhosted.org/packages/08/14/f6ba68107680ffa74b39985f3f30884e41318fbc4250caa423c79b4788bb/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3", size = 5860358, upload-time = "2026-07-01T11:56:32.68Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] [[package]] name = "plotly" -version = "6.8.0" +version = "6.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "narwhals" }, { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/94/fd/d72c292d78aadb93d1a9bcd76bf3c678271040c7cf10abe5788b33040a39/plotly-6.8.0.tar.gz", hash = "sha256:e088e7ddc68d4f70e3d66659224727a45296d71d2b8284181862d3d8f1f0d88f", size = 6915161, upload-time = "2026-06-03T18:33:40.226Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/07/795c79dbce40c39bece88e69d049babbd23ffa95b5d117f248db8ea03abb/plotly-6.9.0.tar.gz", hash = "sha256:967ad33e8c704fed051800d11d985eb206a9c795c14206b30a6f463ed9c67d0d", size = 6919903, upload-time = "2026-07-09T14:55:59.982Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/14/abe5ce876ab5b66ee3c691bf537fcd43d037aea55d447aacf74630a8f31e/plotly-6.8.0-py3-none-any.whl", hash = "sha256:13c5c4a0f70b74cab1913eda0de49b826df5931708eb6f9c3010040614700ec8", size = 9902055, upload-time = "2026-06-03T18:33:34.26Z" }, + { url = "https://files.pythonhosted.org/packages/24/18/d8544811ab076f876c4892b3714f5b0dad335e1dc33aef826df431b8325d/plotly-6.9.0-py3-none-any.whl", hash = "sha256:36bebe2f1bb13884774fe61689c329071446f6ce4a8927fb1f0d6fb24f581236", size = 9909646, upload-time = "2026-07-09T14:55:55.421Z" }, ] [[package]] @@ -3914,24 +1924,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "pox" -version = "0.3.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/58/4385741dea1d74fe9dfed7ff42975266634ef8000f2c8e96717079c916b1/pox-0.3.7.tar.gz", hash = "sha256:0652f6f2103fe6d4ba638beb6fa8d3e8a68fd44bcb63315c614118515bcc3afb", size = 119442, upload-time = "2026-01-19T02:09:12.573Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/ac/4d5f104edf2aae2fec85567ec1d1969010de8124c5c45514f25e14900b65/pox-0.3.7-py3-none-any.whl", hash = "sha256:82a495249d13371314c1a5b5626a115e067ef5215d49530bf5efa37fbc25b56a", size = 29402, upload-time = "2026-01-19T02:09:11.024Z" }, -] - -[[package]] -name = "ppft" -version = "1.7.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/d2/281aa3466e948283d51b83238fb456f65e14f8ade5f8627822578cd2708f/ppft-1.7.8.tar.gz", hash = "sha256:5f696d4f397ae9b0af39b1faffb31957c51dfbc5a3815856472d4f4e872937ee", size = 136349, upload-time = "2026-01-19T03:03:13.439Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/e1/d1b380af6443e7c33aeb40617ebdc17c39dc30095235643cc518e3908203/ppft-1.7.8-py3-none-any.whl", hash = "sha256:d3e0e395215b14afc3dd5adfc032ccecfda2d4ed50dc7ded076cd1d215442843", size = 56759, upload-time = "2026-01-19T03:03:11.896Z" }, -] - [[package]] name = "pre-commit" version = "4.6.0" @@ -3948,27 +1940,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, ] -[[package]] -name = "prettytable" -version = "3.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/45/b0847d88d6cfeb4413566738c8bbf1e1995fad3d42515327ff32cc1eb578/prettytable-3.17.0.tar.gz", hash = "sha256:59f2590776527f3c9e8cf9fe7b66dd215837cca96a9c39567414cbc632e8ddb0", size = 67892, upload-time = "2025-11-14T17:33:20.212Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/8c/83087ebc47ab0396ce092363001fa37c17153119ee282700c0713a195853/prettytable-3.17.0-py3-none-any.whl", hash = "sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287", size = 34433, upload-time = "2025-11-14T17:33:19.093Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/fb/d9aa83ffe43ce1f19e557c0971d04b90561b0cfd50762aafb01968285553/prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28", size = 86035, upload-time = "2026-04-09T19:53:42.359Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9b/d4b1e644385499c8346fa9b622a3f030dce14cd6ef8a1871c221a17a67e7/prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1", size = 64154, upload-time = "2026-04-09T19:53:41.324Z" }, -] - [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -4026,17 +1997,16 @@ wheels = [ [[package]] name = "protobuf" -version = "6.33.6" +version = "5.26.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/e5/7e22ca7201a6b1040aae7787d0fe6cd970311da376a86fdafa5182be1d1b/protobuf-5.26.1.tar.gz", hash = "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51", size = 393518, upload-time = "2024-03-27T20:37:59.807Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/7a/eb/966e06bffa17a052f32d4f89f1721e358e47d554c6b0c73b487e5aa99f5e/protobuf-5.26.1-cp310-abi3-win32.whl", hash = "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23", size = 400018, upload-time = "2024-03-27T20:37:37.079Z" }, + { url = "https://files.pythonhosted.org/packages/8d/83/d70cb6fedb1f38318af01f0035f2103732630af0ca323c0198122b49323b/protobuf-5.26.1-cp310-abi3-win_amd64.whl", hash = "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33", size = 420904, upload-time = "2024-03-27T20:37:40.413Z" }, + { url = "https://files.pythonhosted.org/packages/1e/40/2eb2bf643d4b060b1602a25748b48d75431f4951be2470f8ae136952b3d3/protobuf-5.26.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d", size = 404043, upload-time = "2024-03-27T20:37:42.8Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ac/6f4f48aa5472bf4b58f962a0c910e26bb7b648c141435cbe32b797768985/protobuf-5.26.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca", size = 300876, upload-time = "2024-03-27T20:37:45.085Z" }, + { url = "https://files.pythonhosted.org/packages/2c/2a/d2741cad35fa5f06d9c59dda3274e5727ca11075dfd7de3f69c100efdcad/protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7", size = 302823, upload-time = "2024-03-27T20:37:46.908Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ba/1c8528299fe9082f9400e6f4eb021cb3311943b2bff9267d5e24e4b4a4d8/protobuf-5.26.1-py3-none-any.whl", hash = "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932", size = 161228, upload-time = "2024-03-27T20:37:58.154Z" }, ] [[package]] @@ -4055,65 +2025,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] -[[package]] -name = "ptyprocess" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, -] - [[package]] name = "pyarrow" -version = "24.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, - { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, - { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, - { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, - { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, - { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, - { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.3" +version = "25.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, + { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" }, + { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, ] [[package]] @@ -4223,58 +2154,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl", hash = "sha256:869c0efadd2708c0be1f916eb669f3d664ca684bc57ffb7ecc08e70d5e93fee6", size = 37087, upload-time = "2025-06-17T20:09:55.25Z" }, ] -[[package]] -name = "pyepo" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "configspace" }, - { name = "numpy" }, - { name = "pathos" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "torch" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/15/cc90a323384dd7183025b31d56d5511fdde80684da1e5755605331d24ed0/pyepo-1.1.1.tar.gz", hash = "sha256:2136c99333737a897d267b76c68ba12670b35fbce413fc08a50568fe11fc4ad8", size = 51231, upload-time = "2026-02-11T08:47:52.327Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/95/b76c9330445f5e2705adcfc8500062b3465ce5501786327ef6a4e45eb4db/pyepo-1.1.1-py3-none-any.whl", hash = "sha256:b00126004966a90e92236a9a699939157ccc60a3c1b5119d9cfc37b999ea08c7", size = 84175, upload-time = "2026-02-11T08:47:51.234Z" }, -] - [[package]] name = "pygit2" -version = "1.19.2" +version = "1.19.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/a4/10ce00feef5c43eddacab19ae6610c4d4ef3ab77e544e9ee938772cd1c17/pygit2-1.19.2.tar.gz", hash = "sha256:cbeb3dbca9ca6ee3d5ea5d02f5e844c2d6084a2d5d6621e3e06aa2b11c645bfd", size = 803448, upload-time = "2026-03-29T14:57:27.565Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/9c/388648963f4be4bde89e32ca1a6f60adabb5f782c0e78598790b56e41967/pygit2-1.19.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:70c7efc426bdae6b67465a03729b79277e7757a29a7d6550b40c18ed36cb7232", size = 5706937, upload-time = "2026-03-29T14:56:02.061Z" }, - { url = "https://files.pythonhosted.org/packages/02/4c/e89013ff45350affac11f5893b3b7b555be35d5f279ff89c1d9310872378/pygit2-1.19.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b96d6ed7251eef70cfd4126269f1044fa47bc6da6367300027c5e5d74789f7f", size = 5695668, upload-time = "2026-03-29T14:56:03.92Z" }, - { url = "https://files.pythonhosted.org/packages/6f/84/db7281407c4481b64559f400f87f60190cc59615637bbc6f0afb8681dd7d/pygit2-1.19.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f3235db6b553b8fb4d3c1dc86af9be1eab445f1d6c42f4ade5cf5f60efd333", size = 6034309, upload-time = "2026-03-29T14:56:05.282Z" }, - { url = "https://files.pythonhosted.org/packages/b9/06/d8623933341e79220ab0c14c1e2bc5a78645738ce62699f942065e8699c4/pygit2-1.19.2-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02a35d56126f82a303668f4198c138627b3e9820f9f1eec38fff0409be274b9e", size = 4637953, upload-time = "2026-03-29T14:56:06.589Z" }, - { url = "https://files.pythonhosted.org/packages/42/27/6b20c5d424297623b22737f54a5c67ea00d498b5d2ebb98d3175d01de10c/pygit2-1.19.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e59a2e9eddd59edf999403c266c891dfc171eb95939d229ed614bc21e0c95804", size = 5794511, upload-time = "2026-03-29T14:56:07.953Z" }, - { url = "https://files.pythonhosted.org/packages/ec/cb/75a09f2319dc8798d80085b059506462af95b576d782c447f502dc807553/pygit2-1.19.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d0d2437bd5f8dbd652e8a6c318cbcaa245c0528ee48f6d64f4aaef8fd9b36b93", size = 6039969, upload-time = "2026-03-29T14:56:09.441Z" }, - { url = "https://files.pythonhosted.org/packages/34/20/fd2ac2f397fed5fab1f2838f2460226734f5a616371a388d8b0d7c995b7f/pygit2-1.19.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:60d011496e57436b0c8e3fbd4d12745777427b3f33a60710ec3d94d2f76304b7", size = 5764242, upload-time = "2026-03-29T14:56:10.947Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c0/bee8c2fce9d577cdc167e82d8999a57f997c213b5226fbaa9b977e3ce95f/pygit2-1.19.2-cp311-cp311-win32.whl", hash = "sha256:9b0d5a44ca6d77a8c0e2526f6556d9b37cc85d44983ff3549bf5adbf95d289c4", size = 945852, upload-time = "2026-03-29T14:56:12.209Z" }, - { url = "https://files.pythonhosted.org/packages/e0/e9/9ecafac82a8729ebfc948636147235770354ee48956ad43ed628b8396a68/pygit2-1.19.2-cp311-cp311-win_amd64.whl", hash = "sha256:0d9c795155086c95ef890c87b50e02792146cfaede2c715698e6988a122373e7", size = 1163883, upload-time = "2026-03-29T14:56:13.28Z" }, - { url = "https://files.pythonhosted.org/packages/02/09/24f3f55ebda489755d757dd254a612caa19e6a2bd2cbc5ccf7127e4caa30/pygit2-1.19.2-cp311-cp311-win_arm64.whl", hash = "sha256:837f0a9a0093cbb213176284d29f0ab754ded3e5af967e7ec6419d590a7da92a", size = 969221, upload-time = "2026-03-29T14:56:14.441Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2d/4fdeb7c6e044588cef9f0fca8b93fbc40fcdb2dfc64367999f45e88c0e7e/pygit2-1.19.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf479077d48a60b09569a5bb50866d8609f434f8982058594b0d2e2950bd6fce", size = 5704810, upload-time = "2026-03-29T14:56:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/c7/d8/926415c996ca283c4f7ccf63322ea23135ff17ecd1d2faaba704f6b4d883/pygit2-1.19.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6e6e7eb5fb49203735627b8e1d410afe19e7d610c9a9733c11084fabd17f0920", size = 5696366, upload-time = "2026-03-29T14:56:17.241Z" }, - { url = "https://files.pythonhosted.org/packages/8c/5e/c0329db9c980552c5c853dc1e429d13d55c691749db0540ef6bc77c04a98/pygit2-1.19.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a810da2d108d6bd16115c72a1c3d69fa1528ef927719bdfc94d2cdbc4198288", size = 6035334, upload-time = "2026-03-29T14:56:19.555Z" }, - { url = "https://files.pythonhosted.org/packages/a6/39/e08003a59a4d58bbba923d1c2be683a84b7b30c7270a19ff3ea02f9558df/pygit2-1.19.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b8ae5a822afb2771cbacf7c75140e663bc801c44eaaf2e4017f850cb27227c", size = 4636920, upload-time = "2026-03-29T14:56:20.93Z" }, - { url = "https://files.pythonhosted.org/packages/c9/3c/d3a9ed478add4cd77403416897459750ef2f6a06d8febe452ba03f5b7a27/pygit2-1.19.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:330430b6c1a3e6d45d1f5f950734d37d849c07924b5b0475cd995a7e541e6ab1", size = 5798652, upload-time = "2026-03-29T14:56:22.773Z" }, - { url = "https://files.pythonhosted.org/packages/d4/15/5440f00005db1769062ecc9fab7059ac7ae89217a06e1976734f53c2d040/pygit2-1.19.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b7f165d1ddfa1e0f205c1115ee10f5fea700fd3584c727b0d61a57192238449", size = 6041142, upload-time = "2026-03-29T14:56:24.618Z" }, - { url = "https://files.pythonhosted.org/packages/9d/5b/b9f9979a56606a661c0cff24c7aa6f5b1ad34118e116b7d67295be42aaad/pygit2-1.19.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e46ec6a97a5c43704473e42a926f7f20f9934ceef4f4891660313f573c4f0ab8", size = 5769220, upload-time = "2026-03-29T14:56:26.423Z" }, - { url = "https://files.pythonhosted.org/packages/30/81/9594e604eb19ae02f6a2023840e25574b0abcfc8d58c03cf96c59dd4ba72/pygit2-1.19.2-cp312-cp312-win32.whl", hash = "sha256:6b4de5469e88e7b069143f7a5d6336a4b3e7d911de4633ef18c113e416feb948", size = 946691, upload-time = "2026-03-29T14:56:27.813Z" }, - { url = "https://files.pythonhosted.org/packages/35/2d/c9bcdaef8f57ba0cdf129a6823f95cadd8ead002f38fba3465732c7517a8/pygit2-1.19.2-cp312-cp312-win_amd64.whl", hash = "sha256:f064748202928f4e882501521229e378e0b7b69b0e7c433cdb2626d007745973", size = 1164290, upload-time = "2026-03-29T14:56:29.13Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d4/6e9c98d227a8e816e2ffc7304f733e8b924afd8198b16888972fedbe05bd/pygit2-1.19.2-cp312-cp312-win_arm64.whl", hash = "sha256:222f439d751799dc74c3fa75f187abdbc415d12f9a091efa66f0c9ff51893d32", size = 969330, upload-time = "2026-03-29T14:56:30.363Z" }, - { url = "https://files.pythonhosted.org/packages/f8/47/8ca340fc8f0f5ec8ab9f8e96bb814a64a95f3836034fac00bb733c1f357c/pygit2-1.19.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4c2d397c887ff5a26b48ebd1bb9c66d2195ad377f0a44e05b79c462fff4040cd", size = 5649264, upload-time = "2026-03-29T14:57:19.193Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c4/8d26b20cb09ae862302b7b023e9089c70946cbdaf4f3cb8c1d4c7ba94a09/pygit2-1.19.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:69a0d377ee46110bbeea9e4191edee05132d1e7ac84b7cdebc640bc45868a2ec", size = 5646929, upload-time = "2026-03-29T14:57:20.717Z" }, - { url = "https://files.pythonhosted.org/packages/d3/35/b71ac88cda21ad440577543a317052eca9ab4f0119e9c2d74baa135731c4/pygit2-1.19.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57d113a3eb61621ce16ceaa4bae7a93ffe525fd69da905445a0cf798d3601815", size = 5562858, upload-time = "2026-03-29T14:57:22.962Z" }, - { url = "https://files.pythonhosted.org/packages/08/d9/c419105e997031a34a1a7d87e832a3a0e5a4c1501bc2784250d000b1c044/pygit2-1.19.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e0bc207abbef4d3be3bd37e0711e6974a148d41806fdc932aef9bb244b157c4", size = 5315754, upload-time = "2026-03-29T14:57:24.956Z" }, - { url = "https://files.pythonhosted.org/packages/e8/20/b52bc0ef2c5358d08e20e2d9fa8ae911283f3d20a7b9e52ec4a338b94983/pygit2-1.19.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:219c03bdbca59bd1df12b8bc7974b429872f4267aa2287ec0237c268593c0c5e", size = 1132796, upload-time = "2026-03-29T14:57:26.411Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/a6/44/415aa93422b4bfc21a6448acb7e16280d5f33a9a3fae38a384e37b046ae4/pygit2-1.19.3.tar.gz", hash = "sha256:a543e6d4ebb43825564935758dc234e770016fed673b84370d46ae9580558831", size = 810489, upload-time = "2026-06-13T08:06:04.982Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/3a/36e8827f45d9daf79ea94cbc6148e6243df1fcabc7cac55cd152c5fa99a8/pygit2-1.19.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59bca67a3f2af724659494bec9fed288b99242be13d76349d0170177512712fa", size = 5710933, upload-time = "2026-06-13T08:04:44.661Z" }, + { url = "https://files.pythonhosted.org/packages/38/77/9e87bbe723c50d6841740449c6e80a7aa63da01af59ca84b1169450dea7c/pygit2-1.19.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bb2e65666356f072d774f935aa0858f68ea34644d7a1467ea4918470748a2722", size = 5699637, upload-time = "2026-06-13T08:04:46.804Z" }, + { url = "https://files.pythonhosted.org/packages/2c/89/d0ba3e7f3cea1ec7c359d0456fc2cc3d5010c2a8a5a26da5298284e32a1c/pygit2-1.19.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0424822de48cb466f54c45c4d73924be201fd8ad347ccf0545fe7e601f056ebb", size = 6035985, upload-time = "2026-06-13T08:04:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/29/d0/63e37c75f8822b6d2cb0e1877a5e3de7e29a15e3ed2fe48c37b47164ff71/pygit2-1.19.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6aeb43e12fde33ec981209da90f66076e956538e457e4a50ba1665f97e3b2ea", size = 4639470, upload-time = "2026-06-13T08:04:50.116Z" }, + { url = "https://files.pythonhosted.org/packages/52/30/23ad5ac73519086eab4953cd9999eb24d57625826fcd28de2e2b57a9bf62/pygit2-1.19.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eaf59b7f6435c276850d34392503b55e703392094ff4c87c3c0a4161b3d90133", size = 5795950, upload-time = "2026-06-13T08:04:51.696Z" }, + { url = "https://files.pythonhosted.org/packages/ec/1d/cad147ad317bc61e549784db8ee89391e868dd3da77174d57c6a960e4241/pygit2-1.19.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e536656457dd306d0ebae2ac442c2c8f8798ea2333095b85391bc3423f4ac1b3", size = 6041627, upload-time = "2026-06-13T08:04:53.232Z" }, + { url = "https://files.pythonhosted.org/packages/af/03/bcef734c7c46ab8702ce7f06f30f45716a7d8f97ec45fafdb20baef40473/pygit2-1.19.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38d52f8d30bf969e3bfcab22cf5cdc4dc69590e06e8e948b1bb25cccb3bbbe3d", size = 5766485, upload-time = "2026-06-13T08:04:55.035Z" }, + { url = "https://files.pythonhosted.org/packages/42/b6/1f04fab2433f6ad10bd8820c1296f557d2fc9131abaff73d89783d8b85ac/pygit2-1.19.3-cp311-cp311-win32.whl", hash = "sha256:8b7a0a16372b245d065efb10a4ef5d61dc08794a69fa8ed6dd1ff918409b7d84", size = 944332, upload-time = "2026-06-13T08:04:56.358Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/a88310e96bfbd47feea5359d2380d90326ec5de3a114772ae7e515f3653d/pygit2-1.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:2b8f748836ac00fd785373ef8f5c9be1ed509e357cb270cc3f5674c0ad44d44d", size = 1254403, upload-time = "2026-06-13T08:04:57.447Z" }, + { url = "https://files.pythonhosted.org/packages/97/ed/aa3a0a44c2d4419b1c01804ea1ff232bdbcaaf1a1a7f1f8274ff62d9534e/pygit2-1.19.3-cp311-cp311-win_arm64.whl", hash = "sha256:98f2241ef66ec97ab21a2a1a6851c1305929c7a9e9e3e94db5faeb5d2ed06aa2", size = 969498, upload-time = "2026-06-13T08:04:58.863Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9c/9bc9a8d727b2ae8ae77306ecfb77a5dcf836da4997d4f7053c893f8e0d11/pygit2-1.19.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d696ee240cc8b84e747b04443d85768c859ef0b88abe4a3505dc0b8e2a953ca0", size = 5708797, upload-time = "2026-06-13T08:05:00.097Z" }, + { url = "https://files.pythonhosted.org/packages/63/7e/9099ea2f90791549185f8ac737a8b448e05cf3882cc79928a380be9bf38c/pygit2-1.19.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86318a704bef836067fa9ac06c56591738d548dcdf2ea24ec227257e3f8115f8", size = 5700295, upload-time = "2026-06-13T08:05:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/39/84/e9610f041dc43699fca8af55050f9bf9fe0d340ab9902ab8e3fa67bece23/pygit2-1.19.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc3ddb3e5d16cf662d2d0faeb53259e31ae584cc990cd76df909beeb4a36736a", size = 6036822, upload-time = "2026-06-13T08:05:03.473Z" }, + { url = "https://files.pythonhosted.org/packages/64/bd/419ec17df3f1f1182de8c8fe800b698ff2a489cc50ec2d34990092bf3d61/pygit2-1.19.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73820fb34339fe3c52731d1c46c298a3239ee75b61bc3a2d9b45f8b8873acc47", size = 4638181, upload-time = "2026-06-13T08:05:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/90/5f/aecca9c7f4ffdc1facc4b2402ea40073848e67c8c607a5f0698e1e8a61e4/pygit2-1.19.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f6ad50608f51b053bf6c321ae2c31cc57f15e4131a4609261c44e29bdae7cb", size = 5799910, upload-time = "2026-06-13T08:05:06.48Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2a/e131af7752f75f70e04f7f3ae724481d0e2db9c6c688a2081ffb5948657e/pygit2-1.19.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b2c63c7a371dbd5825e5fb7069b759c79aa5dec97806d49cb2c9f9a8a373209d", size = 6042766, upload-time = "2026-06-13T08:05:08.052Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/978ee5233379b2aa725c290a7cc3baca9b141f4fc37d55d9a473631b529b/pygit2-1.19.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:977eb52cd4134ff60d3faa168327a03f8724278c38bfcd347d8e3a11041ee9cd", size = 5771434, upload-time = "2026-06-13T08:05:09.398Z" }, + { url = "https://files.pythonhosted.org/packages/26/33/7c506492fffe3e92d9e911fae43a4b4a3dee43682b71b7793d586c25acfc/pygit2-1.19.3-cp312-cp312-win32.whl", hash = "sha256:14a2734d793d2d937d3bfc9d35b280ec83fc97a91b1934b1013554d41e6d3d70", size = 945030, upload-time = "2026-06-13T08:05:11.187Z" }, + { url = "https://files.pythonhosted.org/packages/82/ff/57ad08d1e87ae6e4208f80923ce1a6ea8a03ea776376e39d64816637b168/pygit2-1.19.3-cp312-cp312-win_amd64.whl", hash = "sha256:a7caeaf46aaa8e51c512af9ac1377f388e4836c9d042e134bbc5e22519fbd1bf", size = 1254825, upload-time = "2026-06-13T08:05:12.315Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cf/476c217a4ab3c9b4abf2506fb1135e54b19d205ce7ddecb1d961bf93a9dc/pygit2-1.19.3-cp312-cp312-win_arm64.whl", hash = "sha256:2aafa010e3ac227913f398c8a680bce419e396dfe651685fffb779ed1c109a00", size = 969650, upload-time = "2026-06-13T08:05:13.438Z" }, + { url = "https://files.pythonhosted.org/packages/fe/52/7645ca57a4987f202487baf86a1d8feaa8c4b45e9bed53415074f17cee94/pygit2-1.19.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a1fe33a036aa2172934cafbdd17e2a0f3a73477727fc147e94f881bfe3961c2", size = 5653015, upload-time = "2026-06-13T08:05:57.131Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/4be52d0e6e58c401b18d54603530fbf0f6988f63081f39b62fffca3cf54f/pygit2-1.19.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:09db53b92c546e621218fb82a97149dc7a155a8323a3dbce6b5048da70b0c00c", size = 5650631, upload-time = "2026-06-13T08:05:58.632Z" }, + { url = "https://files.pythonhosted.org/packages/ab/fb/bf14acbc711b40d1a1517c429d8c2a6b0cb6e8513bf1b25f13c79dbb04fa/pygit2-1.19.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c14d61de0c803188a0481d394a8ba61c7e1c38210a1e525097f18a0ad563009e", size = 5563807, upload-time = "2026-06-13T08:06:00.232Z" }, + { url = "https://files.pythonhosted.org/packages/45/b8/826cdd7f00bfcf52ae46c3ad405c8279c5431fb91b4171021366efec6a40/pygit2-1.19.3-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:806d1e22d3828ea6767fb025f355374d6dd562161a0312f995723ab15e1de42d", size = 5316588, upload-time = "2026-06-13T08:06:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/62/47/9a5d33c552d52e554efe7a39b5022b5455709d75093089ac867750379df0/pygit2-1.19.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:5af3422728f61958b6a0faaf7de754ecae5bed689adc31d1b727da3c697a752b", size = 1224962, upload-time = "2026-06-13T08:06:03.897Z" }, ] [[package]] @@ -4321,9 +2234,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pypdf" +version = "6.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/72/7dfd5ff1c9c37de97a731701f51af091325f123d9d4270361c9c69e4431f/pypdf-6.14.2.tar.gz", hash = "sha256:7873f502fe4385e79539b21d872392dc0c4e3714327c15881cbc7fbfd1f95b25", size = 6491182, upload-time = "2026-06-23T14:18:30.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e6/136aa8993a2ae7214e0b0ef2edaa0d2e08d1d4e4982635b08a835ff31ec8/pypdf-6.14.2-py3-none-any.whl", hash = "sha256:3f07891af76dc002657e04993ab9b4de81de29f9013b9761d0b7968bff12e946", size = 349514, upload-time = "2026-06-23T14:18:28.867Z" }, +] + [[package]] name = "pytest" -version = "9.0.3" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -4332,9 +2254,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] @@ -4365,15 +2287,15 @@ wheels = [ [[package]] name = "python-discovery" -version = "1.3.0" +version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/e0/cc5a8653e9a24f6cf84768f05064aa8ed5a83dcefd5e2a043db14a1c5f44/python_discovery-1.3.0.tar.gz", hash = "sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0", size = 63925, upload-time = "2026-05-05T14:38:39.769Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/81/58c70036dffeccb7fe7d79d6260c69f7a28272bbd3909c29a01ea9422744/python_discovery-1.4.4.tar.gz", hash = "sha256:5cad33982d412c1f3ffb8f9ca4ea292c9680bca3942451d30b69c37fce53a4a3", size = 72212, upload-time = "2026-07-08T23:06:50.691Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/d4/24d543ab8b8158b7f5a97113c831205f5c900c92c8762b1e7f44b7ea0405/python_discovery-1.3.0-py3-none-any.whl", hash = "sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f", size = 33124, upload-time = "2026-05-05T14:38:38.539Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ae/84bc0d2440c95772272bb6f4b3d09ccf08b2898fce89b3d4f969a9fc74e9/python_discovery-1.4.4-py3-none-any.whl", hash = "sha256:abebe9120b43453b68c908acfb1e72a19d1a959ed2cb620ad38fc57d08056dbe", size = 34181, upload-time = "2026-07-08T23:06:49.402Z" }, ] [[package]] @@ -4385,68 +2307,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] -[[package]] -name = "python-json-logger" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/ff/3cc9165fd44106973cd7ac9facb674a65ed853494592541d339bdc9a30eb/python_json_logger-4.1.0.tar.gz", hash = "sha256:b396b9e3ed782b09ff9d6e4f1683d46c83ad0d35d2e407c09a9ebbf038f88195", size = 17573, upload-time = "2026-03-29T04:39:56.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/be/0631a861af4d1c875f096c07d34e9a63639560a717130e7a87cbc82b7e3f/python_json_logger-4.1.0-py3-none-any.whl", hash = "sha256:132994765cf75bf44554be9aa49b06ef2345d23661a96720262716438141b6b2", size = 15021, upload-time = "2026-03-29T04:39:55.266Z" }, -] - -[[package]] -name = "python-slugify" -version = "8.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "text-unidecode" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, -] - -[[package]] -name = "pytimeparse" -version = "1.1.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/5d/231f5f33c81e09682708fb323f9e4041408d8223e2f0fb9742843328778f/pytimeparse-1.1.8.tar.gz", hash = "sha256:e86136477be924d7e670646a98561957e8ca7308d44841e21f5ddea757556a0a", size = 9403, upload-time = "2018-05-18T17:40:42.76Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/b4/afd75551a3b910abd1d922dbd45e49e5deeb4d47dc50209ce489ba9844dd/pytimeparse-1.1.8-py2.py3-none-any.whl", hash = "sha256:04b7be6cc8bd9f5647a6325444926c3ac34ee6bc7e69da4367ba282f076036bd", size = 9969, upload-time = "2018-05-18T17:40:41.28Z" }, -] - -[[package]] -name = "pytz" -version = "2026.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, -] - [[package]] name = "pywin32" -version = "311" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/af/449a6a91e5d6db51420875c54f6aff7c97a86a3b13a0b4f1a5c13b988de3/pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151", size = 8697031, upload-time = "2025-07-14T20:13:13.266Z" }, - { url = "https://files.pythonhosted.org/packages/51/8f/9bb81dd5bb77d22243d33c8397f09377056d5c687aa6d4042bea7fbf8364/pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503", size = 9508308, upload-time = "2025-07-14T20:13:15.147Z" }, - { url = "https://files.pythonhosted.org/packages/44/7b/9c2ab54f74a138c491aba1b1cd0795ba61f144c711daea84a88b63dc0f6c/pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2", size = 8703930, upload-time = "2025-07-14T20:13:16.945Z" }, - { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, - { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, -] - -[[package]] -name = "pywinpty" -version = "3.0.3" +version = "312" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/54/37c7370ba91f579235049dc26cd2c5e657d2a943e01820844ffc81f32176/pywinpty-3.0.3.tar.gz", hash = "sha256:523441dc34d231fb361b4b00f8c99d3f16de02f5005fd544a0183112bcc22412", size = 31309, upload-time = "2026-02-04T21:51:09.524Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/c3/3e75075c7f71735f22b66fab0481f2c98e3a4d58cba55cb50ba29114bcf6/pywinpty-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:dff25a9a6435f527d7c65608a7e62783fc12076e7d44487a4911ee91be5a8ac8", size = 2114430, upload-time = "2026-02-04T21:54:19.485Z" }, - { url = "https://files.pythonhosted.org/packages/8d/1e/8a54166a8c5e4f5cb516514bdf4090be4d51a71e8d9f6d98c0aa00fe45d4/pywinpty-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:fbc1e230e5b193eef4431cba3f39996a288f9958f9c9f092c8a961d930ee8f68", size = 236191, upload-time = "2026-02-04T21:50:36.239Z" }, - { url = "https://files.pythonhosted.org/packages/7c/d4/aeb5e1784d2c5bff6e189138a9ca91a090117459cea0c30378e1f2db3d54/pywinpty-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c9081df0e49ffa86d15db4a6ba61530630e48707f987df42c9d3313537e81fc0", size = 2113098, upload-time = "2026-02-04T21:54:37.711Z" }, - { url = "https://files.pythonhosted.org/packages/b9/53/7278223c493ccfe4883239cf06c823c56460a8010e0fc778eef67858dc14/pywinpty-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:15e79d870e18b678fb8a5a6105fd38496b55697c66e6fc0378236026bc4d59e9", size = 234901, upload-time = "2026-02-04T21:53:31.35Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f5/10a6e845a00fc5e7afd0a988b744f403d4d57162a28d160a093c4d9322f0/pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c", size = 6362659, upload-time = "2026-06-04T07:49:21.349Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/dcd2d62b5944b6d5db53413a5899016ccd57ffcb7278f3f81655d25d2027/pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a", size = 6928825, upload-time = "2026-06-04T07:49:23.934Z" }, + { url = "https://files.pythonhosted.org/packages/b7/56/3cbb433fe4501cdba2eb9040f56a4e1a8243faa4186b25295564d1a7a79d/pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47", size = 6721875, upload-time = "2026-06-04T07:49:26.416Z" }, + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, ] [[package]] @@ -4476,49 +2347,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, ] -[[package]] -name = "pyzmq" -version = "27.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/5d/305323ba86b284e6fcb0d842d6adaa2999035f70f8c38a9b6d21ad28c3d4/pyzmq-27.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:226b091818d461a3bef763805e75685e478ac17e9008f49fce2d3e52b3d58b86", size = 1333328, upload-time = "2025-09-08T23:07:45.946Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a0/fc7e78a23748ad5443ac3275943457e8452da67fda347e05260261108cbc/pyzmq-27.1.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0790a0161c281ca9723f804871b4027f2e8b5a528d357c8952d08cd1a9c15581", size = 908803, upload-time = "2025-09-08T23:07:47.551Z" }, - { url = "https://files.pythonhosted.org/packages/7e/22/37d15eb05f3bdfa4abea6f6d96eb3bb58585fbd3e4e0ded4e743bc650c97/pyzmq-27.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c895a6f35476b0c3a54e3eb6ccf41bf3018de937016e6e18748317f25d4e925f", size = 668836, upload-time = "2025-09-08T23:07:49.436Z" }, - { url = "https://files.pythonhosted.org/packages/b1/c4/2a6fe5111a01005fc7af3878259ce17684fabb8852815eda6225620f3c59/pyzmq-27.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bbf8d3630bf96550b3be8e1fc0fea5cbdc8d5466c1192887bd94869da17a63e", size = 857038, upload-time = "2025-09-08T23:07:51.234Z" }, - { url = "https://files.pythonhosted.org/packages/cb/eb/bfdcb41d0db9cd233d6fb22dc131583774135505ada800ebf14dfb0a7c40/pyzmq-27.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:15c8bd0fe0dabf808e2d7a681398c4e5ded70a551ab47482067a572c054c8e2e", size = 1657531, upload-time = "2025-09-08T23:07:52.795Z" }, - { url = "https://files.pythonhosted.org/packages/ab/21/e3180ca269ed4a0de5c34417dfe71a8ae80421198be83ee619a8a485b0c7/pyzmq-27.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:bafcb3dd171b4ae9f19ee6380dfc71ce0390fefaf26b504c0e5f628d7c8c54f2", size = 2034786, upload-time = "2025-09-08T23:07:55.047Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b1/5e21d0b517434b7f33588ff76c177c5a167858cc38ef740608898cd329f2/pyzmq-27.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e829529fcaa09937189178115c49c504e69289abd39967cd8a4c215761373394", size = 1894220, upload-time = "2025-09-08T23:07:57.172Z" }, - { url = "https://files.pythonhosted.org/packages/03/f2/44913a6ff6941905efc24a1acf3d3cb6146b636c546c7406c38c49c403d4/pyzmq-27.1.0-cp311-cp311-win32.whl", hash = "sha256:6df079c47d5902af6db298ec92151db82ecb557af663098b92f2508c398bb54f", size = 567155, upload-time = "2025-09-08T23:07:59.05Z" }, - { url = "https://files.pythonhosted.org/packages/23/6d/d8d92a0eb270a925c9b4dd039c0b4dc10abc2fcbc48331788824ef113935/pyzmq-27.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:190cbf120fbc0fc4957b56866830def56628934a9d112aec0e2507aa6a032b97", size = 633428, upload-time = "2025-09-08T23:08:00.663Z" }, - { url = "https://files.pythonhosted.org/packages/ae/14/01afebc96c5abbbd713ecfc7469cfb1bc801c819a74ed5c9fad9a48801cb/pyzmq-27.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:eca6b47df11a132d1745eb3b5b5e557a7dae2c303277aa0e69c6ba91b8736e07", size = 559497, upload-time = "2025-09-08T23:08:02.15Z" }, - { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, - { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, - { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, - { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, - { url = "https://files.pythonhosted.org/packages/4c/c6/c4dcdecdbaa70969ee1fdced6d7b8f60cfabe64d25361f27ac4665a70620/pyzmq-27.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:18770c8d3563715387139060d37859c02ce40718d1faf299abddcdcc6a649066", size = 836265, upload-time = "2025-09-08T23:09:49.376Z" }, - { url = "https://files.pythonhosted.org/packages/3e/79/f38c92eeaeb03a2ccc2ba9866f0439593bb08c5e3b714ac1d553e5c96e25/pyzmq-27.1.0-pp311-pypy311_pp73-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ac25465d42f92e990f8d8b0546b01c391ad431c3bf447683fdc40565941d0604", size = 800208, upload-time = "2025-09-08T23:09:51.073Z" }, - { url = "https://files.pythonhosted.org/packages/49/0e/3f0d0d335c6b3abb9b7b723776d0b21fa7f3a6c819a0db6097059aada160/pyzmq-27.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53b40f8ae006f2734ee7608d59ed661419f087521edbfc2149c3932e9c14808c", size = 567747, upload-time = "2025-09-08T23:09:52.698Z" }, - { url = "https://files.pythonhosted.org/packages/a1/cf/f2b3784d536250ffd4be70e049f3b60981235d70c6e8ce7e3ef21e1adb25/pyzmq-27.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f605d884e7c8be8fe1aa94e0a783bf3f591b84c24e4bc4f3e7564c82ac25e271", size = 747371, upload-time = "2025-09-08T23:09:54.563Z" }, - { url = "https://files.pythonhosted.org/packages/01/1b/5dbe84eefc86f48473947e2f41711aded97eecef1231f4558f1f02713c12/pyzmq-27.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c9f7f6e13dff2e44a6afeaf2cf54cee5929ad64afaf4d40b50f93c58fc687355", size = 544862, upload-time = "2025-09-08T23:09:56.509Z" }, -] - [[package]] name = "qdldl" version = "0.1.9.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "scipy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/51/4e/452984a63df9421cf8e7d25e8e6a44832cf0247a5e7b65e437cd516a0f8f/qdldl-0.1.9.post1.tar.gz", hash = "sha256:da2016d541c26cefc79bca4d8b5bebfa00f35db19704abb20efbd1c08df3b4c7", size = 76295, upload-time = "2026-02-19T16:48:36.651Z" } wheels = [ @@ -4536,20 +2373,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/51/2a683eddd1f0cf50440e0d173ccc4cb500775a8449e27651963873cdb4df/qdldl-0.1.9.post1-cp312-cp312-win_arm64.whl", hash = "sha256:ba3e19399553821b5ceee0c082fdb4453d00a38bd420b76dddec92ce2a5a065c", size = 98901, upload-time = "2026-02-19T16:47:56.553Z" }, ] -[[package]] -name = "referencing" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -4565,51 +2388,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] -[[package]] -name = "requests-toolbelt" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, -] - -[[package]] -name = "rfc3339-validator" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, -] - -[[package]] -name = "rfc3986-validator" -version = "0.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/88/f270de456dd7d11dcc808abfa291ecdd3f45ff44e3b549ffa01b126464d0/rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055", size = 6760, upload-time = "2019-10-28T16:00:19.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242, upload-time = "2019-10-28T16:00:13.976Z" }, -] - -[[package]] -name = "rfc3987-syntax" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, -] - [[package]] name = "rich" version = "15.0.0" @@ -4629,65 +2407,17 @@ version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cvxpy" }, - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "scikit-learn" }, - { name = "scipy" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/d2/329bcd5e097601b1e89367dbb2a3ac94756c539544cbf5946c0e0995c1dd/ropwr-1.2.0.tar.gz", hash = "sha256:af0d8b7d6df279ce180f409660216ee3dd228f86a2127a1d9e7aa8f44adac8a2", size = 19372, upload-time = "2026-01-23T12:24:12.118Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/84/9a/b91950807ef5e4a6f461bd1d59a1277140f08c2c31508bc2143ec597e4df/ropwr-1.2.0-py3-none-any.whl", hash = "sha256:d7abb9bad6813850c3349d17515b37eed86e59e3d5c17aeae86115fde33f4dd9", size = 17510, upload-time = "2026-01-23T12:24:10.748Z" }, ] -[[package]] -name = "rpds-py" -version = "0.30.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, - { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, - { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, - { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, - { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, - { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, - { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, - { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, - { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, - { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, - { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, - { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, - { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, - { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, - { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, - { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, - { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, - { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, - { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, - { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, - { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, - { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, - { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, - { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, - { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, - { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, - { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, - { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, - { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, - { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, - { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, - { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, - { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, - { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, -] - [[package]] name = "ruamel-yaml" version = "0.19.1" @@ -4699,53 +2429,41 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/5f7ec371001337d8fa61701c186ff8b613ecac1651848c5950f4c4d5f2e9/ruff-0.15.16.tar.gz", hash = "sha256:d05e78d38c78caf020b03789e25106c93017db5a0cb6e2819885018c61343b78", size = 4714267, upload-time = "2026-06-04T16:33:09.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/42/53ef1c3953f157956db9bf7861e3bc50b9b887ce93300aa48cdba8336fe6/ruff-0.15.16-py3-none-linux_armv6l.whl", hash = "sha256:6ac3c0b3969cc6cf6b158c4e2f8f682acb58e7d700d8a44b65ecdc72d66ab0b2", size = 10709025, upload-time = "2026-06-04T16:32:51.935Z" }, - { url = "https://files.pythonhosted.org/packages/93/9a/a79159346f19134a956607754e57d8d128f7a4c00f4ad2f7514d224c172c/ruff-0.15.16-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:197c207ed75ffba54a0dec23db4aa939a27a3053073e085e0042433cbdc58e4a", size = 11063550, upload-time = "2026-06-04T16:32:42.24Z" }, - { url = "https://files.pythonhosted.org/packages/bc/72/3ce2ac000a5299ec238e01f51397b3b653c93b077d9b1bfe8715bb895f20/ruff-0.15.16-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3a39fec45ab316cc23e7558f23fea4a70403ddb5648ea9a4a3854a16973d0071", size = 10421345, upload-time = "2026-06-04T16:32:37.251Z" }, - { url = "https://files.pythonhosted.org/packages/b0/c2/cc7fad3ec9169373f5b6a18f1917b91080feec40c3f9658334a1d28e2f03/ruff-0.15.16-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba93191d79003116b95128c9d306e045200fdbd0bccb782b110f3cd1d4abc5cf", size = 10757217, upload-time = "2026-06-04T16:32:54.722Z" }, - { url = "https://files.pythonhosted.org/packages/69/d2/3474009eaa0a65b31fa7152a2fad5e2f050c640ceb1e6b02ee6922e94c82/ruff-0.15.16-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6ee4b90520630120ef032aa5cc10db483852dff950e78b1d717e2993a61ac8d", size = 10507035, upload-time = "2026-06-04T16:33:05.343Z" }, - { url = "https://files.pythonhosted.org/packages/ca/81/b7ae6ccbd11f0c8dc3d5d67fc4be9b57ff57ca86ba56152021378e1277f2/ruff-0.15.16-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e4215bc938bc3c8215c1472c1aa437e310fee20cd427335fec9d7e609563628", size = 11255291, upload-time = "2026-06-04T16:32:49.49Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e1/46e526f1a7cc90857ce6ddf25fbb77eb6568651ac38d71b033af07076dd5/ruff-0.15.16-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c8d26be963b090f10e29abc8b3e74a2a321f6fa34e02424e30b5af89350ecbb", size = 12124922, upload-time = "2026-06-04T16:33:07.821Z" }, - { url = "https://files.pythonhosted.org/packages/1a/da/5c791b088b596b24d0deb967fa28ae02ad751a140c0b9ea81c5ab915d6c0/ruff-0.15.16-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f198cf4123602a2280ed46c307bcbafe41758d6fee5b456b6b6058ca1514b3b4", size = 11332186, upload-time = "2026-06-04T16:33:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/72/11/5da87abe20047c8962361473923ebb2f62b595250126aadfad8c20649c1e/ruff-0.15.16-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb27515fa6240fb586ae82b901a59e67d24acff86f2190b433dc542fe0435aeb", size = 11373541, upload-time = "2026-06-04T16:32:47.007Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2a/8554754c23a854ae3fd6b507e36ad61ddb121e298c6d5d617dec94ed0f14/ruff-0.15.16-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a267c46ba1593fc26b8eecbea050b39d40c0b6bb7781ee11c90a02cd10032951", size = 11353014, upload-time = "2026-06-04T16:32:34.795Z" }, - { url = "https://files.pythonhosted.org/packages/62/25/62ea41529ec89f742ea3fed9cb1059c72877ec7cf9b9e99ac9cf3294d1d9/ruff-0.15.16-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:528c68f39a91498a8d50e91ff5985df3d105782bab49cc378e73ac26bff083e8", size = 10737467, upload-time = "2026-06-04T16:32:26.348Z" }, - { url = "https://files.pythonhosted.org/packages/90/17/334d3ad9de4d40f9dd58fdd09e35ce64553bb501e2f19a839e2fb6be14fc/ruff-0.15.16-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7ed55c58950df60589a9a7a5d2f8fa5f54ebd287163be805adfe6ee95a9de123", size = 10521910, upload-time = "2026-06-04T16:32:32.54Z" }, - { url = "https://files.pythonhosted.org/packages/4d/bd/3ac7c6ae77a885c1004b3dda2446ea401768d24f851c14b4ad4b24f6639c/ruff-0.15.16-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d482feaf51512b50f9790ceb417a56a61dd1e9d9bf967662b9ed27c01b34f53a", size = 10979190, upload-time = "2026-06-04T16:32:57.492Z" }, - { url = "https://files.pythonhosted.org/packages/33/d7/609546e6a413c3f216fbf2a50c928f97c80939154f6a0503114094a86191/ruff-0.15.16-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e15bc8c94513dae2a40cc9ef07c94fdd4ecc9e29dabebeebe170f952322c9e3", size = 11477014, upload-time = "2026-06-04T16:32:44.687Z" }, - { url = "https://files.pythonhosted.org/packages/74/0d/f2cd247ad32633a5c36e97141a2c21b11c6279f7957bc2ff360b1e08fddd/ruff-0.15.16-py3-none-win32.whl", hash = "sha256:580378f7bd4aa25f72e74aa54948a9622f142b1e509521dd10902e886681cc1e", size = 10735541, upload-time = "2026-06-04T16:32:30.145Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, - { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, +version = "0.15.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/06/ae069393fc66e8ff33036d4b368003833bf6e88ccf182e17e7a2f1c754fd/ruff-0.15.22.tar.gz", hash = "sha256:3f15175b1fb580126f58285a5dae6b2ea89000136d980c64499211f116b54809", size = 4785063, upload-time = "2026-07-16T15:14:13.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/18/ee54b7ae1e121be7a28ea6da4b67564ebb0530e183a54415ab7e3bcd2c4e/ruff-0.15.22-py3-none-linux_armv6l.whl", hash = "sha256:44423e73493737f5e7c5b41d475483898ff37afcdae38bc3da5085e29af1c2d8", size = 10781258, upload-time = "2026-07-16T15:13:19.452Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d2/2520cb14761ddbeaf57642a76942fc36adcbdbe53b4532241995f6fc485c/ruff-0.15.22-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b82c6482946e9eda7ff2e091d25b8bad3f718684e1916d41bd56873cee05b697", size = 10999477, upload-time = "2026-07-16T15:13:23.318Z" }, + { url = "https://files.pythonhosted.org/packages/c9/10/74e53572aa758dfaa678c2a2646b5c5515d884b7ca56be4d2ce03ca4b560/ruff-0.15.22-py3-none-macosx_11_0_arm64.whl", hash = "sha256:11c1c715af53a09f714e011106bffc419751ec8232fcb5da42173284ea3fec6f", size = 10466716, upload-time = "2026-07-16T15:13:26.162Z" }, + { url = "https://files.pythonhosted.org/packages/1e/cc/44eaaf0844e028182f2d0a8f2190d0f359159aed0a9e5ab861d892f1ae2a/ruff-0.15.22-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742a29cf29bddb7c8327895d6a10e0e6c5b38a96dd407af9b5d0857f809c0576", size = 10892644, upload-time = "2026-07-16T15:13:29.229Z" }, + { url = "https://files.pythonhosted.org/packages/9f/21/8edf559014d2b0f82beea19cfb713993ad802ccda16868769979c6090a84/ruff-0.15.22-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72af58b951b0ae395935ae79763dc349bc0eb706319d28f7a33ad2cfb3cfc178", size = 10576719, upload-time = "2026-07-16T15:13:32.35Z" }, + { url = "https://files.pythonhosted.org/packages/bf/1e/3a13abd392a3b50b62e5938a831f9ab6e588358cacad5c18545b716d2182/ruff-0.15.22-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d425005c1835eb24e2ee4161cb90e8db263415f4a71c8c72c33abaa6c0c224", size = 11376494, upload-time = "2026-07-16T15:13:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3e/422d3d95bcf04dd78e1aeac22184d4f9a8fb2c01865d39d44618484a0317/ruff-0.15.22-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b9b3f8779a4f08c969defc3c8c35abffaa757e601ed5ae66d6d1db6519969a", size = 12208370, upload-time = "2026-07-16T15:13:39.185Z" }, + { url = "https://files.pythonhosted.org/packages/1e/91/5d065a0e0a02bf4813f5119ad278462eed081d2b832eb7c021ade0ec9e65/ruff-0.15.22-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e0dd1b2e4d3d585f897a0d137cbf4eaf6223bef4e8ce34d6bb12556c5f9249e", size = 11581098, upload-time = "2026-07-16T15:13:42.132Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f9/a0d4871d12fae702eb1f41b686caf05f1f8b124dc6db6f784f53d74918fa/ruff-0.15.22-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365523eb91d9224e1bcb03b022fbf0facb8f9e23792a2c53d9d4b3924bdbdebb", size = 11399422, upload-time = "2026-07-16T15:13:45.2Z" }, + { url = "https://files.pythonhosted.org/packages/18/80/c843a5176cddbceb0b7e8dd41cf9993490796c1c469348d384f5a5c13c56/ruff-0.15.22-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:fabfd168afdf29fee5be98b831efa9683c94d7c5a3b58b9ce5a2e38444589a74", size = 11381683, upload-time = "2026-07-16T15:13:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/8485de0ae92239438a36cfc51350db9b9e85c9ebdfaea91b18e422706662/ruff-0.15.22-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:225dbf095a87f1d9f90f5fd7924d2613ee452a75a4308c63a8f50f761787aa7c", size = 10850295, upload-time = "2026-07-16T15:13:51.655Z" }, + { url = "https://files.pythonhosted.org/packages/fa/91/24977ec2ec72eaf15e4394ace2959fdff2dd1e14f03e005e838023407169/ruff-0.15.22-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1877d63b9d24ed278744f1523fd11b85540566d54641f97c566d7d9dc5ca5296", size = 10579640, upload-time = "2026-07-16T15:13:54.79Z" }, + { url = "https://files.pythonhosted.org/packages/9c/47/9b51216951974df1f263ac19da550d34252e0ed7218c25f10c5ef9ed7517/ruff-0.15.22-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a1606c510bd7215680d32efab38965f7cdec3ef69f5170a3f4791404ffdd5262", size = 11105077, upload-time = "2026-07-16T15:13:57.915Z" }, + { url = "https://files.pythonhosted.org/packages/c2/47/20e9d4a3b8016778acea5fc32bb50d35d207500a17ddb529ffa6996feef8/ruff-0.15.22-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:630479b18625f5ffc373f77603a22a9f8ac0acd7ff0501178b5db28ec71e9c64", size = 11490980, upload-time = "2026-07-16T15:14:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/4d/76/3f72d8fc38c1cb77b38c56a70da9d0c17700cc1cc50f9649c9d3c8f5ba71/ruff-0.15.22-py3-none-win32.whl", hash = "sha256:e5ba0e4a13fd14abbed2a77b517a3911290c6c6c59ef67784328d1668fab76cf", size = 10789165, upload-time = "2026-07-16T15:14:04.16Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/4965251734c2b6fcdca1b1b187d20bcac3af0ee5b083b89c910bb961ce3a/ruff-0.15.22-py3-none-win_amd64.whl", hash = "sha256:9be63ba1eb936acd2d1342fb8337c356353706fce233b2a15a09a97037e6acde", size = 11938297, upload-time = "2026-07-16T15:14:07.316Z" }, + { url = "https://files.pythonhosted.org/packages/57/c9/e69b1ff4c8b69093ef08b8919ab767af0569666865b39c30a8795d88d3c6/ruff-0.15.22-py3-none-win_arm64.whl", hash = "sha256:e1168075b72158510839f250027659cdd78476f40507dd517892304c41318661", size = 11298172, upload-time = "2026-07-16T15:14:10.51Z" }, ] [[package]] name = "s3fs" -version = "2026.4.0" +version = "2026.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore" }, { name = "aiohttp" }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/d8/76f3dc1558bdf4494b117a9f7a9cc0a5d9d34edadc9e5d7ceabc5a6a7c37/s3fs-2026.4.0.tar.gz", hash = "sha256:5bdce0abb00b0435ee150807a45fea727451dbc22de4cbc116464f8504ab9d37", size = 85986, upload-time = "2026-04-29T20:52:51.748Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a4/9d1ea10ebc9e028a289a72fec84da170689549a8102c8aacfcad26bc5035/s3fs-2026.4.0-py3-none-any.whl", hash = "sha256:de0d2a1f33cdf03831fd2382d278c6e4e31fe57c3bf2f703c61f8aec6b703e2a", size = 32392, upload-time = "2026-04-29T20:52:50.295Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9b/ec/7c692cde9125b77e84b307354d4fb705f98b8ccad59a036d5957ca75bfc3/s3transfer-0.17.0.tar.gz", hash = "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", size = 155337, upload-time = "2026-04-29T22:07:36.33Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/00/6677343dc919d6c072bb04d80210afdd22c16838a8d16b3315c122dc728f/s3fs-2026.6.0.tar.gz", hash = "sha256:b28de7082d0a4f72392884bdc497e34a4a1582f675d214c7da0acf6e950a0083", size = 87358, upload-time = "2026-06-16T02:05:48.719Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0b/f68a968b49876eae0f2a515387093cebb2eb9451380a96741cc20efac0d0/s3fs-2026.6.0-py3-none-any.whl", hash = "sha256:60576e31bb31193c1f643f32b4c6439548720ea6918ac702e21cd757c80b5db8", size = 32573, upload-time = "2026-06-16T02:05:47.608Z" }, ] [[package]] @@ -4755,8 +2473,10 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "joblib" }, { name = "narwhals" }, - { name = "numpy" }, - { name = "scipy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } @@ -4779,8 +2499,13 @@ wheels = [ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -4806,6 +2531,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, ] +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, +] + [[package]] name = "scmrepo" version = "3.6.2" @@ -4832,8 +2583,10 @@ name = "scs" version = "3.2.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "scipy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/59/5cb7f9612a5a3ff6efd4ab2d899902a536cc5974a7edb589084c5577291c/scs-3.2.11.tar.gz", hash = "sha256:2a5455cf2093d07f84f2f848c199faed52e79cdb3a11fe250b5622b6bbac4913", size = 1691825, upload-time = "2026-01-09T17:53:54.074Z" } wheels = [ @@ -4849,20 +2602,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/fe/5d8f6048a90abc3aa053b5ac2acf3885dc46af94c3baf7d9ccf201a1ce19/scs-3.2.11-cp312-cp312-win_amd64.whl", hash = "sha256:2504266ff8e6a226f7ecb987567c93e6e996534cbf479a60a5a886549446205e", size = 7478461, upload-time = "2026-01-09T17:53:11.899Z" }, ] -[[package]] -name = "seaborn" -version = "0.13.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib" }, - { name = "numpy" }, - { name = "pandas" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, -] - [[package]] name = "semver" version = "3.0.4" @@ -4872,31 +2611,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, ] -[[package]] -name = "send2trash" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c5/f0/184b4b5f8d00f2a92cf96eec8967a3d550b52cf94362dad1100df9e48d57/send2trash-2.1.0.tar.gz", hash = "sha256:1c72b39f09457db3c05ce1d19158c2cbef4c32b8bedd02c155e49282b7ea7459", size = 17255, upload-time = "2026-01-14T06:27:36.056Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/78/504fdd027da3b84ff1aecd9f6957e65f35134534ccc6da8628eb71e76d3f/send2trash-2.1.0-py3-none-any.whl", hash = "sha256:0da2f112e6d6bb22de6aa6daa7e144831a4febf2a87261451c4ad849fe9a873c", size = 17610, upload-time = "2026-01-14T06:27:35.218Z" }, -] - [[package]] name = "setuptools" -version = "81.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0d/1c/73e719955c59b8e424d015ab450f51c0af856ae46ea2da83eba51cc88de1/setuptools-81.0.0.tar.gz", hash = "sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a", size = 1198299, upload-time = "2026-02-06T21:10:39.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] @@ -4910,11 +2631,11 @@ wheels = [ [[package]] name = "shtab" -version = "1.8.0" +version = "1.8.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/7a/7f131b6082d8b592c32e4312d0a6da3d0b28b8f0d305ddd93e49c9d89929/shtab-1.8.0.tar.gz", hash = "sha256:75f16d42178882b7f7126a0c2cb3c848daed2f4f5a276dd1ded75921cc4d073a", size = 46062, upload-time = "2025-11-18T10:57:47.601Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/b3/b7c99318ba2b0d7d7e71b78318c162690dd5639cd06e2d3bca9292737e4e/shtab-1.8.1.tar.gz", hash = "sha256:3edaf857ba164a0d7bb64e958eab8d6756d5aee3949c7f36a7e779bf1ce3b897", size = 46449, upload-time = "2026-07-03T11:43:26.663Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/e1/202a31727b0d096a04380f78e809074d7a1d0a22d9d5a39fea1d2353fd02/shtab-1.8.0-py3-none-any.whl", hash = "sha256:f0922a82174b4007e06ac0bac4f79abd826c5cca88e201bfd927f889803c571d", size = 14457, upload-time = "2025-11-18T10:57:45.906Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/35d4ec80d8ee39e792cd6e0d4f3dd84ea6783f13b7c47b4c319d9062aaf0/shtab-1.8.1-py3-none-any.whl", hash = "sha256:4fc418e0173e66385e4605044a26a4910e73dde2dc23e809acc2f0275e842426", size = 14905, upload-time = "2026-07-03T11:43:25.374Z" }, ] [[package]] @@ -4926,22 +2647,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "skops" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "packaging" }, - { name = "prettytable" }, - { name = "scikit-learn" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c8/9f/46448c4e41a4c5ee4bdb74b3758af48e5ff0faeffe40f4e301bfc7594894/skops-0.14.0.tar.gz", hash = "sha256:6c8c0e047f691a3a582c3258943eecafcbfd79c8c7eef66260f3703e363254f0", size = 608084, upload-time = "2026-04-20T18:23:55.336Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/0e/3ae19fa941522cd98e119762e7181d371c8dba0b2d72bfaf9522692e329c/skops-0.14.0-py3-none-any.whl", hash = "sha256:60a5db78a9db46ccee2139a0ba13ab5afb1c96f4749b382e75a371291bbe3e36", size = 132198, upload-time = "2026-04-20T18:23:54.018Z" }, -] - [[package]] name = "smmap" version = "5.0.3" @@ -4951,19 +2656,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] -[[package]] -name = "snowplow-tracker" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ff/77/1ab6e5bafb9c80d8128f065a355377a04ac5b3c38eb719d920a9909d346e/snowplow_tracker-1.1.0.tar.gz", hash = "sha256:95d8fdc8bd542fd12a0b9a076852239cbaf0599eda8721deaf5f93f7138fe755", size = 34135, upload-time = "2025-02-21T10:58:48.112Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/10/1c76269cbf2d6e127f4415044d9ddb0295858230678bbf4bfba905593c82/snowplow_tracker-1.1.0-py3-none-any.whl", hash = "sha256:24ea32ddac9cca547421bf9ab162f5f33c00711c6ef118ad5f78093cee962224", size = 44128, upload-time = "2025-02-21T10:58:45.818Z" }, -] - [[package]] name = "sortedcontainers" version = "2.4.0" @@ -4973,21 +2665,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] -[[package]] -name = "soupsieve" -version = "2.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, -] - [[package]] name = "sparsediffpy" version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/e7/6a3227a25a79a440e5ec5eff1e90f5911515a7c219ee95fc6dfe5d74ec30/sparsediffpy-0.3.0.tar.gz", hash = "sha256:fdd9115db63ee228d09e1917365b263a16811645c6d32ee7dce50ada09b3d5a5", size = 180927, upload-time = "2026-05-14T06:57:48.196Z" } wheels = [ @@ -5003,42 +2687,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a7/487fcf2157472235411e30795f1b8270e311db73e1bb58b3fe73d70c19d3/sparsediffpy-0.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:695be68fedc2c1b6fc258d05e37e20b4081c38e020ec5c9d862e42266d1ece84", size = 130146, upload-time = "2026-05-14T06:57:26.56Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.49" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/b5/e3617cc67420f8f403efebd7b043128f94775e57e5b84e7255203390ceae/sqlalchemy-2.0.49-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5070135e1b7409c4161133aa525419b0062088ed77c92b1da95366ec5cbebbe", size = 2159126, upload-time = "2026-04-03T16:50:13.242Z" }, - { url = "https://files.pythonhosted.org/packages/20/9b/91ca80403b17cd389622a642699e5f6564096b698e7cdcbcbb6409898bc4/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ac7a3e245fd0310fd31495eb61af772e637bdf7d88ee81e7f10a3f271bff014", size = 3315509, upload-time = "2026-04-03T16:54:49.332Z" }, - { url = "https://files.pythonhosted.org/packages/b1/61/0722511d98c54de95acb327824cb759e8653789af2b1944ab1cc69d32565/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d4e5a0ceba319942fa6b585cf82539288a61e314ef006c1209f734551ab9536", size = 3315014, upload-time = "2026-04-03T16:56:56.376Z" }, - { url = "https://files.pythonhosted.org/packages/46/55/d514a653ffeb4cebf4b54c47bec32ee28ad89d39fafba16eeed1d81dccd5/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ddcb27fb39171de36e207600116ac9dfd4ae46f86c82a9bf3934043e80ebb88", size = 3267388, upload-time = "2026-04-03T16:54:51.272Z" }, - { url = "https://files.pythonhosted.org/packages/2f/16/0dcc56cb6d3335c1671a2258f5d2cb8267c9a2260e27fde53cbfb1b3540a/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:32fe6a41ad97302db2931f05bb91abbcc65b5ce4c675cd44b972428dd2947700", size = 3289602, upload-time = "2026-04-03T16:56:57.63Z" }, - { url = "https://files.pythonhosted.org/packages/51/6c/f8ab6fb04470a133cd80608db40aa292e6bae5f162c3a3d4ab19544a67af/sqlalchemy-2.0.49-cp311-cp311-win32.whl", hash = "sha256:46d51518d53edfbe0563662c96954dc8fcace9832332b914375f45a99b77cc9a", size = 2119044, upload-time = "2026-04-03T17:00:53.455Z" }, - { url = "https://files.pythonhosted.org/packages/c4/59/55a6d627d04b6ebb290693681d7683c7da001eddf90b60cfcc41ee907978/sqlalchemy-2.0.49-cp311-cp311-win_amd64.whl", hash = "sha256:951d4a210744813be63019f3df343bf233b7432aadf0db54c75802247330d3af", size = 2143642, upload-time = "2026-04-03T17:00:54.769Z" }, - { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, - { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, - { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, - { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, - { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, - { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, -] - -[[package]] -name = "sqlparse" -version = "0.5.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815, upload-time = "2025-12-19T07:17:45.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, -] - [[package]] name = "sqltrie" version = "0.11.2" @@ -5053,45 +2701,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/a7/96dd20ed6c4008ca57aa14bd89588eff1dfc163f45067cf715df290dc211/sqltrie-0.11.2-py3-none-any.whl", hash = "sha256:4afb1390bbe8a6900a53709b76213a436fbaf352de0b99ba9b0d395d4a0ca6b6", size = 17140, upload-time = "2025-02-19T15:11:34.044Z" }, ] -[[package]] -name = "stack-data" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asttokens" }, - { name = "executing" }, - { name = "pure-eval" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, -] - -[[package]] -name = "starlette" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - [[package]] name = "tabulate" version = "0.10.0" @@ -5101,38 +2710,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, ] -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - -[[package]] -name = "terminado" -version = "0.18.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess", marker = "os_name != 'nt'" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, - { name = "tornado" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, -] - -[[package]] -name = "text-unidecode" -version = "1.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, -] - [[package]] name = "threadpoolctl" version = "3.6.0" @@ -5142,18 +2719,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] -[[package]] -name = "tinycss2" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/fd/7a5ee21fd08ff70d3d33a5781c255cbe779659bd03278feb98b19ee550f4/tinycss2-1.4.0.tar.gz", hash = "sha256:10c0972f6fc0fbee87c3edb76549357415e94548c1ae10ebccdea16fb404a9b7", size = 87085, upload-time = "2024-10-24T14:58:29.895Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610, upload-time = "2024-10-24T14:58:28.029Z" }, -] - [[package]] name = "tomli" version = "2.4.1" @@ -5183,153 +2748,32 @@ wheels = [ [[package]] name = "tomlkit" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" }, -] - -[[package]] -name = "torch" -version = "2.12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e3/750b3e3548635ceac03ba255daa26dbc7ed66ca3484dc4b4d955ab7f4501/torch-2.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:107df6888624bdea41508f9aeb6149d9333c737a5530ceecb56c904e811369ae", size = 426379894, upload-time = "2026-06-17T21:06:55.077Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ca/ed24783da629ff3e640ba3f70a7639e9045d3d88b93ee6bc47b8a28a1f2c/torch-2.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:6e29e7e74d05bda7d955c75e99459f878ebd970ef851b4057edbd3b34a5eb4a3", size = 532169264, upload-time = "2026-06-17T21:08:17.65Z" }, - { url = "https://files.pythonhosted.org/packages/46/61/c63f0158446f3a98ea672b004d761b848911eba567ea4a624c7db5aadc04/torch-2.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:a513506cfda3c1c78dabeb6574c1597538c0254b3d39af174dde35d8177f4ce3", size = 122953086, upload-time = "2026-06-17T21:08:27.69Z" }, - { url = "https://files.pythonhosted.org/packages/f0/54/efb7ebca77970012b0cc21687a55d70eb2ba514b2c2b8e18d9fb1222f3be/torch-2.12.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d2dd0f2c5f7ccbddaf34cade0deaf476808368f902b9cdb7f36a2ab42301bc0e", size = 87991951, upload-time = "2026-06-17T21:07:49.309Z" }, - { url = "https://files.pythonhosted.org/packages/1e/00/4210d76ca7424981f04033ebe7e48816ab83287a62538747a58825db770c/torch-2.12.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2de4e19b88a481482c6c75291f2d6a52eda3ce51f311b29aa9b68499c830c07c", size = 426382721, upload-time = "2026-06-17T21:06:41.842Z" }, - { url = "https://files.pythonhosted.org/packages/76/1f/bc9f5a5aa569307076365f25afcebacb22e9c754b1bcfbaaa146627c7fda/torch-2.12.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:649e4ced014ba646f76f8cb9c9726735a6323eb321b7919f942790a923f90921", size = 532261322, upload-time = "2026-06-17T21:06:06.673Z" }, - { url = "https://files.pythonhosted.org/packages/9e/49/c549461daa008159d006a76a991fbc2f26fa8bac27a4030c858463dcb20f/torch-2.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:e86550597877fb272ddc52db2f85b82cb601ea7bd932576a0340152cae2200b3", size = 122988095, upload-time = "2026-06-17T21:07:44.9Z" }, -] - -[[package]] -name = "tornado" -version = "6.5.7" +version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/dc/c7043cab6fed8ae159fc1923ce829ada35c4dbd797d408a43858ffaf9639/tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163", size = 448543, upload-time = "2026-06-08T17:34:38.052Z" }, - { url = "https://files.pythonhosted.org/packages/92/4f/090b1431e5a43df696feceffc268c5383cc079ecb5f08ce58f917109aafe/tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100", size = 446707, upload-time = "2026-06-08T17:34:39.594Z" }, - { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, - { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, - { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, - { url = "https://files.pythonhosted.org/packages/5c/42/5f0e56c01e8d9d36f4e23f367b85ae6cae0c1ecddd5e6977d8388ad27488/tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4", size = 451047, upload-time = "2026-06-08T17:34:46.784Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/b393076ffb21b469eec5b328a0534cf03a3b90bfc6b1f09507cdd075d938/tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4", size = 451485, upload-time = "2026-06-08T17:34:48.248Z" }, - { url = "https://files.pythonhosted.org/packages/71/2e/7b1c769803121b809112cf9a00681c472eae1d80e32d7ec0e0bd61d0d0e1/tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796", size = 450506, upload-time = "2026-06-08T17:34:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, ] [[package]] name = "tqdm" -version = "4.67.3" +version = "4.68.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, -] - -[[package]] -name = "traitlets" -version = "5.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/22/40f55b26baeab80c2d7b3f1db0682f8954e4617fee7d90ce634022ef05c6/traitlets-5.15.0.tar.gz", hash = "sha256:4fead733f81cf1c4c938e06f8ca4633896833c9d89eff878159457f4d4392971", size = 163197, upload-time = "2026-05-06T08:05:58.016Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/98/a9937a969d018a23badfea0b381f66783649d48e0ea6c41923265c3cbeb3/traitlets-5.15.0-py3-none-any.whl", hash = "sha256:fb36a18867a6803deab09f3c5e0fa81bb7b26a5c9e82501c9933f759166eff40", size = 85877, upload-time = "2026-05-06T08:05:55.853Z" }, -] - -[[package]] -name = "treelib" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/31/145bdbee73d7ee4ac4e879c37faa196a32208b288ca4f308c1ad8db3f010/treelib-1.8.0.tar.gz", hash = "sha256:e1be2c6b66ffbfae85079fc4c76fb4909946d01d915ee29ff6795de53aed5d55", size = 28607, upload-time = "2025-06-29T15:06:49.293Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/24/32361f5d0e2eff7ff1881ac6833b6b090cfe34515b1ee9082636cbe69442/treelib-1.8.0-py3-none-any.whl", hash = "sha256:5235d1ebf988c5026f26ce6e5e0cd470007f16d4978185f5c9b3eee8a25aef81", size = 30728, upload-time = "2025-06-29T15:06:48.248Z" }, -] - -[[package]] -name = "triton" -version = "3.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/f9/19d842d06a08559534fa1eaab6ca551b1bcf40f06620bddec1babaa2772d/triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6", size = 184664887, upload-time = "2026-06-17T20:03:42.913Z" }, - { url = "https://files.pythonhosted.org/packages/cd/5e/fce69606f7f240297f163e25539906732b199530d486ce67ae319877e821/triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5", size = 197701306, upload-time = "2026-06-17T19:53:13.673Z" }, - { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, - { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, -] - -[[package]] -name = "typeguard" -version = "4.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/e8/66e25efcc18542d58706ce4e50415710593721aae26e794ab1dec34fb66f/typeguard-4.5.1.tar.gz", hash = "sha256:f6f8ecbbc819c9bc749983cc67c02391e16a9b43b8b27f15dc70ed7c4a007274", size = 80121, upload-time = "2026-02-19T16:09:03.392Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/88/b55b3117287a8540b76dbdd87733808d4d01c8067a3b339408c250bb3600/typeguard-4.5.1-py3-none-any.whl", hash = "sha256:44d2bf329d49a244110a090b55f5f91aa82d9a9834ebfd30bcc73651e4a8cc40", size = 36745, upload-time = "2026-02-19T16:09:01.6Z" }, -] - -[[package]] -name = "typer" -version = "0.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/5f/57ff8b434839e70dab45601284ea413e947a63799891b7553e5960a793a8/tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520", size = 792418, upload-time = "2026-07-07T09:58:18.369Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, + { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, ] [[package]] name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] - -[[package]] -name = "typing-inspect" -version = "0.9.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] @@ -5346,32 +2790,23 @@ wheels = [ [[package]] name = "tzdata" -version = "2026.2" +version = "2026.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, ] [[package]] name = "tzlocal" -version = "5.3.1" +version = "5.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, -] - -[[package]] -name = "uri-template" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", size = 21678, upload-time = "2023-06-21T01:49:05.374Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363", size = 11140, upload-time = "2023-06-21T01:49:03.467Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, ] [[package]] @@ -5383,30 +2818,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] -[[package]] -name = "uvicorn" -version = "0.46.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/93/041fca8274050e40e6791f267d82e0e2e27dd165627bd640d3e0e378d877/uvicorn-0.46.0.tar.gz", hash = "sha256:fb9da0926999cc6cb22dc7cd71a94a632f078e6ae47ff683c5c420750fb7413d", size = 88758, upload-time = "2026-04-23T07:16:00.151Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/a3/5b1562db76a5a488274b2332a97199b32d0442aca0ed193697fd47786316/uvicorn-0.46.0-py3-none-any.whl", hash = "sha256:bbebbcbed972d162afca128605223022bedd345b7bc7855ce66deb31487a9048", size = 70926, upload-time = "2026-04-23T07:15:58.355Z" }, -] - -[[package]] -name = "venn-abers" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pandas" }, - { name = "scikit-learn" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f1/3c/dedb9bc9e042e5c10810d7bb7ccac98f12d119a98e3a355ae476fc431886/venn_abers-1.5.3.tar.gz", hash = "sha256:b5fdd4b4a5a1e5edb76d4496235221da127084261324e752a22b424b4f7daf16", size = 19754, upload-time = "2026-05-08T16:41:29.646Z" } - [[package]] name = "vine" version = "5.1.0" @@ -5418,7 +2829,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.3.1" +version = "21.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -5426,9 +2837,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ec/0d/915c02c94d207b85580eb09bffab54438a709e7288524094fe781da526c2/virtualenv-21.3.1.tar.gz", hash = "sha256:c2305bc1fddeec40699b8370d13f8d431b0701f00ce895061ce493aeded4426b", size = 7613791, upload-time = "2026-05-05T01:34:31.402Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/d9/b477fddb68840b570af8b22afe9b035cbc277b5fb7b33dea390617a8b10f/virtualenv-21.6.1.tar.gz", hash = "sha256:15f978b7cd329f24855ff4a0c4b4899cc7678589f49adbdcbbb4d3232e641128", size = 5526620, upload-time = "2026-07-10T19:33:53.312Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/4f/f71e641e504111a5a74e3a20bc52d01bd86788b22699dd3fee1c63253cf6/virtualenv-21.3.1-py3-none-any.whl", hash = "sha256:d1a71cf58f2f9228fff23a1f6ec15d39785c6b32e03658d104974247145edd35", size = 7594539, upload-time = "2026-05-05T01:34:28.98Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7c/4e7225d46d634a0d8d534dd8a6ce0c319d09b4d0cf0337eb314ca4789d8c/virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b", size = 5506392, upload-time = "2026-07-10T19:33:51.629Z" }, ] [[package]] @@ -5440,70 +2851,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/00/0e0da784245c93cf346150ab67634177bf277f93b7a162bb56c928c39c04/voluptuous-0.16.0-py3-none-any.whl", hash = "sha256:ee342095263e1b5afbd4d418cb5adc92810eebfd07696bb033a261210df33db4", size = 31931, upload-time = "2025-12-18T23:18:44.694Z" }, ] -[[package]] -name = "waitress" -version = "3.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bf/cb/04ddb054f45faa306a230769e868c28b8065ea196891f09004ebace5b184/waitress-3.0.2.tar.gz", hash = "sha256:682aaaf2af0c44ada4abfb70ded36393f0e307f4ab9456a215ce0020baefc31f", size = 179901, upload-time = "2024-11-16T20:02:35.195Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/57/a27182528c90ef38d82b636a11f606b0cbb0e17588ed205435f8affe3368/waitress-3.0.2-py3-none-any.whl", hash = "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e", size = 56232, upload-time = "2024-11-16T20:02:33.858Z" }, -] - [[package]] name = "wcwidth" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, -] - -[[package]] -name = "webcolors" -version = "25.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz", hash = "sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf", size = 53491, upload-time = "2025-10-31T07:51:03.977Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" }, -] - -[[package]] -name = "webencodings" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, -] - -[[package]] -name = "websocket-client" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, -] - -[[package]] -name = "werkzeug" -version = "3.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, -] - -[[package]] -name = "widgetsnbextension" -version = "4.0.15" +version = "0.8.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/c67440c7fb409a71b7404b7aefcd7569a9c0d6bd071299bf4198ae7a5d95/widgetsnbextension-4.0.15.tar.gz", hash = "sha256:de8610639996f1567952d763a5a41af8af37f2575a41f9852a38f947eb82a3b9", size = 1097402, upload-time = "2025-11-01T21:15:55.178Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, ] [[package]] @@ -5517,83 +2871,81 @@ wheels = [ [[package]] name = "wrapt" -version = "2.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/64/925f213fdcbb9baeb1530449ac71a4d57fc361c053d06bf78d0c5c7cd80c/wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e", size = 81678, upload-time = "2026-03-06T02:53:25.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/81/60c4471fce95afa5922ca09b88a25f03c93343f759aae0f31fb4412a85c7/wrapt-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:96159a0ee2b0277d44201c3b5be479a9979cf154e8c82fa5df49586a8e7679bb", size = 60666, upload-time = "2026-03-06T02:52:58.934Z" }, - { url = "https://files.pythonhosted.org/packages/6b/be/80e80e39e7cb90b006a0eaf11c73ac3a62bbfb3068469aec15cc0bc795de/wrapt-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98ba61833a77b747901e9012072f038795de7fc77849f1faa965464f3f87ff2d", size = 61601, upload-time = "2026-03-06T02:53:00.487Z" }, - { url = "https://files.pythonhosted.org/packages/b0/be/d7c88cd9293c859fc74b232abdc65a229bb953997995d6912fc85af18323/wrapt-2.1.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:767c0dbbe76cae2a60dd2b235ac0c87c9cccf4898aef8062e57bead46b5f6894", size = 114057, upload-time = "2026-03-06T02:52:44.08Z" }, - { url = "https://files.pythonhosted.org/packages/ea/25/36c04602831a4d685d45a93b3abea61eca7fe35dab6c842d6f5d570ef94a/wrapt-2.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c691a6bc752c0cc4711cc0c00896fcd0f116abc253609ef64ef930032821842", size = 116099, upload-time = "2026-03-06T02:54:56.74Z" }, - { url = "https://files.pythonhosted.org/packages/5c/4e/98a6eb417ef551dc277bec1253d5246b25003cf36fdf3913b65cb7657a56/wrapt-2.1.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f3b7d73012ea75aee5844de58c88f44cf62d0d62711e39da5a82824a7c4626a8", size = 112457, upload-time = "2026-03-06T02:53:52.842Z" }, - { url = "https://files.pythonhosted.org/packages/cb/a6/a6f7186a5297cad8ec53fd7578533b28f795fdf5372368c74bd7e6e9841c/wrapt-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:577dff354e7acd9d411eaf4bfe76b724c89c89c8fc9b7e127ee28c5f7bcb25b6", size = 115351, upload-time = "2026-03-06T02:53:32.684Z" }, - { url = "https://files.pythonhosted.org/packages/97/6f/06e66189e721dbebd5cf20e138acc4d1150288ce118462f2fcbff92d38db/wrapt-2.1.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d7b6fd105f8b24e5bd23ccf41cb1d1099796524bcc6f7fbb8fe576c44befbc9", size = 111748, upload-time = "2026-03-06T02:53:08.455Z" }, - { url = "https://files.pythonhosted.org/packages/ef/43/4808b86f499a51370fbdbdfa6cb91e9b9169e762716456471b619fca7a70/wrapt-2.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:866abdbf4612e0b34764922ef8b1c5668867610a718d3053d59e24a5e5fcfc15", size = 113783, upload-time = "2026-03-06T02:53:02.02Z" }, - { url = "https://files.pythonhosted.org/packages/91/2c/a3f28b8fa7ac2cefa01cfcaca3471f9b0460608d012b693998cd61ef43df/wrapt-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5a0a0a3a882393095573344075189eb2d566e0fd205a2b6414e9997b1b800a8b", size = 57977, upload-time = "2026-03-06T02:53:27.844Z" }, - { url = "https://files.pythonhosted.org/packages/3f/c3/2b1c7bd07a27b1db885a2fab469b707bdd35bddf30a113b4917a7e2139d2/wrapt-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:64a07a71d2730ba56f11d1a4b91f7817dc79bc134c11516b75d1921a7c6fcda1", size = 60336, upload-time = "2026-03-06T02:54:28.104Z" }, - { url = "https://files.pythonhosted.org/packages/ec/5c/76ece7b401b088daa6503d6264dd80f9a727df3e6042802de9a223084ea2/wrapt-2.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:b89f095fe98bc12107f82a9f7d570dc83a0870291aeb6b1d7a7d35575f55d98a", size = 58756, upload-time = "2026-03-06T02:53:16.319Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/1db817582c49c7fcbb7df6809d0f515af29d7c2fbf57eb44c36e98fb1492/wrapt-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ff2aad9c4cda28a8f0653fc2d487596458c2a3f475e56ba02909e950a9efa6a9", size = 61255, upload-time = "2026-03-06T02:52:45.663Z" }, - { url = "https://files.pythonhosted.org/packages/a2/16/9b02a6b99c09227c93cd4b73acc3678114154ec38da53043c0ddc1fba0dc/wrapt-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6433ea84e1cfacf32021d2a4ee909554ade7fd392caa6f7c13f1f4bf7b8e8748", size = 61848, upload-time = "2026-03-06T02:53:48.728Z" }, - { url = "https://files.pythonhosted.org/packages/af/aa/ead46a88f9ec3a432a4832dfedb84092fc35af2d0ba40cd04aea3889f247/wrapt-2.1.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c20b757c268d30d6215916a5fa8461048d023865d888e437fab451139cad6c8e", size = 121433, upload-time = "2026-03-06T02:54:40.328Z" }, - { url = "https://files.pythonhosted.org/packages/3a/9f/742c7c7cdf58b59085a1ee4b6c37b013f66ac33673a7ef4aaed5e992bc33/wrapt-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79847b83eb38e70d93dc392c7c5b587efe65b3e7afcc167aa8abd5d60e8761c8", size = 123013, upload-time = "2026-03-06T02:53:26.58Z" }, - { url = "https://files.pythonhosted.org/packages/e8/44/2c3dd45d53236b7ed7c646fcf212251dc19e48e599debd3926b52310fafb/wrapt-2.1.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f8fba1bae256186a83d1875b2b1f4e2d1242e8fac0f58ec0d7e41b26967b965c", size = 117326, upload-time = "2026-03-06T02:53:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/74/e2/b17d66abc26bd96f89dec0ecd0ef03da4a1286e6ff793839ec431b9fae57/wrapt-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e3d3b35eedcf5f7d022291ecd7533321c4775f7b9cd0050a31a68499ba45757c", size = 121444, upload-time = "2026-03-06T02:54:09.5Z" }, - { url = "https://files.pythonhosted.org/packages/3c/62/e2977843fdf9f03daf1586a0ff49060b1b2fc7ff85a7ea82b6217c1ae36e/wrapt-2.1.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6f2c5390460de57fa9582bc8a1b7a6c86e1a41dfad74c5225fc07044c15cc8d1", size = 116237, upload-time = "2026-03-06T02:54:03.884Z" }, - { url = "https://files.pythonhosted.org/packages/88/dd/27fc67914e68d740bce512f11734aec08696e6b17641fef8867c00c949fc/wrapt-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7dfa9f2cf65d027b951d05c662cc99ee3bd01f6e4691ed39848a7a5fffc902b2", size = 120563, upload-time = "2026-03-06T02:53:20.412Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9f/b750b3692ed2ef4705cb305bd68858e73010492b80e43d2a4faa5573cbe7/wrapt-2.1.2-cp312-cp312-win32.whl", hash = "sha256:eba8155747eb2cae4a0b913d9ebd12a1db4d860fc4c829d7578c7b989bd3f2f0", size = 58198, upload-time = "2026-03-06T02:53:37.732Z" }, - { url = "https://files.pythonhosted.org/packages/8e/b2/feecfe29f28483d888d76a48f03c4c4d8afea944dbee2b0cd3380f9df032/wrapt-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1c51c738d7d9faa0b3601708e7e2eda9bf779e1b601dce6c77411f2a1b324a63", size = 60441, upload-time = "2026-03-06T02:52:47.138Z" }, - { url = "https://files.pythonhosted.org/packages/44/e1/e328f605d6e208547ea9fd120804fcdec68536ac748987a68c47c606eea8/wrapt-2.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:c8e46ae8e4032792eb2f677dbd0d557170a8e5524d22acc55199f43efedd39bf", size = 58836, upload-time = "2026-03-06T02:53:22.053Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993, upload-time = "2026-03-06T02:53:12.905Z" }, +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] [[package]] name = "yarl" -version = "1.23.0" +version = "1.24.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/aa/60da938b8f0997ba3a911263c40d82b6f645a67902a490b46f3355e10fae/yarl-1.23.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99", size = 123641, upload-time = "2026-03-01T22:04:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/24/84/e237607faf4e099dbb8a4f511cfd5efcb5f75918baad200ff7380635631b/yarl-1.23.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c", size = 86248, upload-time = "2026-03-01T22:04:44.757Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0d/71ceabc14c146ba8ee3804ca7b3d42b1664c8440439de5214d366fec7d3a/yarl-1.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432", size = 85988, upload-time = "2026-03-01T22:04:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/8c/6c/4a90d59c572e46b270ca132aca66954f1175abd691f74c1ef4c6711828e2/yarl-1.23.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a", size = 100566, upload-time = "2026-03-01T22:04:47.639Z" }, - { url = "https://files.pythonhosted.org/packages/49/fb/c438fb5108047e629f6282a371e6e91cf3f97ee087c4fb748a1f32ceef55/yarl-1.23.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05", size = 92079, upload-time = "2026-03-01T22:04:48.925Z" }, - { url = "https://files.pythonhosted.org/packages/d9/13/d269aa1aed3e4f50a5a103f96327210cc5fa5dd2d50882778f13c7a14606/yarl-1.23.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83", size = 108741, upload-time = "2026-03-01T22:04:50.838Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/115b16f22c37ea4437d323e472945bea97301c8ec6089868fa560abab590/yarl-1.23.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c", size = 108099, upload-time = "2026-03-01T22:04:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/9a/64/c53487d9f4968045b8afa51aed7ca44f58b2589e772f32745f3744476c82/yarl-1.23.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598", size = 102678, upload-time = "2026-03-01T22:04:55.176Z" }, - { url = "https://files.pythonhosted.org/packages/85/59/cd98e556fbb2bf8fab29c1a722f67ad45c5f3447cac798ab85620d1e70af/yarl-1.23.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b", size = 100803, upload-time = "2026-03-01T22:04:56.588Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c0/b39770b56d4a9f0bb5f77e2f1763cd2d75cc2f6c0131e3b4c360348fcd65/yarl-1.23.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c", size = 100163, upload-time = "2026-03-01T22:04:58.492Z" }, - { url = "https://files.pythonhosted.org/packages/e7/64/6980f99ab00e1f0ff67cb84766c93d595b067eed07439cfccfc8fb28c1a6/yarl-1.23.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788", size = 93859, upload-time = "2026-03-01T22:05:00.268Z" }, - { url = "https://files.pythonhosted.org/packages/38/69/912e6c5e146793e5d4b5fe39ff5b00f4d22463dfd5a162bec565ac757673/yarl-1.23.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222", size = 108202, upload-time = "2026-03-01T22:05:02.273Z" }, - { url = "https://files.pythonhosted.org/packages/59/97/35ca6767524687ad64e5f5c31ad54bc76d585585a9fcb40f649e7e82ffed/yarl-1.23.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb", size = 99866, upload-time = "2026-03-01T22:05:03.597Z" }, - { url = "https://files.pythonhosted.org/packages/d3/1c/1a3387ee6d73589f6f2a220ae06f2984f6c20b40c734989b0a44f5987308/yarl-1.23.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc", size = 107852, upload-time = "2026-03-01T22:05:04.986Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/35c0750fcd5a3f781058bfd954515dd4b1eab45e218cbb85cf11132215f1/yarl-1.23.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2", size = 102919, upload-time = "2026-03-01T22:05:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1c/9a1979aec4a81896d597bcb2177827f2dbee3f5b7cc48b2d0dadb644b41d/yarl-1.23.0-cp311-cp311-win32.whl", hash = "sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5", size = 82602, upload-time = "2026-03-01T22:05:08.444Z" }, - { url = "https://files.pythonhosted.org/packages/93/22/b85eca6fa2ad9491af48c973e4c8cf6b103a73dbb271fe3346949449fca0/yarl-1.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46", size = 87461, upload-time = "2026-03-01T22:05:10.145Z" }, - { url = "https://files.pythonhosted.org/packages/93/95/07e3553fe6f113e6864a20bdc53a78113cda3b9ced8784ee52a52c9f80d8/yarl-1.23.0-cp311-cp311-win_arm64.whl", hash = "sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928", size = 82336, upload-time = "2026-03-01T22:05:11.554Z" }, - { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737, upload-time = "2026-03-01T22:05:12.897Z" }, - { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029, upload-time = "2026-03-01T22:05:14.376Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310, upload-time = "2026-03-01T22:05:15.71Z" }, - { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587, upload-time = "2026-03-01T22:05:17.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528, upload-time = "2026-03-01T22:05:18.804Z" }, - { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339, upload-time = "2026-03-01T22:05:20.235Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061, upload-time = "2026-03-01T22:05:22.268Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132, upload-time = "2026-03-01T22:05:23.638Z" }, - { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289, upload-time = "2026-03-01T22:05:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950, upload-time = "2026-03-01T22:05:27.318Z" }, - { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960, upload-time = "2026-03-01T22:05:28.738Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703, upload-time = "2026-03-01T22:05:30.438Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325, upload-time = "2026-03-01T22:05:31.835Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067, upload-time = "2026-03-01T22:05:33.358Z" }, - { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285, upload-time = "2026-03-01T22:05:35.4Z" }, - { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359, upload-time = "2026-03-01T22:05:36.811Z" }, - { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674, upload-time = "2026-03-01T22:05:38.171Z" }, - { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879, upload-time = "2026-03-01T22:05:40.006Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/c5/1ce244152ff2839645e7cae92f90e7bafcb2c52bea7ff586ac714f14f5df/yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1", size = 128971, upload-time = "2026-05-19T21:28:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/87/5a/00f36967203ed89cb3acd2c8ed526cc3fed9418eb70ce128160a911c8499/yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c", size = 91507, upload-time = "2026-05-19T21:28:22.556Z" }, + { url = "https://files.pythonhosted.org/packages/31/d0/1fb0c1cd27288f39f6974da4318c32768d72c9890984541fdf1e2e32a51d/yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d", size = 91343, upload-time = "2026-05-19T21:28:24.092Z" }, + { url = "https://files.pythonhosted.org/packages/03/ce/d4a646508bed2f8dec6435b40166fe9308dd191262033d3f307b2bbcaecd/yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae", size = 105704, upload-time = "2026-05-19T21:28:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/b3278e82d8bc41485bcf6d856cd0433262593de615b1d3dc43bd3f5bead4/yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a", size = 97281, upload-time = "2026-05-19T21:28:27.352Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/4cee6e7c92e487bebe7afc797da0aa54a248ab4e776a68fe369ec29665a5/yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e", size = 114020, upload-time = "2026-05-19T21:28:29.458Z" }, + { url = "https://files.pythonhosted.org/packages/5c/82/111076571545a7d4f9cca3fbd5c6f40615af58642be09f12328f48022468/yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50", size = 111450, upload-time = "2026-05-19T21:28:31.262Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ec/08f671f69a444d704aeecebf92af659b67b97a869942411d0a578b08c334/yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003", size = 106384, upload-time = "2026-05-19T21:28:32.856Z" }, + { url = "https://files.pythonhosted.org/packages/e5/86/ce41e7a7a199340b2330d52b60f25c4074b6636dd0e60b1a80d31a9db042/yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f", size = 106153, upload-time = "2026-05-19T21:28:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5d/31be8a729531ab3e55ac3e7e5c800be8c89ea98947f418b2f6ea259fb6ee/yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f", size = 105322, upload-time = "2026-05-19T21:28:36.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/9b/b57afb22b386ae87ac9940f09878b98d8c333f89113e6fc96fcf4ca9eb64/yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294", size = 99057, upload-time = "2026-05-19T21:28:38.386Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4f/06348c27c8389256c313e8a57d796808fc0264c915dd5e7cfd3c0e314dc7/yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2", size = 113502, upload-time = "2026-05-19T21:28:40.091Z" }, + { url = "https://files.pythonhosted.org/packages/5f/1c/284f307b298e4a17b7943b07d9d7ecc4151537f8d137ba51f3bb6c31ca20/yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c", size = 105253, upload-time = "2026-05-19T21:28:41.987Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bf/0de123bec8619e45c80cbded9085f61b5b4a9eddb8abe6d25d28ee1ec866/yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b", size = 111345, upload-time = "2026-05-19T21:28:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/90/af/0248eb065e51129d2a9b2436cd1b5c772c19a6b04e5b6a186955671e3319/yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5", size = 106558, upload-time = "2026-05-19T21:28:45.806Z" }, + { url = "https://files.pythonhosted.org/packages/21/3c/f960d7a65ef97d8ba9b424fb5128796a4bc710fc6df2ddbbd7dfdc3bbd20/yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45", size = 92808, upload-time = "2026-05-19T21:28:48.465Z" }, + { url = "https://files.pythonhosted.org/packages/03/1a/49fb03750e4de4d2284cd5b885a383133c34eef45bd59631b2bb8b7e81e8/yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122", size = 87610, upload-time = "2026-05-19T21:28:50.07Z" }, + { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957, upload-time = "2026-05-19T21:28:51.695Z" }, + { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164, upload-time = "2026-05-19T21:28:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688, upload-time = "2026-05-19T21:28:54.865Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902, upload-time = "2026-05-19T21:28:56.963Z" }, + { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931, upload-time = "2026-05-19T21:28:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030, upload-time = "2026-05-19T21:29:00.148Z" }, + { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392, upload-time = "2026-05-19T21:29:02.155Z" }, + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487, upload-time = "2026-05-19T21:29:06.491Z" }, + { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333, upload-time = "2026-05-19T21:29:08.267Z" }, + { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025, upload-time = "2026-05-19T21:29:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507, upload-time = "2026-05-19T21:29:12.954Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719, upload-time = "2026-05-19T21:29:14.893Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438, upload-time = "2026-05-19T21:29:16.485Z" }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, + { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901, upload-time = "2026-05-19T21:29:20.014Z" }, + { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229, upload-time = "2026-05-19T21:29:22.1Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, ] [[package]] @@ -5607,12 +2959,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/10/9a/2fef89272d98b799e wheels = [ { url = "https://files.pythonhosted.org/packages/3b/7f/3a614b65bc4b181578b1d50a78663ee02d5d2d3b859712f3d3597c8afe6f/zc_lockfile-4.0-py3-none-any.whl", hash = "sha256:aa3aa295257bebaa09ea9ad5cb288bf9f98f88de6932f96b6659f62715d83581", size = 9143, upload-time = "2025-09-18T07:32:33.517Z" }, ] - -[[package]] -name = "zipp" -version = "3.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964ded15ab726fad40f25fd3d788fd741cc1c5a17d78ee8/zipp-3.23.1.tar.gz", hash = "sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110", size = 25965, upload-time = "2026-04-13T23:21:46.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, -]